You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@commons.apache.org by bo...@apache.org on 2017/04/25 18:46:06 UTC
[06/10] commons-compress git commit: COMPRESS-382 and COMPRESS-386 --
take 4, clean up and allow for overflow via longs.
COMPRESS-382 and COMPRESS-386 -- take 4, clean up and allow
for overflow via longs.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo
Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/26eab98e
Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/26eab98e
Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/26eab98e
Branch: refs/heads/master
Commit: 26eab98e3a0d403587cc915eaa8f7d4b7b894cef
Parents: 0e8ff9c
Author: tballison <ta...@mitre.org>
Authored: Mon Apr 24 22:09:59 2017 -0400
Committer: tballison <ta...@mitre.org>
Committed: Mon Apr 24 22:09:59 2017 -0400
----------------------------------------------------------------------
.../apache/commons/compress/MemoryLimit.java | 15 +++++++++++
.../commons/compress/MemoryLimitException.java | 18 +++++++++-----
.../lzma/LZMACompressorInputStream.java | 2 +-
.../compressors/lzw/LZWInputStream.java | 26 ++++++++------------
.../compressors/xz/XZCompressorInputStream.java | 6 ++---
.../compressors/z/ZCompressorInputStream.java | 3 +--
6 files changed, 42 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimit.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimit.java b/src/main/java/org/apache/commons/compress/MemoryLimit.java
index e03e8e7..b43eb86 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimit.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimit.java
@@ -62,4 +62,19 @@ public class MemoryLimit {
public static int getMemoryLimitInKb() {
return MEMORY_LIMIT_IN_KB;
}
+
+ public static void checkLimitInKb(long memoryNeeded) throws MemoryLimitException {
+ if (memoryNeeded < 0) {
+ throw new IllegalArgumentException("MemoryLimit must be > -1");
+ }
+
+ if (memoryNeeded >> 10 > Integer.MAX_VALUE) {
+ throw new MemoryLimitException(memoryNeeded,
+ (MEMORY_LIMIT_IN_KB < 0) ? Integer.MAX_VALUE : MEMORY_LIMIT_IN_KB);
+ }
+
+ if (MEMORY_LIMIT_IN_KB > -1 && memoryNeeded > MEMORY_LIMIT_IN_KB) {
+ throw new MemoryLimitException(memoryNeeded, MEMORY_LIMIT_IN_KB);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimitException.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
index 8922ed2..a243a17 100644
--- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java
+++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java
@@ -26,15 +26,21 @@ import java.io.IOException;
* if a stream tries to allocate a byte array that is larger than
* the allowable limit.
*
+ * <p/>
+ * Set the global memory limit via {@link MemoryLimit#setMemoryLimitInKb(int)}.
+ *
* @since 1.14
*/
public class MemoryLimitException extends IOException {
- public MemoryLimitException(String message) {
- super(message);
- }
-
- public MemoryLimitException(String message, Exception e) {
- super(message, e);
+ /**
+ *
+ * @param memoryNeeded estimated memory needed
+ * @param memoryLimit memory limit applied
+ */
+ public MemoryLimitException(long memoryNeeded, int memoryLimit) {
+ super("" + memoryNeeded + " KiB of memory would be needed; limit was "
+ + memoryLimit + " KiB. If the file is not corrupt, consider " +
+ "increasing MemoryLimit.MEMORY_LIMIT_IN_KB.");
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
index 6931541..35d7e12 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java
@@ -56,7 +56,7 @@ public class LZMACompressorInputStream extends CompressorInputStream {
in = new LZMAInputStream(inputStream, MemoryLimit.getMemoryLimitInKb());
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress exception
- throw new MemoryLimitException("exceeded calculated memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
index 350b4b0..ed2c476 100644
--- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
+import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.MemoryLimitException;
import org.apache.commons.compress.compressors.CompressorInputStream;
import org.apache.commons.compress.utils.BitInputStream;
@@ -115,25 +116,18 @@ public abstract class LZWInputStream extends CompressorInputStream {
/**
* Initializes the arrays based on the maximum code size.
* @param maxCodeSize maximum code size
- * @param memoryLimitInKb maximum allowed table size in Kb
- * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb
+ *
+ * @throws MemoryLimitException
+ * if the calculated memory usage, based on the maxTableSize,
+ * is > {@link MemoryLimit#MEMORY_LIMIT_IN_KB}
*/
- protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb)
- throws MemoryLimitException {
+ protected void initializeTables(final int maxCodeSize) throws MemoryLimitException {
final int maxTableSize = 1 << maxCodeSize;
- if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) {
- throw new MemoryLimitException("Tried to allocate "+maxTableSize +
- " but memoryLimitInKb only allows "+(memoryLimitInKb*1024));
- }
- initializeTables(maxCodeSize);
- }
- /**
- * Initializes the arrays based on the maximum code size.
- * @param maxCodeSize maximum code size
- */
- protected void initializeTables(final int maxCodeSize) {
- final int maxTableSize = 1 << maxCodeSize;
+ //account for potential overflow
+ long memoryUsageInBytes = (long)maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack))
+ MemoryLimit.checkLimitInKb(memoryUsageInBytes >> 10);
+
prefixes = new int[maxTableSize];
characters = new byte[maxTableSize];
outputStack = new byte[maxTableSize];
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
index 3f977ef..e80cbcf 100644
--- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java
@@ -110,7 +110,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
count(ret == -1 ? -1 : 1);
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
@@ -122,7 +122,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return ret;
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Exceeded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
@@ -132,7 +132,7 @@ public class XZCompressorInputStream extends CompressorInputStream {
return in.skip(n);
} catch (org.tukaani.xz.MemoryLimitException e) {
//convert to commons-compress MemoryLimtException
- throw new MemoryLimitException("Excedded memory limit", e);
+ throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit());
}
}
http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
index ca61cd3..28b69a2 100644
--- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
+++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteOrder;
-import org.apache.commons.compress.MemoryLimit;
import org.apache.commons.compress.compressors.lzw.LZWInputStream;
/**
@@ -52,7 +51,7 @@ public class ZCompressorInputStream extends LZWInputStream {
if (blockMode) {
setClearCode(DEFAULT_CODE_SIZE);
}
- initializeTables(maxCodeSize, MemoryLimit.getMemoryLimitInKb());
+ initializeTables(maxCodeSize);
clearEntries();
}