You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/07/23 21:56:28 UTC
svn commit: r1612935 - in /lucene/dev/trunk/lucene: CHANGES.txt
core/src/java/org/apache/lucene/util/ArrayUtil.java
core/src/java/org/apache/lucene/util/PriorityQueue.java
core/src/test/org/apache/lucene/util/TestArrayUtil.java
Author: mikemccand
Date: Wed Jul 23 19:56:27 2014
New Revision: 1612935
URL: http://svn.apache.org/r1612935
Log:
LUCENE-5844: ArrayUtil.grow/oversize now returns at most Integer.MAX_VALUE - 8
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1612935&r1=1612934&r2=1612935&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed Jul 23 19:56:27 2014
@@ -205,6 +205,10 @@ Bug Fixes
* LUCENE-5838: Fix hunspell when the .aff file has over 64k affixes. (Robert Muir)
+* LUCENE-5844: ArrayUtil.grow/oversize now returns a maximum of
+ Integer.MAX_VALUE - 8 for the maximum array size. (Robert Muir,
+ Mike McCandless)
+
Test Framework
* LUCENE-5786: Unflushed/ truncated events file (hung testing subprocess).
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1612935&r1=1612934&r2=1612935&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/ArrayUtil.java Wed Jul 23 19:56:27 2014
@@ -28,13 +28,10 @@ import java.util.Comparator;
public final class ArrayUtil {
- /** Maximum length for an array; we set this to "a
- * bit" below Integer.MAX_VALUE because the exact max
- * allowed byte[] is JVM dependent, so we want to avoid
- * a case where a large value worked during indexing on
- * one JVM but failed later at search time with a
- * different JVM. */
- public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 256;
+ /** Maximum length for an array (Integer.MAX_VALUE - 8). stackoverflow
+ * consensus seems to be this value and it's also what ArrayList.java
+ * uses as its limit. */
+ public static final int MAX_ARRAY_LENGTH = Integer.MAX_VALUE - 8;
private ArrayUtil() {} // no instance
@@ -169,6 +166,10 @@ public final class ArrayUtil {
return 0;
}
+ if (minTargetSize > MAX_ARRAY_LENGTH) {
+ throw new IllegalArgumentException("requested array size " + minTargetSize + " exceeds maximum array in java (" + MAX_ARRAY_LENGTH + ")");
+ }
+
// asymptotic exponential growth by 1/8th, favors
// spending a bit more CPU to not tie up too much wasted
// RAM:
@@ -184,9 +185,9 @@ public final class ArrayUtil {
int newSize = minTargetSize + extra;
// add 7 to allow for worst case byte alignment addition below:
- if (newSize+7 < 0) {
- // int overflowed -- return max allowed array size
- return Integer.MAX_VALUE;
+ if (newSize+7 < 0 || newSize+7 > MAX_ARRAY_LENGTH) {
+ // int overflowed, or we exceeded the maximum array length
+ return MAX_ARRAY_LENGTH;
}
if (Constants.JRE_IS_64BIT) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java?rev=1612935&r1=1612934&r2=1612935&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/PriorityQueue.java Wed Jul 23 19:56:27 2014
@@ -43,21 +43,13 @@ public abstract class PriorityQueue<T> {
// We allocate 1 extra to avoid if statement in top()
heapSize = 2;
} else {
- if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
- // Don't wrap heapSize to -1, in this case, which
- // causes a confusing NegativeArraySizeException.
- // Note that very likely this will simply then hit
- // an OOME, but at least that's more indicative to
- // caller that this values is too big. We don't +1
- // in this case, but it's very unlikely in practice
- // one will actually insert this many objects into
- // the PQ:
+ // NOTE: we add +1 because all access to heap is
+ // 1-based not 0-based. heap[0] is unused.
+ heapSize = maxSize + 1;
+
+ if (heapSize > ArrayUtil.MAX_ARRAY_LENGTH) {
// Throw exception to prevent confusing OOME:
- throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
- } else {
- // NOTE: we add +1 because all access to heap is
- // 1-based not 0-based. heap[0] is unused.
- heapSize = maxSize + 1;
+ throw new IllegalArgumentException("maxSize must be <= " + (ArrayUtil.MAX_ARRAY_LENGTH-1) + "; got: " + maxSize);
}
}
// T is unbounded type, so this unchecked cast works always:
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java?rev=1612935&r1=1612934&r2=1612935&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestArrayUtil.java Wed Jul 23 19:56:27 2014
@@ -29,7 +29,7 @@ public class TestArrayUtil extends Lucen
long copyCost = 0;
// Make sure ArrayUtil hits Integer.MAX_VALUE, if we insist:
- while(currentSize != Integer.MAX_VALUE) {
+ while (currentSize != ArrayUtil.MAX_ARRAY_LENGTH) {
int nextSize = ArrayUtil.oversize(1+currentSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
assertTrue(nextSize > currentSize);
if (currentSize > 0) {
@@ -44,11 +44,24 @@ public class TestArrayUtil extends Lucen
public void testMaxSize() {
// intentionally pass invalid elemSizes:
for(int elemSize=0;elemSize<10;elemSize++) {
- assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE, elemSize));
- assertEquals(Integer.MAX_VALUE, ArrayUtil.oversize(Integer.MAX_VALUE-1, elemSize));
+ assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH, elemSize));
+ assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH-1, elemSize));
}
}
+ public void testTooBig() {
+ try {
+ ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH+1, 1);
+ fail("did not hit exception");
+ } catch (IllegalArgumentException iae) {
+ // expected
+ }
+ }
+
+ public void testExactLimit() {
+ assertEquals(ArrayUtil.MAX_ARRAY_LENGTH, ArrayUtil.oversize(ArrayUtil.MAX_ARRAY_LENGTH, 1));
+ }
+
public void testInvalidElementSizes() {
final Random rnd = random();
final int num = atLeast(10000);