You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/23 21:38:26 UTC
svn commit: r1341995 - in /lucene/dev/trunk/lucene/suggest/src:
java/org/apache/lucene/search/suggest/fst/Sort.java
test/org/apache/lucene/search/suggest/fst/TestSort.java
Author: simonw
Date: Wed May 23 19:38:26 2012
New Revision: 1341995
URL: http://svn.apache.org/viewvc?rev=1341995&view=rev
Log:
LUCENE-4074: FST Sorter BufferSize causes int overflow if BufferSize > 2048MB
Modified:
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java?rev=1341995&r1=1341994&r2=1341995&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java Wed May 23 19:38:26 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.search.suggest.fst;
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -37,18 +37,18 @@ import org.apache.lucene.util.PriorityQu
* @lucene.internal
*/
public final class Sort {
- public final static int MB = 1024 * 1024;
- public final static int GB = MB * 1024;
+ public final static long MB = 1024 * 1024;
+ public final static long GB = MB * 1024;
/**
* Minimum recommended buffer size for sorting.
*/
- public final static int MIN_BUFFER_SIZE_MB = 32;
+ public final static long MIN_BUFFER_SIZE_MB = 32;
/**
* Absolute minimum required buffer size for sorting.
*/
- public static final int ABSOLUTE_MIN_SORT_BUFFER_SIZE = MB / 2;
+ public static final long ABSOLUTE_MIN_SORT_BUFFER_SIZE = MB / 2;
private static final String MIN_BUFFER_SIZE_MSG = "At least 0.5MB RAM buffer is needed";
/**
@@ -60,7 +60,7 @@ public final class Sort {
* A bit more descriptive unit for constructors.
*
* @see #automatic()
- * @see #megabytes(int)
+ * @see #megabytes(long)
*/
public static final class BufferSize {
final int bytes;
@@ -70,11 +70,19 @@ public final class Sort {
throw new IllegalArgumentException("Buffer too large for Java ("
+ (Integer.MAX_VALUE / MB) + "mb max): " + bytes);
}
+
+ if (bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
+ throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + bytes);
+ }
this.bytes = (int) bytes;
}
-
- public static BufferSize megabytes(int mb) {
+
+ /**
+ * Creates a {@link BufferSize} in MB. The given
+ * values must be $gt; 0 and < 2048.
+ */
+ public static BufferSize megabytes(long mb) {
return new BufferSize(mb * MB);
}
@@ -105,7 +113,7 @@ public final class Sort {
sortBufferByteSize = Math.max(ABSOLUTE_MIN_SORT_BUFFER_SIZE, sortBufferByteSize);
}
}
- return new BufferSize(Math.min(Integer.MAX_VALUE, sortBufferByteSize));
+ return new BufferSize(Math.min((long)Integer.MAX_VALUE, sortBufferByteSize));
}
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java?rev=1341995&r1=1341994&r2=1341995&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java Wed May 23 19:38:26 2012
@@ -63,7 +63,7 @@ public class TestSort extends LuceneTest
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2),
- generateRandom(Sort.MB * 20));
+ generateRandom((int)Sort.MB * 20));
assertTrue(info.mergeRounds > 10);
}
@@ -71,7 +71,7 @@ public class TestSort extends LuceneTest
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
- generateRandom(Sort.MB * 20));
+ generateRandom((int)Sort.MB * 20));
assertEquals(1, sortInfo.mergeRounds);
}
@@ -79,7 +79,7 @@ public class TestSort extends LuceneTest
public void testLargerRandom() throws Exception {
// Sort 100MB worth of data with 15mb buffer.
checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
- generateRandom(Sort.MB * 100));
+ generateRandom((int)Sort.MB * 100));
}
private byte[][] generateRandom(int howMuchData) {
@@ -152,4 +152,31 @@ public class TestSort extends LuceneTest
w.close();
return file;
}
+
+ public void testRamBuffer() {
+ int numIters = atLeast(10000);
+ for (int i = 0; i < numIters; i++) {
+ BufferSize.megabytes(1+random().nextInt(2047));
+ }
+ BufferSize.megabytes(2047);
+ BufferSize.megabytes(1);
+
+ try {
+ BufferSize.megabytes(2048);
+ fail("max mb is 2047");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ BufferSize.megabytes(0);
+ fail("min mb is 0.5");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ BufferSize.megabytes(-1);
+ fail("min mb is 0.5");
+ } catch (IllegalArgumentException e) {
+ }
+ }
}