You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/23 21:47:40 UTC
svn commit: r1341999 - in /lucene/dev/branches/lucene_solr_3_6: ./ lucene/
lucene/contrib/ lucene/contrib/facet/ lucene/contrib/spellchecker/
lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/
lucene/contrib/spellchecker/src/tes...
Author: simonw
Date: Wed May 23 19:47:39 2012
New Revision: 1341999
URL: http://svn.apache.org/viewvc?rev=1341999&view=rev
Log:
LUCENE-4074: FST Sorter BufferSize causes int overflow if BufferSize > 2048MB
Modified:
lucene/dev/branches/lucene_solr_3_6/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/site/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/tools/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/CHANGES.txt (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/example/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/lib/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt?rev=1341999&r1=1341998&r2=1341999&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt Wed May 23 19:47:39 2012
@@ -20,15 +20,19 @@ Bug Fixes
to reset() all state. Protect against AIOOBE in ReversePathHierarchyTokenizer
if skip is large. Fixed wrong final offset calculation in
PathHierarchyTokenizer. (Uwe Schindler, Mike McCandless, Robert Muir)
-
- * LUCENE-4022: Offline Sorter used to build spellcheck & suggest dictionaries
- wrongly used minimum sort buffer resulting in potentially slow FST &
- Dictionary building. (Simon Willnauer)
+
+ * LUCENE-4022: Offline Sorter used to build spellcheck & suggest dictionaries
+ wrongly used minimum sort buffer resulting in potentially slow FST &
+ Dictionary building. (Simon Willnauer)
* LUCENE-4060: Fix a synchronization bug in
DirectoryTaxonomyWriter.addTaxonomies(). Also, the method has been renamed to
addTaxonomy and now takes only one Directory and one OrdinalMap.
(Shai Erera, Gilad Barkai)
+
+ * LUCENE-4074: FST Sorter BufferSize creates a negative buffer size due to an
+ 32 bit signed integer overflow if BufferSize >= 2048MB is passed.
+ (Simon Willnauer)
Tests
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java?rev=1341999&r1=1341998&r2=1341999&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java Wed May 23 19:47:39 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.search.suggest.fst;
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -37,18 +37,18 @@ import org.apache.lucene.util.PriorityQu
* @lucene.internal
*/
public final class Sort {
- public final static int MB = 1024 * 1024;
- public final static int GB = MB * 1024;
+ public final static long MB = 1024 * 1024;
+ public final static long GB = MB * 1024;
/**
* Minimum recommended buffer size for sorting.
*/
- public final static int MIN_BUFFER_SIZE_MB = 32;
+ public final static long MIN_BUFFER_SIZE_MB = 32;
/**
* Absolute minimum required buffer size for sorting.
*/
- public static final int ABSOLUTE_MIN_SORT_BUFFER_SIZE = MB / 2;
+ public static final long ABSOLUTE_MIN_SORT_BUFFER_SIZE = MB / 2;
private static final String MIN_BUFFER_SIZE_MSG = "At least 0.5MB RAM buffer is needed";
/**
@@ -60,7 +60,7 @@ public final class Sort {
* A bit more descriptive unit for constructors.
*
* @see #automatic()
- * @see #megabytes(int)
+ * @see #megabytes(long)
*/
public static final class BufferSize {
final int bytes;
@@ -70,11 +70,19 @@ public final class Sort {
throw new IllegalArgumentException("Buffer too large for Java ("
+ (Integer.MAX_VALUE / MB) + "mb max): " + bytes);
}
+
+ if (bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
+ throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + bytes);
+ }
this.bytes = (int) bytes;
}
-
- public static BufferSize megabytes(int mb) {
+
+ /**
+ * Creates a {@link BufferSize} in MB. The given
+ * values must be $gt; 0 and < 2048.
+ */
+ public static BufferSize megabytes(long mb) {
return new BufferSize(mb * MB);
}
@@ -105,7 +113,7 @@ public final class Sort {
sortBufferByteSize = Math.max(ABSOLUTE_MIN_SORT_BUFFER_SIZE, sortBufferByteSize);
}
}
- return new BufferSize(Math.min(Integer.MAX_VALUE, sortBufferByteSize));
+ return new BufferSize(Math.min((long)Integer.MAX_VALUE, sortBufferByteSize));
}
}
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/fst/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/fst/TestSort.java?rev=1341999&r1=1341998&r2=1341999&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/fst/TestSort.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/fst/TestSort.java Wed May 23 19:47:39 2012
@@ -63,7 +63,7 @@ public class TestSort extends LuceneTest
public void testIntermediateMerges() throws Exception {
// Sort 20 mb worth of data with 1mb buffer, binary merging.
SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2),
- generateRandom(Sort.MB * 20));
+ generateRandom((int)Sort.MB * 20));
assertTrue(info.mergeRounds > 10);
}
@@ -71,7 +71,7 @@ public class TestSort extends LuceneTest
public void testSmallRandom() throws Exception {
// Sort 20 mb worth of data with 1mb buffer.
SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
- generateRandom(Sort.MB * 20));
+ generateRandom((int)Sort.MB * 20));
assertEquals(1, sortInfo.mergeRounds);
}
@@ -79,7 +79,7 @@ public class TestSort extends LuceneTest
public void testLargerRandom() throws Exception {
// Sort 100MB worth of data with 15mb buffer.
checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES),
- generateRandom(Sort.MB * 100));
+ generateRandom((int)Sort.MB * 100));
}
private byte[][] generateRandom(int howMuchData) {
@@ -152,4 +152,31 @@ public class TestSort extends LuceneTest
w.close();
return file;
}
+
+ public void testRamBuffer() {
+ int numIters = atLeast(10000);
+ for (int i = 0; i < numIters; i++) {
+ BufferSize.megabytes(1+random().nextInt(2047));
+ }
+ BufferSize.megabytes(2047);
+ BufferSize.megabytes(1);
+
+ try {
+ BufferSize.megabytes(2048);
+ fail("max mb is 2047");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ BufferSize.megabytes(0);
+ fail("min mb is 0.5");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ BufferSize.megabytes(-1);
+ fail("min mb is 0.5");
+ } catch (IllegalArgumentException e) {
+ }
+ }
}