You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/09 20:10:13 UTC
svn commit: r1336323 - in /lucene/dev/branches/lucene_solr_3_6: ./ lucene/
lucene/contrib/ lucene/contrib/facet/ lucene/contrib/spellchecker/
lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/
lucene/core/src/ lucene/core/src/te...
Author: simonw
Date: Wed May 9 18:10:12 2012
New Revision: 1336323
URL: http://svn.apache.org/viewvc?rev=1336323&view=rev
Log:
LUCENE-4022: Offline Sorter wrongly uses MIN_BUFFER_SIZE if there is more memory available
Modified:
lucene/dev/branches/lucene_solr_3_6/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/analysis/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/site/ (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java (props changed)
lucene/dev/branches/lucene_solr_3_6/lucene/tools/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/CHANGES.txt (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/common-build.xml (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/contrib/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/core/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/example/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/lib/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/solrj/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/test-framework/ (props changed)
lucene/dev/branches/lucene_solr_3_6/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt?rev=1336323&r1=1336322&r2=1336323&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt Wed May 9 18:10:12 2012
@@ -21,6 +21,10 @@ Bug Fixes
if skip is large. Fixed wrong final offset calculation in
PathHierarchyTokenizer. (Uwe Schindler, Mike McCandless, Robert Muir)
+ * LUCENE-4022: Offline Sorter used to build spellcheck & suggest dictionaries
+ wrongly used minimum sort buffer resulting in potentially slow FST &
+ Dictionary building. (Simon Willnauer)
+
Tests
* LUCENE-3919, LUCENE-3969, LUCENE-3990: more thorough testing of analysis
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java?rev=1336323&r1=1336322&r2=1336323&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java Wed May 9 18:10:12 2012
@@ -80,29 +80,35 @@ public final class Sort {
/**
* Approximately half of the currently available free heap, but no less
- * than {@link #MIN_BUFFER_SIZE_MB}. However if current heap allocation
- * is insufficient for sorting consult with max allowed heap size.
+ * than {@link #ABSOLUTE_MIN_SORT_BUFFER_SIZE}. However if current heap allocation
+ * is insufficient or if there is a large portion of unallocated heap-space available
+ * for sorting consult with max allowed heap size.
*/
public static BufferSize automatic() {
Runtime rt = Runtime.getRuntime();
// take sizes in "conservative" order
- long max = rt.maxMemory();
- long total = rt.totalMemory();
- long free = rt.freeMemory();
-
+ final long max = rt.maxMemory(); // max allocated
+ final long total = rt.totalMemory(); // currently allocated
+ final long free = rt.freeMemory(); // unused portion of currently allocated
+ final long totalAvailableBytes = max - total + free;
+
// by free mem (attempting to not grow the heap for this)
- long half = free/2;
- if (half >= ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
- return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+ long sortBufferByteSize = free/2;
+ final long minBufferSizeBytes = MIN_BUFFER_SIZE_MB*MB;
+ if (sortBufferByteSize < minBufferSizeBytes
+ || totalAvailableBytes > 10 * minBufferSizeBytes) { // lets see if we need/should to grow the heap
+ if (totalAvailableBytes/2 > minBufferSizeBytes){ // there is enough mem for a reasonable buffer
+ sortBufferByteSize = totalAvailableBytes/2; // grow the heap
+ } else {
+ //heap seems smallish lets be conservative fall back to the free/2
+ sortBufferByteSize = Math.max(ABSOLUTE_MIN_SORT_BUFFER_SIZE, sortBufferByteSize);
+ }
}
-
- // by max mem (heap will grow)
- half = (max - total) / 2;
- return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+ return new BufferSize(Math.min(Integer.MAX_VALUE, sortBufferByteSize));
}
}
-
+
/**
* Sort info (debugging mostly).
*/
@@ -493,4 +499,4 @@ public final class Sort {
public Comparator<BytesRef> getComparator() {
return comparator;
}
-}
\ No newline at end of file
+}