You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2012/05/09 20:10:13 UTC

svn commit: r1336323 - in /lucene/dev/branches/lucene_solr_3_6: ./ lucene/ lucene/contrib/ lucene/contrib/facet/ lucene/contrib/spellchecker/ lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/ lucene/core/src/ lucene/core/src/te...

Author: simonw
Date: Wed May  9 18:10:12 2012
New Revision: 1336323

URL: http://svn.apache.org/viewvc?rev=1336323&view=rev
Log:
LUCENE-4022: Offline Sorter wrongly uses MIN_BUFFER_SIZE if there is more memory available

Modified:
    lucene/dev/branches/lucene_solr_3_6/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
    lucene/dev/branches/lucene_solr_3_6/lucene/contrib/facet/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/analysis/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/site/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java   (props changed)
    lucene/dev/branches/lucene_solr_3_6/lucene/tools/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/CHANGES.txt   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/NOTICE.txt   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/common-build.xml   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/contrib/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/core/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/example/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/lib/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/solrj/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/test-framework/   (props changed)
    lucene/dev/branches/lucene_solr_3_6/solr/testlogging.properties   (props changed)

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt?rev=1336323&r1=1336322&r2=1336323&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/CHANGES.txt Wed May  9 18:10:12 2012
@@ -21,6 +21,10 @@ Bug Fixes
    if skip is large. Fixed wrong final offset calculation in 
    PathHierarchyTokenizer. (Uwe Schindler, Mike McCandless, Robert Muir)
 
+  * LUCENE-4022: Offline Sorter used to build spellcheck & suggest dictionaries
+    wrongly used minimum sort buffer resulting in potentially slow FST &
+    Dictionary building. (Simon Willnauer) 
+
 Tests
  
  * LUCENE-3919, LUCENE-3969, LUCENE-3990: more thorough testing of analysis 

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java?rev=1336323&r1=1336322&r2=1336323&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/suggest/fst/Sort.java Wed May  9 18:10:12 2012
@@ -80,29 +80,35 @@ public final class Sort {
   
     /** 
      * Approximately half of the currently available free heap, but no less
-     * than {@link #MIN_BUFFER_SIZE_MB}. However if current heap allocation 
-     * is insufficient for sorting consult with max allowed heap size. 
+     * than {@link #ABSOLUTE_MIN_SORT_BUFFER_SIZE}. However if current heap allocation 
+     * is insufficient or if there is a large portion of unallocated heap-space available 
+     * for sorting consult with max allowed heap size. 
      */
     public static BufferSize automatic() {
       Runtime rt = Runtime.getRuntime();
       
       // take sizes in "conservative" order
-      long max = rt.maxMemory();
-      long total = rt.totalMemory();
-      long free = rt.freeMemory();
-
+      final long max = rt.maxMemory(); // max allocated
+      final long total = rt.totalMemory(); // currently allocated
+      final long free = rt.freeMemory(); // unused portion of currently allocated
+      final long totalAvailableBytes = max - total + free;
+      
       // by free mem (attempting to not grow the heap for this)
-      long half = free/2;
-      if (half >= ABSOLUTE_MIN_SORT_BUFFER_SIZE) { 
-        return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+      long sortBufferByteSize = free/2;
+      final long minBufferSizeBytes = MIN_BUFFER_SIZE_MB*MB;
+      if (sortBufferByteSize <  minBufferSizeBytes
+          || totalAvailableBytes > 10 * minBufferSizeBytes) { // lets see if we need/should to grow the heap 
+        if (totalAvailableBytes/2 > minBufferSizeBytes){ // there is enough mem for a reasonable buffer
+          sortBufferByteSize = totalAvailableBytes/2; // grow the heap
+        } else {
+          //heap seems smallish lets be conservative fall back to the free/2 
+          sortBufferByteSize = Math.max(ABSOLUTE_MIN_SORT_BUFFER_SIZE, sortBufferByteSize);
+        }
       }
-      
-      // by max mem (heap will grow)
-      half = (max - total) / 2;
-      return new BufferSize(Math.min(MIN_BUFFER_SIZE_MB * MB, half));
+      return new BufferSize(Math.min(Integer.MAX_VALUE, sortBufferByteSize));
     }
   }
-
+  
   /**
    * Sort info (debugging mostly).
    */
@@ -493,4 +499,4 @@ public final class Sort {
   public Comparator<BytesRef> getComparator() {
     return comparator;
   }  
-}
\ No newline at end of file
+}