svn commit: r948516 - in /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index: DocumentsWriter.java IndexWriter.java TermsHashPerField.java

[mi...@apache.org]

Author: mikemccand
Date: Wed May 26 18:14:15 2010
New Revision: 948516

URL: http://svn.apache.org/viewvc?rev=948516&view=rev
Log:
LUCENE-2467: more mem reduction during indexing

Modified:
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHashPerField.java

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=948516&r1=948515&r2=948516&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/DocumentsWriter.java Wed May 26 18:14:15 2010
@@ -160,6 +160,13 @@ final class DocumentsWriter {
     public boolean testPoint(String name) {
       return docWriter.writer.testPoint(name);
     }
+
+    public void clear() {
+      // don't hold onto doc nor analyzer, in case it is
+      // largish:
+      doc = null;
+      analyzer = null;
+    }
   }
 
   /** Consumer returns this on each doc.  This holds any
@@ -819,10 +826,16 @@ final class DocumentsWriter {
     try {
       // This call is not synchronized and does all the
       // work
-      final DocWriter perDoc = state.consumer.processDocument();
-        
+      final DocWriter perDoc;
+      try {
+        perDoc = state.consumer.processDocument();
+      } finally {
+        docState.clear();
+      }
+
       // This call is synchronized but fast
       finishDocument(state, perDoc);
+
       success = true;
     } finally {
       if (!success) {

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java?rev=948516&r1=948515&r2=948516&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/IndexWriter.java Wed May 26 18:14:15 2010
@@ -4244,6 +4244,9 @@ public class IndexWriter {
     // Make sure no threads are actively adding a document.
     // Returns true if docWriter is currently aborting, in
     // which case we skip flushing this segment
+    if (infoStream != null) {
+      message("flush: now pause all indexing threads");
+    }
     if (docWriter.pauseAllThreads()) {
       docWriter.resumeAllThreads();
       return false;

Modified: lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=948516&r1=948515&r2=948516&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/index/TermsHashPerField.java Wed May 26 18:14:15 2010
@@ -71,19 +71,14 @@ final class TermsHashPerField extends In
   void shrinkHash(int targetSize) {
     assert postingsCompacted || numPostings == 0;
 
-    // Cannot use ArrayUtil.shrink because we require power
-    // of 2:
-    int newSize = postingsHash.length;
-    while(newSize >= 8 && newSize/4 > targetSize) {
-      newSize /= 2;
-    }
-
+    final int newSize = 4;
     if (newSize != postingsHash.length) {
       postingsHash = new RawPostingList[newSize];
       postingsHashSize = newSize;
       postingsHashHalfSize = newSize/2;
       postingsHashMask = newSize-1;
     }
+    Arrays.fill(postingsHash, null);
   }
 
   public void reset() {