You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/09/19 18:18:18 UTC

svn commit: r1387664 - in /lucene/dev/branches/lucene_solr_3_6/lucene: ./ core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/ test-framework/src/java/org/apache/lucene/index/

Author: mikemccand
Date: Wed Sep 19 16:18:18 2012
New Revision: 1387664

URL: http://svn.apache.org/viewvc?rev=1387664&view=rev
Log:
LUCENE-4398: fix over-counting bug causing too-early segment flushing when field names keep changing across documents

Modified:
    lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
    lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt Wed Sep 19 16:18:18 2012
@@ -28,6 +28,12 @@ Bug Fixes:
   had a custom Similarity where coord(1,1) != 1F, then the rewritten
   query would be scored differently.  (Robert Muir)
 
+* LUCENE-4398: If you index many different field names in your
+  documents then due to a bug in how it measures its RAM
+  usage, IndexWriter would flush each segment too early eventually
+  reaching the point where it flushes after every doc.  (Tim Smith via
+  Mike McCandless)
+
 Documentation
 
 * LUCENE-4302: Fix facet userguide to have HTML loose doctype like

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Wed Sep 19 16:18:18 2012
@@ -24,4 +24,5 @@ abstract class DocFieldConsumerPerField 
   /** Processes all occurrences of a single field */
   abstract void processFields(Fieldable[] fields, int count) throws IOException;
   abstract void abort();
+  abstract void close();
 }

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Wed Sep 19 16:18:18 2012
@@ -50,7 +50,7 @@ final class DocFieldProcessor extends Do
   public void flush(Collection<DocConsumerPerThread> threads, SegmentWriteState state) throws IOException {
 
     Map<DocFieldConsumerPerThread, Collection<DocFieldConsumerPerField>> childThreadsAndFields = new HashMap<DocFieldConsumerPerThread, Collection<DocFieldConsumerPerField>>();
-    for ( DocConsumerPerThread thread : threads) {
+    for (DocConsumerPerThread thread : threads) {
       DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread) thread;
       childThreadsAndFields.put(perThread.consumer, perThread.fields());
       perThread.trimFields(state);

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Wed Sep 19 16:18:18 2012
@@ -146,6 +146,8 @@ final class DocFieldProcessorPerThread e
           if (state.infoStream != null)
             state.infoStream.println("  purge field=" + perField.fieldInfo.name);
 
+          perField.consumer.close();
+
           totalFieldCount--;
 
         } else {

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Wed Sep 19 16:18:18 2012
@@ -61,6 +61,11 @@ final class DocInverterPerField extends 
   }
 
   @Override
+  public void close() {
+    consumer.close();
+  }
+
+  @Override
   public void processFields(final Fieldable[] fields,
                             final int count) throws IOException {
 

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Wed Sep 19 16:18:18 2012
@@ -41,4 +41,6 @@ abstract class InvertedDocConsumerPerFie
 
   // Called on hitting an aborting exception
   abstract void abort();
+
+  abstract void close();
 }

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java Wed Sep 19 16:18:18 2012
@@ -79,6 +79,26 @@ final class TermsHashPerField extends In
       nextPerField = null;
   }
 
+  @Override
+  public void close() {
+    if (perThread.termsHash.trackAllocations) {
+      try {
+        if (postingsHash != null) {
+          bytesUsed(-postingsHash.length * RamUsageEstimator.NUM_BYTES_INT);
+          postingsHash = null;
+        }
+        if (postingsArray != null) {
+          bytesUsed(-postingsArray.bytesPerPosting() * postingsArray.size);
+          postingsArray = null;
+        }
+      } finally {
+        if (nextPerField != null) {
+          nextPerField.close();
+        }
+      }
+    }
+  }
+
   private void initPostingsArray() {
     postingsArray = consumer.createPostingsArray(2);
     bytesUsed(postingsArray.size * postingsArray.bytesPerPosting());

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Sep 19 16:18:18 2012
@@ -1909,4 +1909,53 @@ public class TestIndexWriter extends Luc
     r.close();
     dir.close();
   }
+
+  // LUCENE-4398
+  public void testRotatingFieldNames() throws Exception {
+    Directory dir = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testChangingFields"));
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+    iwc.setRAMBufferSizeMB(0.2);
+    iwc.setMaxBufferedDocs(-1);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    int upto = 0;
+
+    int secondFlushDocCount = -1;
+    for(int iter=0;iter<10;iter++) {
+      final int startFlushCount = w.getFlushCount();
+      int docCount = 0;
+      while(w.getFlushCount() == startFlushCount) {
+        Document doc = new Document();
+        for(int i=0;i<10;i++) {
+          Field f = new Field("field" + (upto++), "content", Field.Store.NO, Field.Index.ANALYZED);
+          f.setOmitNorms(true);
+          doc.add(f);
+        }
+        w.addDocument(doc);
+        docCount++;
+      }
+
+      if (VERBOSE) {
+        System.out.println("TEST: iter=" + iter + " flushed after docCount=" + docCount);
+      }
+
+      if (iter == 1) {
+        // Use 2nd not 1st flush because 1st flush is
+        // "unusually" high because all allocations are
+        // fresh:
+        secondFlushDocCount = docCount;
+      } else if (iter > 1) {
+        assertTrue("flushed after too few docs: 2nd segment flushed at docCount=" + secondFlushDocCount + ", but current segment flushed after docCount=" + docCount + "; iter=" + iter, ((float) docCount) / secondFlushDocCount > 0.9);
+      }
+
+      if (upto > 5000) {
+        // Start re-using field names after a while
+        // ... important because otherwise we can OOME due
+        // to too many FieldInfo instances.
+        upto = 0;
+      }
+    }
+
+    w.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Sep 19 16:18:18 2012
@@ -90,7 +90,7 @@ public class RandomIndexWriter implement
     w = new MockIndexWriter(r, dir, c);
     flushAt = _TestUtil.nextInt(r, 10, 1000);
     if (LuceneTestCase.VERBOSE) {
-      System.out.println("RIW config=" + w.getConfig());
+      System.out.println("RIW dir=" + dir + " config=" + w.getConfig());
     }
 
     // Make sure we sometimes test indices that don't get