You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/09/19 18:18:18 UTC
svn commit: r1387664 - in /lucene/dev/branches/lucene_solr_3_6/lucene: ./
core/src/java/org/apache/lucene/index/ core/src/test/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/index/
Author: mikemccand
Date: Wed Sep 19 16:18:18 2012
New Revision: 1387664
URL: http://svn.apache.org/viewvc?rev=1387664&view=rev
Log:
LUCENE-4398: fix over-counting bug causing too-early segment flushing when field names keep changing across documents
Modified:
lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/CHANGES.txt Wed Sep 19 16:18:18 2012
@@ -28,6 +28,12 @@ Bug Fixes:
had a custom Similarity where coord(1,1) != 1F, then the rewritten
query would be scored differently. (Robert Muir)
+* LUCENE-4398: If you index many different field names in your
+ documents then due to a bug in how it measures its RAM
+ usage, IndexWriter would flush each segment too early eventually
+ reaching the point where it flushes after every doc. (Tim Smith via
+ Mike McCandless)
+
Documentation
* LUCENE-4302: Fix facet userguide to have HTML loose doctype like
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Wed Sep 19 16:18:18 2012
@@ -24,4 +24,5 @@ abstract class DocFieldConsumerPerField
/** Processes all occurrences of a single field */
abstract void processFields(Fieldable[] fields, int count) throws IOException;
abstract void abort();
+ abstract void close();
}
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java Wed Sep 19 16:18:18 2012
@@ -50,7 +50,7 @@ final class DocFieldProcessor extends Do
public void flush(Collection<DocConsumerPerThread> threads, SegmentWriteState state) throws IOException {
Map<DocFieldConsumerPerThread, Collection<DocFieldConsumerPerField>> childThreadsAndFields = new HashMap<DocFieldConsumerPerThread, Collection<DocFieldConsumerPerField>>();
- for ( DocConsumerPerThread thread : threads) {
+ for (DocConsumerPerThread thread : threads) {
DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread) thread;
childThreadsAndFields.put(perThread.consumer, perThread.fields());
perThread.trimFields(state);
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Wed Sep 19 16:18:18 2012
@@ -146,6 +146,8 @@ final class DocFieldProcessorPerThread e
if (state.infoStream != null)
state.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ perField.consumer.close();
+
totalFieldCount--;
} else {
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java Wed Sep 19 16:18:18 2012
@@ -61,6 +61,11 @@ final class DocInverterPerField extends
}
@Override
+ public void close() {
+ consumer.close();
+ }
+
+ @Override
public void processFields(final Fieldable[] fields,
final int count) throws IOException {
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Wed Sep 19 16:18:18 2012
@@ -41,4 +41,6 @@ abstract class InvertedDocConsumerPerFie
// Called on hitting an aborting exception
abstract void abort();
+
+ abstract void close();
}
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java Wed Sep 19 16:18:18 2012
@@ -79,6 +79,26 @@ final class TermsHashPerField extends In
nextPerField = null;
}
+ @Override
+ public void close() {
+ if (perThread.termsHash.trackAllocations) {
+ try {
+ if (postingsHash != null) {
+ bytesUsed(-postingsHash.length * RamUsageEstimator.NUM_BYTES_INT);
+ postingsHash = null;
+ }
+ if (postingsArray != null) {
+ bytesUsed(-postingsArray.bytesPerPosting() * postingsArray.size);
+ postingsArray = null;
+ }
+ } finally {
+ if (nextPerField != null) {
+ nextPerField.close();
+ }
+ }
+ }
+ }
+
private void initPostingsArray() {
postingsArray = consumer.createPostingsArray(2);
bytesUsed(postingsArray.size * postingsArray.bytesPerPosting());
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Wed Sep 19 16:18:18 2012
@@ -1909,4 +1909,53 @@ public class TestIndexWriter extends Luc
r.close();
dir.close();
}
+
+ // LUCENE-4398
+ public void testRotatingFieldNames() throws Exception {
+ Directory dir = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testChangingFields"));
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ iwc.setRAMBufferSizeMB(0.2);
+ iwc.setMaxBufferedDocs(-1);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ int upto = 0;
+
+ int secondFlushDocCount = -1;
+ for(int iter=0;iter<10;iter++) {
+ final int startFlushCount = w.getFlushCount();
+ int docCount = 0;
+ while(w.getFlushCount() == startFlushCount) {
+ Document doc = new Document();
+ for(int i=0;i<10;i++) {
+ Field f = new Field("field" + (upto++), "content", Field.Store.NO, Field.Index.ANALYZED);
+ f.setOmitNorms(true);
+ doc.add(f);
+ }
+ w.addDocument(doc);
+ docCount++;
+ }
+
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter + " flushed after docCount=" + docCount);
+ }
+
+ if (iter == 1) {
+ // Use 2nd not 1st flush because 1st flush is
+ // "unusually" high because all allocations are
+ // fresh:
+ secondFlushDocCount = docCount;
+ } else if (iter > 1) {
+ assertTrue("flushed after too few docs: 2nd segment flushed at docCount=" + secondFlushDocCount + ", but current segment flushed after docCount=" + docCount + "; iter=" + iter, ((float) docCount) / secondFlushDocCount > 0.9);
+ }
+
+ if (upto > 5000) {
+ // Start re-using field names after a while
+ // ... important because otherwise we can OOME due
+ // to too many FieldInfo instances.
+ upto = 0;
+ }
+ }
+
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1387664&r1=1387663&r2=1387664&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene_solr_3_6/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Wed Sep 19 16:18:18 2012
@@ -90,7 +90,7 @@ public class RandomIndexWriter implement
w = new MockIndexWriter(r, dir, c);
flushAt = _TestUtil.nextInt(r, 10, 1000);
if (LuceneTestCase.VERBOSE) {
- System.out.println("RIW config=" + w.getConfig());
+ System.out.println("RIW dir=" + dir + " config=" + w.getConfig());
}
// Make sure we sometimes test indices that don't get