You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/10 01:32:34 UTC

svn commit: r1593649 - in /lucene/dev/branches/lucene_solr_4_8/lucene/core/src: java/org/apache/lucene/index/ test/org/apache/lucene/index/

Author: mikemccand
Date: Fri May  9 23:32:34 2014
New Revision: 1593649

URL: http://svn.apache.org/r1593649
Log:
LUCENE-5644: favor an already initialized ThreadState

Modified:
    lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
    lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
    lucene/dev/branches/lucene_solr_4_8/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java

Modified: lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1593649&r1=1593648&r2=1593649&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Fri May  9 23:32:34 2014
@@ -263,6 +263,27 @@ final class DocumentsWriterPerThreadPool
           // Important that we are LIFO here! This way if number of concurrent indexing threads was once high, but has now reduced, we only use a
           // limited number of thread states:
           threadState = freeList[freeCount-1];
+
+          if (threadState.dwpt == null) {
+            // This thread-state is not initialized, e.g. it
+            // was just flushed. See if we can instead find
+            // another free thread state that already has docs
+            // indexed. This way if incoming thread concurrency
+            // has decreased, we don't leave docs
+            // indefinitely buffered, tying up RAM.  This
+            // will instead get those thread states flushed,
+            // freeing up RAM for larger segment flushes:
+            for(int i=0;i<freeCount;i++) {
+              if (freeList[i].dwpt != null) {
+                // Use this one instead, and swap it with
+                // the un-initialized one:
+                ThreadState ts = freeList[i];
+                freeList[i] = threadState;
+                threadState = ts;
+                break;
+              }
+            }
+          }
           freeCount--;
           break;
         } else if (numThreadStatesActive < threadStates.length) {

Modified: lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1593649&r1=1593648&r2=1593649&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene_solr_4_8/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Fri May  9 23:32:34 2014
@@ -17,9 +17,20 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.DocValuesProducer;
+import org.apache.lucene.codecs.FieldInfosFormat;
 import org.apache.lucene.codecs.StoredFieldsReader;
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.index.FieldInfo.DocValuesType;
@@ -31,16 +42,6 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.CloseableThreadLocal;
 import org.apache.lucene.util.IOUtils;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.IdentityHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
 /**
  * IndexReader implementation over a single segment. 
  * <p>
@@ -218,7 +219,9 @@ public final class SegmentReader extends
     
     try {
       final String segmentSuffix = info.getFieldInfosGen() == -1 ? "" : Long.toString(info.getFieldInfosGen(), Character.MAX_RADIX);
-      return info.info.getCodec().fieldInfosFormat().getFieldInfosReader().read(dir, info.info.name, segmentSuffix, IOContext.READONCE);
+      Codec codec = info.info.getCodec();
+      FieldInfosFormat fisFormat = codec.fieldInfosFormat();
+      return fisFormat.getFieldInfosReader().read(dir, info.info.name, segmentSuffix, IOContext.READONCE);
     } finally {
       if (closeDir) {
         dir.close();

Modified: lucene/dev/branches/lucene_solr_4_8/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene_solr_4_8/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java?rev=1593649&r1=1593648&r2=1593649&view=diff
==============================================================================
--- lucene/dev/branches/lucene_solr_4_8/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java (original)
+++ lucene/dev/branches/lucene_solr_4_8/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterThreadsToSegments.java Fri May  9 23:32:34 2014
@@ -19,16 +19,21 @@ package org.apache.lucene.index;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.lucene46.Lucene46SegmentInfoFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;
@@ -266,4 +271,74 @@ public class TestIndexWriterThreadsToSeg
     }
     dir.close();
   }
+
+  public void testDocsStuckInRAMForever() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    iwc.setRAMBufferSizeMB(.2);
+    Codec codec = Codec.forName("Lucene46");
+    iwc.setCodec(codec);
+    iwc.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+    final IndexWriter w = new IndexWriter(dir, iwc);
+    final CountDownLatch startingGun = new CountDownLatch(1);
+    Thread[] threads = new Thread[2];
+    for(int i=0;i<threads.length;i++) {
+      final int threadID = i;
+      threads[i] = new Thread() {
+          @Override
+          public void run() {
+            try {
+              startingGun.await();
+              for(int j=0;j<1000;j++) {
+                Document doc = new Document();
+                doc.add(newStringField("field", "threadID" + threadID, Field.Store.NO));
+                w.addDocument(doc);
+              }
+            } catch (Exception e) {
+              throw new RuntimeException(e);
+            }
+          }
+        };
+      threads[i].start();
+    }
+
+    startingGun.countDown();
+    for(Thread t : threads) {
+      t.join();
+    }
+
+    Set<String> segSeen = new HashSet<>();
+    int thread0Count = 0;
+    int thread1Count = 0;
+
+    // At this point the writer should have 2 thread states w/ docs; now we index with only 1 thread until we see all 1000 thread0 & thread1
+    // docs flushed.  If the writer incorrectly holds onto previously indexed docs forever then this will run forever:
+    while (thread0Count < 1000 || thread1Count < 1000) {
+      Document doc = new Document();
+      doc.add(newStringField("field", "threadIDmain", Field.Store.NO));
+      w.addDocument(doc);
+
+      for(String fileName : dir.listAll()) {
+        if (fileName.endsWith(".si")) {
+          String segName = IndexFileNames.parseSegmentName(fileName);
+          if (segSeen.contains(segName) == false) {
+            segSeen.add(segName);
+            SegmentInfo si = new Lucene46SegmentInfoFormat().getSegmentInfoReader().read(dir, segName, IOContext.DEFAULT);
+            si.setCodec(codec);
+            SegmentCommitInfo sci = new SegmentCommitInfo(si, 0, -1, -1);
+            SegmentReader sr = new SegmentReader(sci, 1, IOContext.DEFAULT);
+            try {
+              thread0Count += sr.docFreq(new Term("field", "threadID0"));
+              thread1Count += sr.docFreq(new Term("field", "threadID1"));
+            } finally {
+              sr.close();
+            }
+          }
+        }
+      }
+    }
+
+    w.close();
+    dir.close();
+  }
 }