You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/06/14 18:07:32 UTC

svn commit: r1135669 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test-framework/ lucene/backwards/src/test/ lucene/contrib/ lucene/contrib/misc/src/java/org/apache/lucene/index/ lucene/contrib/misc/src/test/or...

Author: mikemccand
Date: Tue Jun 14 16:07:32 2011
New Revision: 1135669

URL: http://svn.apache.org/viewvc?rev=1135669&view=rev
Log:
LUCENE-2955: add NRTManager/ReopenThread for simple control over NRT reopening and controlled visiblity of specific indexing changes

Added:
    lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java
      - copied, changed from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java
    lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManagerReopenThread.java
      - copied unchanged from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManagerReopenThread.java
    lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
      - copied, changed from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/   (props changed)
    lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Tue Jun 14 16:07:32 2011
@@ -20,6 +20,11 @@ New Features
    document sharing the same group was indexed as a doc block
    (IndexWriter.add/updateDocuments).  (Mike McCandless)
 
+ * LUCENE-2955: Added NRTManager and NRTManagerReopenThread, to
+   simplify handling NRT reopen with multiple search threads, and to
+   allow an app to control which indexing changes must be visible to
+   which search requests.  (Mike McCandless)
+
 API Changes
 
  * LUCENE-3141: add getter method to access fragInfos in FieldFragList.

Copied: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java (from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java&p1=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java&r1=1135658&r2=1135669&rev=1135669&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java Tue Jun 14 16:07:32 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.Collection;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.CopyOnWriteArrayList;
@@ -132,13 +133,13 @@ public class NRTManager implements Close
     return indexingGen.get();
   }
 
-  public long updateDocuments(Term t, Iterable<Document> docs, Analyzer a) throws IOException {
+  public long updateDocuments(Term t, Collection<Document> docs, Analyzer a) throws IOException {
     writer.updateDocuments(t, docs, a);
     // Return gen as of when indexing finished:
     return indexingGen.get();
   }
 
-  public long updateDocuments(Term t, Iterable<Document> docs) throws IOException {
+  public long updateDocuments(Term t, Collection<Document> docs) throws IOException {
     writer.updateDocuments(t, docs);
     // Return gen as of when indexing finished:
     return indexingGen.get();
@@ -162,7 +163,7 @@ public class NRTManager implements Close
     return indexingGen.get();
   }
 
-  public long addDocuments(Iterable<Document> docs, Analyzer a) throws IOException {
+  public long addDocuments(Collection<Document> docs, Analyzer a) throws IOException {
     writer.addDocuments(docs, a);
     // Return gen as of when indexing finished:
     return indexingGen.get();
@@ -174,7 +175,7 @@ public class NRTManager implements Close
     return indexingGen.get();
   }
 
-  public long addDocuments(Iterable<Document> docs) throws IOException {
+  public long addDocuments(Collection<Document> docs) throws IOException {
     writer.addDocuments(docs);
     // Return gen as of when indexing finished:
     return indexingGen.get();

Copied: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java&p1=lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java&r1=1135658&r2=1135669&rev=1135669&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Tue Jun 14 16:07:32 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
@@ -46,7 +45,6 @@ import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
@@ -101,11 +99,6 @@ public class TestNRTManager extends Luce
 
     final long t0 = System.currentTimeMillis();
 
-    if (CodecProvider.getDefault().getDefaultFieldCodec().equals("SimpleText")) {
-      // no
-      CodecProvider.getDefault().setDefaultFieldCodec("Standard");
-    }
-
     final LineFileDocs docs = new LineFileDocs(random);
     final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
     final MockDirectoryWrapper _dir = newFSDirectory(tempDir);
@@ -113,16 +106,18 @@ public class TestNRTManager extends Luce
     Directory dir = _dir;
     final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
 
-    // newIWConfig makes smallish max seg size, which
-    // results in tons and tons of segments for this test
-    // when run nightly:
-    MergePolicy mp = conf.getMergePolicy();
-    if (mp instanceof TieredMergePolicy) {
-      ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
-    } else if (mp instanceof LogByteSizeMergePolicy) {
-      ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
-    } else if (mp instanceof LogMergePolicy) {
-      ((LogMergePolicy) mp).setMaxMergeDocs(100000);
+    if (LuceneTestCase.TEST_NIGHTLY) {
+      // newIWConfig makes smallish max seg size, which
+      // results in tons and tons of segments for this test
+      // when run nightly:
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof TieredMergePolicy) {
+        ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
+      } else if (mp instanceof LogByteSizeMergePolicy) {
+        ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
+      } else if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setMaxMergeDocs(100000);
+      }
     }
 
     conf.setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() {
@@ -132,11 +127,10 @@ public class TestNRTManager extends Luce
           System.out.println("TEST: now warm merged reader=" + reader);
         }
         final int maxDoc = reader.maxDoc();
-        final Bits delDocs = reader.getDeletedDocs();
         int sum = 0;
         final int inc = Math.max(1, maxDoc/50);
         for(int docID=0;docID<maxDoc;docID += inc) {
-          if (delDocs == null || !delDocs.get(docID)) {
+          if (!reader.isDeleted(docID)) {
             final Document doc = reader.document(docID);
             sum += doc.getFields().size();
           }
@@ -194,7 +188,7 @@ public class TestNRTManager extends Luce
       System.out.println("TEST: " + NUM_INDEX_THREADS + " index threads; " + NUM_SEARCH_THREADS + " search threads");
     }
 
-    final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
+    final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
 
     final AtomicBoolean failed = new AtomicBoolean();
     final AtomicInteger addCount = new AtomicInteger();
@@ -487,16 +481,8 @@ public class TestNRTManager extends Luce
                 try {
                   smokeTestSearcher(s);
                   if (s.getIndexReader().numDocs() > 0) {
-                    Fields fields = MultiFields.getFields(s.getIndexReader());
-                    if (fields == null) {
-                      continue;
-                    }
-                    Terms terms = fields.terms("body");
-                    if (terms == null) {
-                      continue;
-                    }
 
-                    TermsEnum termsEnum = terms.iterator();
+                    TermEnum termEnum = s.getIndexReader().terms(new Term("body", ""));
                     int seenTermCount = 0;
                     int shift;
                     int trigger;
@@ -509,7 +495,7 @@ public class TestNRTManager extends Luce
                     }
 
                     while(System.currentTimeMillis() < stopTime) {
-                      BytesRef term = termsEnum.next();
+                      Term term = termEnum.term();
                       if (term == null) {
                         if (seenTermCount == 0) {
                           break;
@@ -524,7 +510,7 @@ public class TestNRTManager extends Luce
                           //System.out.println("trigger " + trigger);
                           shift = random.nextInt(totTermCount.get()/10);
                         }
-                        termsEnum.seek(new BytesRef(""));
+                        termEnum = s.getIndexReader().terms(new Term("body", ""));
                         continue;
                       }
                       seenTermCount++;
@@ -536,8 +522,9 @@ public class TestNRTManager extends Luce
                         //if (VERBOSE) {
                         //System.out.println(Thread.currentThread().getName() + " now search body:" + term.utf8ToString());
                         //}
-                        totHits.addAndGet(runQuery(s, new TermQuery(new Term("body", term))));
+                        totHits.addAndGet(runQuery(s, new TermQuery(term)));
                       }
+                      termEnum.next();
                     }
                     if (VERBOSE) {
                       System.out.println(Thread.currentThread().getName() + ": search done");

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jun 14 16:07:32 2011
@@ -4696,6 +4696,7 @@ public class IndexWriter implements Clos
   }
 
   synchronized boolean nrtIsCurrent(SegmentInfos infos) {
+    //System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
     return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
   }
 

Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Jun 14 16:07:32 2011
@@ -335,6 +335,7 @@ public class _TestUtil {
     } else if (mp instanceof TieredMergePolicy) {
       TieredMergePolicy tmp = (TieredMergePolicy) mp;
       tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
+      tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
     }
 
     MergeScheduler ms = w.getConfig().getMergeScheduler();

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Tue Jun 14 16:07:32 2011
@@ -418,6 +418,7 @@ public class TestNRTThreads extends Luce
                       //}
                       totHits.addAndGet(runQuery(s, new TermQuery(term)));
                     }
+                    termEnum.next();
                   }
                   if (VERBOSE) {
                     System.out.println(Thread.currentThread().getName() + ": search done");