You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/06/14 18:07:32 UTC
svn commit: r1135669 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ lucene/contrib/
lucene/contrib/misc/src/java/org/apache/lucene/index/
lucene/contrib/misc/src/test/or...
Author: mikemccand
Date: Tue Jun 14 16:07:32 2011
New Revision: 1135669
URL: http://svn.apache.org/viewvc?rev=1135669&view=rev
Log:
LUCENE-2955: add NRTManager/ReopenThread for simple control over NRT reopening and controlled visiblity of specific indexing changes
Added:
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java
- copied, changed from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManagerReopenThread.java
- copied unchanged from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManagerReopenThread.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
- copied, changed from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Tue Jun 14 16:07:32 2011
@@ -20,6 +20,11 @@ New Features
document sharing the same group was indexed as a doc block
(IndexWriter.add/updateDocuments). (Mike McCandless)
+ * LUCENE-2955: Added NRTManager and NRTManagerReopenThread, to
+ simplify handling NRT reopen with multiple search threads, and to
+ allow an app to control which indexing changes must be visible to
+ which search requests. (Mike McCandless)
+
API Changes
* LUCENE-3141: add getter method to access fragInfos in FieldFragList.
Copied: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java (from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java&p1=lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java&r1=1135658&r2=1135669&rev=1135669&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/java/org/apache/lucene/index/NRTManager.java Tue Jun 14 16:07:32 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
+import java.util.Collection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -132,13 +133,13 @@ public class NRTManager implements Close
return indexingGen.get();
}
- public long updateDocuments(Term t, Iterable<Document> docs, Analyzer a) throws IOException {
+ public long updateDocuments(Term t, Collection<Document> docs, Analyzer a) throws IOException {
writer.updateDocuments(t, docs, a);
// Return gen as of when indexing finished:
return indexingGen.get();
}
- public long updateDocuments(Term t, Iterable<Document> docs) throws IOException {
+ public long updateDocuments(Term t, Collection<Document> docs) throws IOException {
writer.updateDocuments(t, docs);
// Return gen as of when indexing finished:
return indexingGen.get();
@@ -162,7 +163,7 @@ public class NRTManager implements Close
return indexingGen.get();
}
- public long addDocuments(Iterable<Document> docs, Analyzer a) throws IOException {
+ public long addDocuments(Collection<Document> docs, Analyzer a) throws IOException {
writer.addDocuments(docs, a);
// Return gen as of when indexing finished:
return indexingGen.get();
@@ -174,7 +175,7 @@ public class NRTManager implements Close
return indexingGen.get();
}
- public long addDocuments(Iterable<Document> docs) throws IOException {
+ public long addDocuments(Collection<Document> docs) throws IOException {
writer.addDocuments(docs);
// Return gen as of when indexing finished:
return indexingGen.get();
Copied: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (from r1135658, lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?p2=lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java&p1=lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java&r1=1135658&r2=1135669&rev=1135669&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Tue Jun 14 16:07:32 2011
@@ -34,7 +34,6 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
@@ -46,7 +45,6 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
@@ -101,11 +99,6 @@ public class TestNRTManager extends Luce
final long t0 = System.currentTimeMillis();
- if (CodecProvider.getDefault().getDefaultFieldCodec().equals("SimpleText")) {
- // no
- CodecProvider.getDefault().setDefaultFieldCodec("Standard");
- }
-
final LineFileDocs docs = new LineFileDocs(random);
final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
final MockDirectoryWrapper _dir = newFSDirectory(tempDir);
@@ -113,16 +106,18 @@ public class TestNRTManager extends Luce
Directory dir = _dir;
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
- // newIWConfig makes smallish max seg size, which
- // results in tons and tons of segments for this test
- // when run nightly:
- MergePolicy mp = conf.getMergePolicy();
- if (mp instanceof TieredMergePolicy) {
- ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
- } else if (mp instanceof LogByteSizeMergePolicy) {
- ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
- } else if (mp instanceof LogMergePolicy) {
- ((LogMergePolicy) mp).setMaxMergeDocs(100000);
+ if (LuceneTestCase.TEST_NIGHTLY) {
+ // newIWConfig makes smallish max seg size, which
+ // results in tons and tons of segments for this test
+ // when run nightly:
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof TieredMergePolicy) {
+ ((TieredMergePolicy) mp).setMaxMergedSegmentMB(5000.);
+ } else if (mp instanceof LogByteSizeMergePolicy) {
+ ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1000.);
+ } else if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setMaxMergeDocs(100000);
+ }
}
conf.setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() {
@@ -132,11 +127,10 @@ public class TestNRTManager extends Luce
System.out.println("TEST: now warm merged reader=" + reader);
}
final int maxDoc = reader.maxDoc();
- final Bits delDocs = reader.getDeletedDocs();
int sum = 0;
final int inc = Math.max(1, maxDoc/50);
for(int docID=0;docID<maxDoc;docID += inc) {
- if (delDocs == null || !delDocs.get(docID)) {
+ if (!reader.isDeleted(docID)) {
final Document doc = reader.document(docID);
sum += doc.getFields().size();
}
@@ -194,7 +188,7 @@ public class TestNRTManager extends Luce
System.out.println("TEST: " + NUM_INDEX_THREADS + " index threads; " + NUM_SEARCH_THREADS + " search threads");
}
- final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
+ final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
final AtomicBoolean failed = new AtomicBoolean();
final AtomicInteger addCount = new AtomicInteger();
@@ -487,16 +481,8 @@ public class TestNRTManager extends Luce
try {
smokeTestSearcher(s);
if (s.getIndexReader().numDocs() > 0) {
- Fields fields = MultiFields.getFields(s.getIndexReader());
- if (fields == null) {
- continue;
- }
- Terms terms = fields.terms("body");
- if (terms == null) {
- continue;
- }
- TermsEnum termsEnum = terms.iterator();
+ TermEnum termEnum = s.getIndexReader().terms(new Term("body", ""));
int seenTermCount = 0;
int shift;
int trigger;
@@ -509,7 +495,7 @@ public class TestNRTManager extends Luce
}
while(System.currentTimeMillis() < stopTime) {
- BytesRef term = termsEnum.next();
+ Term term = termEnum.term();
if (term == null) {
if (seenTermCount == 0) {
break;
@@ -524,7 +510,7 @@ public class TestNRTManager extends Luce
//System.out.println("trigger " + trigger);
shift = random.nextInt(totTermCount.get()/10);
}
- termsEnum.seek(new BytesRef(""));
+ termEnum = s.getIndexReader().terms(new Term("body", ""));
continue;
}
seenTermCount++;
@@ -536,8 +522,9 @@ public class TestNRTManager extends Luce
//if (VERBOSE) {
//System.out.println(Thread.currentThread().getName() + " now search body:" + term.utf8ToString());
//}
- totHits.addAndGet(runQuery(s, new TermQuery(new Term("body", term))));
+ totHits.addAndGet(runQuery(s, new TermQuery(term)));
}
+ termEnum.next();
}
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": search done");
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jun 14 16:07:32 2011
@@ -4696,6 +4696,7 @@ public class IndexWriter implements Clos
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
+ //System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Tue Jun 14 16:07:32 2011
@@ -335,6 +335,7 @@ public class _TestUtil {
} else if (mp instanceof TieredMergePolicy) {
TieredMergePolicy tmp = (TieredMergePolicy) mp;
tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
+ tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
}
MergeScheduler ms = w.getConfig().getMergeScheduler();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1135669&r1=1135668&r2=1135669&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Tue Jun 14 16:07:32 2011
@@ -418,6 +418,7 @@ public class TestNRTThreads extends Luce
//}
totHits.addAndGet(runQuery(s, new TermQuery(term)));
}
+ termEnum.next();
}
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": search done");