You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by mi...@apache.org on 2010/03/22 11:21:08 UTC
svn commit: r926020 - in /lucene/solr/branches/newtrunk/lucene: ./
src/java/org/apache/lucene/index/ src/test/org/apache/lucene/index/
Author: mikemccand
Date: Mon Mar 22 10:21:07 2010
New Revision: 926020
URL: http://svn.apache.org/viewvc?rev=926020&view=rev
Log:
LUCENE-2297: allow reader pooling to be enabled (through IndexWriterConfig) even if you're not pulling NRT readers
Modified:
lucene/solr/branches/newtrunk/lucene/CHANGES.txt
lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
Modified: lucene/solr/branches/newtrunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/CHANGES.txt?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/CHANGES.txt (original)
+++ lucene/solr/branches/newtrunk/lucene/CHANGES.txt Mon Mar 22 10:21:07 2010
@@ -193,6 +193,10 @@ New features
documents (previously this was hardwired to 5), using
IndexWriterConfig.setMaxThreadStates. (Mike McCandless)
+* LUCENE-2297: Enable turning on reader pooling inside IndexWriter
+ even when getReader (near-real-timer reader) is not in use, through
+ IndexWriterConfig.enable/disableReaderPooling. (Mike McCandless)
+
Optimizations
* LUCENE-2075: Terms dict cache is now shared across threads instead
Modified: lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Mar 22 10:21:07 2010
@@ -1081,7 +1081,8 @@ public class IndexWriter implements Clos
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
mergedSegmentWarmer = conf.getMergedSegmentWarmer();
-
+ poolReaders = conf.getReaderPooling();
+
OpenMode mode = conf.getOpenMode();
boolean create;
if (mode == OpenMode.CREATE) {
Modified: lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/solr/branches/newtrunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Mon Mar 22 10:21:07 2010
@@ -84,6 +84,9 @@ public final class IndexWriterConfig imp
* others to finish. */
public final static int DEFAULT_MAX_THREAD_STATES = 8;
+ /** Default setting for {@link #setIndexWriterPooling}. */
+ public final static boolean DEFAULT_READER_POOLING = false;
+
/**
* Sets the default (for any instance) maximum time to wait for a write lock
* (in milliseconds).
@@ -118,6 +121,7 @@ public final class IndexWriterConfig imp
private IndexReaderWarmer mergedSegmentWarmer;
private MergePolicy mergePolicy;
private int maxThreadStates;
+ private boolean readerPooling;
// required for clone
private Version matchVersion;
@@ -147,6 +151,7 @@ public final class IndexWriterConfig imp
mergedSegmentWarmer = null;
mergePolicy = new LogByteSizeMergePolicy();
maxThreadStates = DEFAULT_MAX_THREAD_STATES;
+ readerPooling = DEFAULT_READER_POOLING;
}
@Override
@@ -506,6 +511,15 @@ public final class IndexWriterConfig imp
}
/**
+ * Returns the current MergePolicy in use by this writer.
+ *
+ * @see #setMergePolicy(MergePolicy)
+ */
+ public MergePolicy getMergePolicy() {
+ return mergePolicy;
+ }
+
+ /**
* Sets the max number of simultaneous threads that may be indexing documents
* at once in IndexWriter. Values < 1 are invalid and if passed
* <code>maxThreadStates</code> will be set to
@@ -522,13 +536,23 @@ public final class IndexWriterConfig imp
return maxThreadStates;
}
- /**
- * Returns the current MergePolicy in use by this writer.
- *
- * @see #setMergePolicy(MergePolicy)
- */
- public MergePolicy getMergePolicy() {
- return mergePolicy;
+ /** By default, IndexWriter does not pool the
+ * SegmentReaders it must open for deletions and
+ * merging, unless a near-real-time reader has been
+ * obtained by calling {@link IndexWriter#getReader}.
+ * This method lets you enable pooling without getting a
+ * near-real-time reader. NOTE: if you set this to
+ * false, IndexWriter will still pool readers once
+ * {@link IndexWriter#getReader} is called. */
+ public IndexWriterConfig setReaderPooling(boolean readerPooling) {
+ this.readerPooling = readerPooling;
+ return this;
+ }
+
+ /** Returns true if IndexWriter should pool readers even
+ * if {@link IndexWriter#getReader} has not been called. */
+ public boolean getReaderPooling() {
+ return readerPooling;
}
/** Expert: sets the {@link DocConsumer} chain to be used to process documents. */
@@ -562,6 +586,7 @@ public final class IndexWriterConfig imp
sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n");
sb.append("mergePolicy=").append(mergePolicy).append("\n");
sb.append("maxThreadStates=").append(maxThreadStates).append("\n");
+ sb.append("readerPooling=").append(readerPooling).append("\n");
return sb.toString();
}
}
Modified: lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Mon Mar 22 10:21:07 2010
@@ -78,6 +78,7 @@ public class TestIndexWriterConfig exten
assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS, conf.getMaxBufferedDeleteTerms());
assertEquals(IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, conf.getRAMBufferSizeMB(), 0.0);
assertEquals(IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS, conf.getMaxBufferedDocs());
+ assertEquals(IndexWriterConfig.DEFAULT_READER_POOLING, conf.getReaderPooling());
assertTrue(DocumentsWriter.defaultIndexingChain == conf.getIndexingChain());
assertNull(conf.getMergedSegmentWarmer());
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
@@ -102,6 +103,7 @@ public class TestIndexWriterConfig exten
getters.add("getMergedSegmentWarmer");
getters.add("getMergePolicy");
getters.add("getMaxThreadStates");
+ getters.add("getReaderPooling");
for (Method m : IndexWriterConfig.class.getDeclaredMethods()) {
if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) {
assertTrue("method " + m.getName() + " is not tested for defaults", getters.contains(m.getName()));
@@ -133,6 +135,7 @@ public class TestIndexWriterConfig exten
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS);
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
assertEquals(16.0, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB, 0.0);
+ assertEquals(false, IndexWriterConfig.DEFAULT_READER_POOLING);
assertEquals(8, IndexWriterConfig.DEFAULT_MAX_THREAD_STATES);
}
Modified: lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
URL: http://svn.apache.org/viewvc/lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java?rev=926020&r1=926019&r2=926020&view=diff
==============================================================================
--- lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (original)
+++ lucene/solr/branches/newtrunk/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java Mon Mar 22 10:21:07 2010
@@ -85,7 +85,8 @@ public class TestStressIndexing2 extends
Directory dir2 = new MockRAMDirectory();
// mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
int maxThreadStates = 1+r.nextInt(10);
- Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates);
+ boolean doReaderPooling = r.nextBoolean();
+ Map<String,Document> docs = indexRandom(10, 10, 100, dir1, maxThreadStates, doReaderPooling);
indexSerial(docs, dir2);
// verifying verify
@@ -103,6 +104,7 @@ public class TestStressIndexing2 extends
mergeFactor=r.nextInt(3)+2;
maxBufferedDocs=r.nextInt(3)+2;
int maxThreadStates = 1+r.nextInt(10);
+ boolean doReaderPooling = r.nextBoolean();
seed++;
int nThreads=r.nextInt(5)+1;
@@ -110,7 +112,7 @@ public class TestStressIndexing2 extends
int range=r.nextInt(20)+1;
Directory dir1 = new MockRAMDirectory();
Directory dir2 = new MockRAMDirectory();
- Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates);
+ Map<String,Document> docs = indexRandom(nThreads, iter, range, dir1, maxThreadStates, doReaderPooling);
indexSerial(docs, dir2);
verifyEquals(dir1, dir2, "id");
}
@@ -184,12 +186,14 @@ public class TestStressIndexing2 extends
return dw;
}
- public Map<String,Document> indexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates) throws IOException, InterruptedException {
+ public Map<String,Document> indexRandom(int nThreads, int iterations, int range, Directory dir, int maxThreadStates,
+ boolean doReaderPooling) throws IOException, InterruptedException {
Map<String,Document> docs = new HashMap<String,Document>();
for(int iter=0;iter<3;iter++) {
IndexWriter w = new MockIndexWriter(dir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)
- .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates));
+ .setRAMBufferSizeMB(0.1).setMaxBufferedDocs(maxBufferedDocs).setMaxThreadStates(maxThreadStates)
+ .setReaderPooling(doReaderPooling));
LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
lmp.setUseCompoundFile(false);
lmp.setUseCompoundDocStore(false);