You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/08/14 18:38:24 UTC
svn commit: r1513953 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/core/
solr/core/src/test-files/solr/collection1/conf/
solr/core/src/test/org/apache/solr/core/
solr/core/src/test/org/apache/solr/handler/ ...
Author: rmuir
Date: Wed Aug 14 16:38:23 2013
New Revision: 1513953
URL: http://svn.apache.org/r1513953
Log:
SOLR-4764: When using NRT, just init the reader from IndexWriter
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
- copied unchanged from r1513945, lucene/dev/trunk/solr/core/src/test/org/apache/solr/core/TestNRTOpen.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Wed Aug 14 16:38:23 2013
@@ -97,6 +97,9 @@ Bug Fixes
* SOLR-5135: Harden Collection API deletion of /collections/$collection
ZooKeeper node. (Mark Miller)
+
+* SOLR-4764: When using NRT, just init the first reader from IndexWriter.
+ (Robert Muir, Mark Miller)
Optimizations
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java Wed Aug 14 16:38:23 2013
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.IOException;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
@@ -67,4 +68,21 @@ public abstract class IndexReaderFactory
*/
public abstract DirectoryReader newReader(Directory indexDir, SolrCore core)
throws IOException;
+
+ /**
+ * Creates a new IndexReader instance using the given IndexWriter.
+ * <p>
+ * This is used for opening the initial reader in NRT mode ({@code reopenReaders=true}
+ * in solrconfig.xml)
+ *
+ * @param writer IndexWriter
+ * @param core {@link SolrCore} instance where this reader will be used. NOTE:
+ * this SolrCore instance may not be fully configured yet, but basic things like
+ * {@link SolrCore#getCoreDescriptor()}, {@link SolrCore#getLatestSchema()} and
+ * {@link SolrCore#getSolrConfig()} are valid.
+ * @return An IndexReader instance
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract DirectoryReader newReader(IndexWriter writer, SolrCore core)
+ throws IOException;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrCore.java Wed Aug 14 16:38:23 2013
@@ -795,19 +795,6 @@ public final class SolrCore implements S
}
}
- // Open the searcher *before* the update handler so we don't end up
- // opening
- // one in the middle.
- // With lockless commits in Lucene now, this probably shouldn't be an
- // issue anymore
-
- try {
- getSearcher(false, false, null, true);
- } finally {
- newReaderCreator = null;
- if (iwRef != null) iwRef.decref();
- }
-
String updateHandlerClass = solrConfig.getUpdateHandlerInfo().className;
if (updateHandler == null) {
@@ -819,6 +806,13 @@ public final class SolrCore implements S
: updateHandlerClass, updateHandler);
}
infoRegistry.put("updateHandler", this.updateHandler);
+
+ try {
+ getSearcher(false, false, null, true);
+ } finally {
+ newReaderCreator = null;
+ if (iwRef != null) iwRef.decref();
+ }
// Finally tell anyone who wants to know
resourceLoader.inform(resourceLoader);
@@ -1432,6 +1426,16 @@ public final class SolrCore implements S
DirectoryReader newReader = newReaderCreator.call();
tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig,
(realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
+ } else if (solrConfig.reopenReaders) {
+ RefCounted<IndexWriter> writer = getUpdateHandler().getSolrCoreState().getIndexWriter(this);
+ DirectoryReader newReader = null;
+ try {
+ newReader = indexReaderFactory.newReader(writer.get(), this);
+ } finally {
+ writer.decref();
+ }
+ tmp = new SolrIndexSearcher(this, newIndexDir, getLatestSchema(), getSolrConfig().indexConfig,
+ (realtime ? "realtime":"main"), newReader, true, !realtime, true, directoryFactory);
} else {
// normal open that happens at startup
// verbose("non-reopen START:");
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java Wed Aug 14 16:38:23 2013
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import java.io.IOException;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
/**
@@ -26,6 +27,7 @@ import org.apache.lucene.store.Directory
* {@link DirectoryReader}.
*
* @see DirectoryReader#open(Directory)
+ * @see DirectoryReader#open(IndexWriter, boolean)
*/
public class StandardIndexReaderFactory extends IndexReaderFactory {
@@ -33,4 +35,9 @@ public class StandardIndexReaderFactory
public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
return DirectoryReader.open(indexDir, termInfosIndexDivisor);
}
+
+ @Override
+ public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+ return DirectoryReader.open(writer, true);
+ }
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml Wed Aug 14 16:38:23 2013
@@ -35,6 +35,7 @@ A solrconfig.xml snippet containing inde
<ramBufferSizeMB>${solr.tests.ramBufferSizeMB}</ramBufferSizeMB>
<mergeScheduler class="${solr.tests.mergeScheduler}" />
+ <reopenReaders>${solr.tests.reopenReaders:true}</reopenReaders>
<writeLockTimeout>1000</writeLockTimeout>
<commitLockTimeout>10000</commitLockTimeout>
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/AlternateDirectoryTest.java Wed Aug 14 16:38:23 2013
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.BeforeClass;
@@ -71,6 +72,12 @@ public class AlternateDirectoryTest exte
TestIndexReaderFactory.newReaderCalled = true;
return DirectoryReader.open(indexDir);
}
+
+ @Override
+ public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+ TestIndexReaderFactory.newReaderCalled = true;
+ return DirectoryReader.open(writer, true);
+ }
}
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/core/TestArbitraryIndexDir.java Wed Aug 14 16:38:23 2013
@@ -53,12 +53,15 @@ public class TestArbitraryIndexDir exten
static String savedFactory;
@BeforeClass
public static void beforeClass() {
+ // this test wants to start solr, and then open a separate indexwriter of its own on the same dir.
+ System.setProperty("solr.tests.reopenReaders", "false");
System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
savedFactory = System.getProperty("solr.DirectoryFactory");
System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockFSDirectoryFactory");
}
@AfterClass
public static void afterClass() {
+ System.clearProperty("solr.tests.reopenReaders");
if (savedFactory == null) {
System.clearProperty("solr.directoryFactory");
} else {
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Wed Aug 14 16:38:23 2013
@@ -346,6 +346,9 @@ public class TestReplicationHandler exte
public void testNoWriter() throws Exception {
useFactory(null); // force a persistent directory
+ // read-only setting (no opening from indexwriter)
+ System.setProperty("solr.tests.reopenReaders", "false");
+ try {
// stop and start so they see the new directory setting
slaveJetty.stop();
masterJetty.stop();
@@ -356,6 +359,9 @@ public class TestReplicationHandler exte
slaveClient.commit();
slaveJetty.stop();
slaveJetty.start(true);
+ } finally {
+ System.clearProperty("solr.tests.reopenReaders"); // dont mess with other tests
+ }
// Currently we open a writer on-demand. This is to test that we are correctly testing
// the code path when SolrDeletionPolicy.getLatestCommit() returns null.
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/MultiCoreExampleTestBase.java Wed Aug 14 16:38:23 2013
@@ -46,11 +46,6 @@ public abstract class MultiCoreExampleTe
@Override public String getSolrHome() { return ExternalPaths.EXAMPLE_MULTICORE_HOME; }
- protected void setupCoreContainer() {
- cores = new CoreContainer();
- cores.load();
- }
-
@Override public void setUp() throws Exception {
super.setUp();
@@ -65,11 +60,6 @@ public abstract class MultiCoreExampleTe
System.setProperty( "solr.core0.data.dir", this.dataDir1.getCanonicalPath() );
System.setProperty( "solr.core1.data.dir", this.dataDir2.getCanonicalPath() );
- setupCoreContainer();
-
- SolrCore.log.info("CORES=" + cores + " : " + cores.getCoreNames());
-
-
}
@Override
@@ -84,8 +74,6 @@ public abstract class MultiCoreExampleTe
System.err.println("!!!! WARNING: best effort to remove " + dataDir2.getAbsolutePath() + " FAILED !!!!!");
}
}
-
- cores.shutdown();
}
@Override
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java?rev=1513953&r1=1513952&r2=1513953&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/MultiCoreEmbeddedTest.java Wed Aug 14 16:38:23 2013
@@ -19,6 +19,8 @@ package org.apache.solr.client.solrj.emb
import org.apache.solr.client.solrj.MultiCoreExampleTestBase;
import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrCore;
/**
* This runs SolrServer test using
@@ -33,6 +35,19 @@ public class MultiCoreEmbeddedTest exten
// TODO: fix this test to use MockDirectoryFactory
System.clearProperty("solr.directoryFactory");
super.setUp();
+ setupCoreContainer();
+ SolrCore.log.info("CORES=" + cores + " : " + cores.getCoreNames());
+ }
+
+ protected void setupCoreContainer() {
+ cores = new CoreContainer();
+ cores.load();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ cores.shutdown();
+ super.tearDown();
}
@Override