You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/04/26 04:35:58 UTC
svn commit: r1476030 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/index/ lucene/test-framework/
lucene/test-framework/src/java/org/apache/lucene/util/ ...
Author: rmuir
Date: Fri Apr 26 02:35:57 2013
New Revision: 1476030
URL: http://svn.apache.org/r1476030
Log:
SOLR-4761: add option to plug in mergedSegmentWarmer
Added:
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
- copied unchanged from r1476026, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SimpleMergedSegmentWarmer.java
lucene/dev/branches/branch_4x/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml
- copied unchanged from r1476026, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer.xml
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrConfig.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Apr 26 02:35:57 2013
@@ -38,6 +38,9 @@ New Features
This is necessary when using collectors like
ToParentBlockJoinCollector with DrillSideways. (Mike McCandless)
+* SOLR-4761: Add SimpleMergedSegmentWarmer, which just initializes terms,
+ norms, docvalues, and so on. (Mark Miller, Mike McCandless, Robert Muir)
+
======================= Lucene 4.3.0 =======================
Changes in backwards compatibility policy
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Fri Apr 26 02:35:57 2013
@@ -40,6 +40,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
@@ -966,6 +967,46 @@ public class TestIndexWriterReader exten
assertTrue(didWarm.get());
}
+ public void testSimpleMergedSegmentWramer() throws Exception {
+ Directory dir = newDirectory();
+ final AtomicBoolean didWarm = new AtomicBoolean();
+ InfoStream infoStream = new InfoStream() {
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public void message(String component, String message) {
+ if ("SMSW".equals(component)) {
+ didWarm.set(true);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return true;
+ }
+ };
+ IndexWriter w = new IndexWriter(
+ dir,
+ newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+ setMaxBufferedDocs(2).
+ setReaderPooling(true).
+ setInfoStream(infoStream).
+ setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream)).
+ setMergePolicy(newLogMergePolicy(10))
+ );
+
+ Document doc = new Document();
+ doc.add(newStringField("foo", "bar", Field.Store.NO));
+ for(int i=0;i<20;i++) {
+ w.addDocument(doc);
+ }
+ w.waitForMerges();
+ w.close();
+ dir.close();
+ assertTrue(didWarm.get());
+ }
+
public void testNoTermsIndex() throws Exception {
// Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if
// they're picked.
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri Apr 26 02:35:57 2013
@@ -774,6 +774,9 @@ public abstract class LuceneTestCase ext
} else {
c.setMergePolicy(newLogMergePolicy());
}
+ if (rarely(r)) {
+ c.setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(c.getInfoStream()));
+ }
c.setReaderPooling(r.nextBoolean());
c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4));
return c;
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Fri Apr 26 02:35:57 2013
@@ -41,6 +41,9 @@ New Features
* SOLR-3251: Dynamically add fields to schema. (Steve Rowe, Robert Muir, yonik)
+* SOLR-4761: Add option to plugin a merged segment warmer into solrconfig.xml
+ (Mark Miller, Mike McCandless, Robert Muir)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/core/SolrConfig.java Fri Apr 26 02:35:57 2013
@@ -142,11 +142,10 @@ public class SolrConfig extends Config {
defaultIndexConfig = mainIndexConfig = null;
indexConfigPrefix = "indexConfig";
}
+ reopenReaders = getBool(indexConfigPrefix+"/reopenReaders", true);
// Parse indexConfig section, using mainIndex as backup in case old config is used
indexConfig = new SolrIndexConfig(this, "indexConfig", mainIndexConfig);
-
- reopenReaders = getBool(indexConfigPrefix+"/reopenReaders", true);
-
+
booleanQueryMaxClauseCount = getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
log.info("Using Lucene MatchVersion: " + luceneMatchVersion);
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/update/SolrIndexConfig.java Fri Apr 26 02:35:57 2013
@@ -18,6 +18,8 @@
package org.apache.solr.update;
import org.apache.lucene.index.*;
+import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.Version;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -55,6 +57,8 @@ public class SolrIndexConfig {
public final PluginInfo mergeSchedulerInfo;
public final int termIndexInterval;
+ public final PluginInfo mergedSegmentWarmerInfo;
+
public String infoStreamFile = null;
// Available lock types
@@ -81,6 +85,7 @@ public class SolrIndexConfig {
mergePolicyInfo = null;
mergeSchedulerInfo = null;
defaultMergePolicyClassName = TieredMergePolicy.class.getName();
+ mergedSegmentWarmerInfo = null;
}
/**
@@ -135,6 +140,11 @@ public class SolrIndexConfig {
infoStreamFile= solrConfig.get(prefix + "/infoStream/@file", null);
log.info("IndexWriter infoStream debug log is enabled: " + infoStreamFile);
}
+
+ mergedSegmentWarmerInfo = getPluginInfo(prefix + "/mergedSegmentWarmer", solrConfig, def.mergedSegmentWarmerInfo);
+ if (mergedSegmentWarmerInfo != null && solrConfig.reopenReaders == false) {
+ throw new IllegalArgumentException("Supplying a mergedSegmentWarmer will do nothing since reopenReaders is false");
+ }
}
/*
@@ -182,6 +192,16 @@ public class SolrIndexConfig {
if (maxIndexingThreads != -1) {
iwc.setMaxThreadStates(maxIndexingThreads);
}
+
+ if (mergedSegmentWarmerInfo != null) {
+ // TODO: add infostream -> normal logging system (there is an issue somewhere)
+ IndexReaderWarmer warmer = schema.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className,
+ IndexReaderWarmer.class,
+ null,
+ new Class[] { InfoStream.class },
+ new Object[] { InfoStream.NO_OUTPUT });
+ iwc.setMergedSegmentWarmer(warmer);
+ }
return iwc;
}
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java?rev=1476030&r1=1476029&r2=1476030&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/update/SolrIndexConfigTest.java Fri Apr 26 02:35:57 2013
@@ -19,6 +19,8 @@ package org.apache.solr.update;
import java.io.File;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.SimpleMergedSegmentWarmer;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.schema.IndexSchema;
@@ -55,5 +57,18 @@ public class SolrIndexConfigTest extends
IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema("schema.xml", solrConfig);
solrIndexConfig.toIndexWriterConfig(indexSchema);
}
+
+ public void testMergedSegmentWarmerIndexConfigCreation() throws Exception {
+ SolrConfig solrConfig = new SolrConfig("solr" + File.separator
+ + "collection1", "solrconfig-warmer.xml", null);
+ SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null);
+ assertNotNull(solrIndexConfig);
+ assertNotNull(solrIndexConfig.mergedSegmentWarmerInfo);
+ assertEquals(SimpleMergedSegmentWarmer.class.getName(),
+ solrIndexConfig.mergedSegmentWarmerInfo.className);
+ IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema("schema.xml", solrConfig);
+ IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(indexSchema);
+ assertEquals(SimpleMergedSegmentWarmer.class, iwc.getMergedSegmentWarmer().getClass());
+ }
}