You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/05/14 14:25:51 UTC
svn commit: r1103086 - in /lucene/dev/branches/branch_3x: lucene/
lucene/contrib/ant/src/java/org/apache/lucene/ant/
lucene/contrib/misc/src/test/org/apache/lucene/index/
lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/
lucene/contr...
Author: mikemccand
Date: Sat May 14 12:25:51 2011
New Revision: 1103086
URL: http://svn.apache.org/viewvc?rev=1103086&view=rev
Log:
LUCENE-1076: cutover to TieredMP as default MP as of 3.2
Modified:
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/core/SolrConfig.java
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Sat May 14 12:25:51 2011
@@ -29,6 +29,13 @@ Changes in runtime behavior
returns NumericField instances. (Uwe Schindler, Ryan McKinley,
Mike McCandless)
+* LUCENE-1076: Changed the default merge policy from
+ LogByteSizeMergePolicy to TieredMergePolicy, as of Version.LUCENE_32
+ (passed to IndexWriterConfig), which is able to merge non-contiguous
+ segments. This means docIDs no longer necessarily stay "in order"
+ during indexing. If this is a problem then you can use either of
+ the LogMergePolicy impls. (Mike McCandless)
+
New features
* LUCENE-3082: Added index upgrade tool oal.index.IndexUpgrader
Modified: lucene/dev/branches/branch_3x/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java Sat May 14 12:25:51 2011
@@ -39,7 +39,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
@@ -286,9 +286,9 @@ public class IndexTask extends Task {
IndexWriterConfig conf = new IndexWriterConfig(
Version.LUCENE_CURRENT, analyzer).setOpenMode(
create ? OpenMode.CREATE : OpenMode.APPEND);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundIndex);
- lmp.setMergeFactor(mergeFactor);
+ TieredMergePolicy tmp = (TieredMergePolicy) conf.getMergePolicy();
+ tmp.setUseCompoundFile(useCompoundIndex);
+ tmp.setMaxMergeAtOnce(mergeFactor);
IndexWriter writer = new IndexWriter(dir, conf);
int totalFiles = 0;
int totalIndexed = 0;
Modified: lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java Sat May 14 12:25:51 2011
@@ -34,11 +34,12 @@ public class TestIndexSplitter extends L
_TestUtil.rmDir(destDir);
destDir.mkdirs();
Directory fsDir = newFSDirectory(dir);
+ LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
+ mergePolicy.setNoCFSRatio(1);
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random))
- .setOpenMode(OpenMode.CREATE);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(true);
- ((LogMergePolicy) conf.getMergePolicy()).setNoCFSRatio(1.0);
+ .setOpenMode(OpenMode.CREATE)
+ .setMergePolicy(mergePolicy);
IndexWriter iw = new IndexWriter(fsDir, conf);
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Sat May 14 12:25:51 2011
@@ -29,7 +29,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.BooleanClause;
@@ -42,7 +42,6 @@ import org.apache.lucene.store.AlreadyCl
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.VirtualMethod;
/**
* <p>
@@ -505,7 +504,7 @@ public class SpellChecker implements jav
ensureOpen();
final Directory dir = this.spellIndex;
final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(mergeFactor);
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
IndexSearcher indexSearcher = obtainSearcher();
final List<IndexReader> readers = new ArrayList<IndexReader>();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java Sat May 14 12:25:51 2011
@@ -36,7 +36,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -250,7 +250,7 @@ public class Syns2Index
// override the specific index if it already exists
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
Iterator<String> i1 = word2Nums.keySet().iterator();
while (i1.hasNext()) // for each word
{
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Sat May 14 12:25:51 2011
@@ -132,10 +132,15 @@ public final class IndexWriterConfig imp
/**
* Creates a new config that with defaults that match the specified
- * {@link Version} as well as the default {@link Analyzer}. {@link Version} is
- * a placeholder for future changes. The default settings are relevant to 3.1
- * and before. In the future, if different settings will apply to different
- * versions, they will be documented here.
+ * {@link Version} as well as the default {@link
+ * Analyzer}. If matchVersion is >= {@link
+ * Version#LUCENE_32}, {@link TieredMergePolicy} is used
+ * for merging; else {@link LogByteSizeMergePolicy}.
+ * Note that {@link TieredMergePolicy} is free to select
+ * non-contiguous merges, which means docIDs may not
+ * remain montonic over time. If this is a problem you
+ * should switch to {@link LogByteSizeMergePolicy} or
+ * {@link LogDocMergePolicy}.
*/
public IndexWriterConfig(Version matchVersion, Analyzer analyzer) {
this.matchVersion = matchVersion;
@@ -152,7 +157,11 @@ public final class IndexWriterConfig imp
maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS;
indexingChain = DocumentsWriter.defaultIndexingChain;
mergedSegmentWarmer = null;
- mergePolicy = new LogByteSizeMergePolicy();
+ if (matchVersion.onOrAfter(Version.LUCENE_32)) {
+ mergePolicy = new TieredMergePolicy();
+ } else {
+ mergePolicy = new LogByteSizeMergePolicy();
+ }
maxThreadStates = DEFAULT_MAX_THREAD_STATES;
readerPooling = DEFAULT_READER_POOLING;
readerTermsIndexDivisor = DEFAULT_READER_TERMS_INDEX_DIVISOR;
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java Sat May 14 12:25:51 2011
@@ -129,7 +129,7 @@ public class TestAtomicUpdate extends Lu
IndexWriterConfig conf = new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMaxBufferedDocs(7);
- ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(3);
+ ((TieredMergePolicy) conf.getMergePolicy()).setMaxMergeAtOnce(3);
IndexWriter writer = new MockIndexWriter(directory, conf);
writer.setInfoStream(VERBOSE ? System.out : null);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Sat May 14 12:25:51 2011
@@ -80,7 +80,7 @@ public class TestIndexWriterConfig exten
assertNull(conf.getMergedSegmentWarmer());
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
assertEquals(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.getReaderTermsIndexDivisor());
- assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass());
+ assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
// Sanity check - validate that all getters are covered.
Set<String> getters = new HashSet<String>();
@@ -255,7 +255,7 @@ public class TestIndexWriterConfig exten
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
// Test MergePolicy
- assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass());
+ assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
conf.setMergePolicy(new LogDocMergePolicy());
assertEquals(LogDocMergePolicy.class, conf.getMergePolicy().getClass());
conf.setMergePolicy(null);
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/core/SolrConfig.java?rev=1103086&r1=1103085&r2=1103086&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/core/SolrConfig.java Sat May 14 12:25:51 2011
@@ -55,7 +55,6 @@ import java.util.regex.Pattern;
import java.util.regex.Matcher;
import java.io.FileFilter;
import java.io.IOException;
-import java.io.InputStream;
/**
@@ -131,12 +130,12 @@ public class SolrConfig extends Config {
throws ParserConfigurationException, IOException, SAXException {
super(loader, name, is, "/config/");
initLibs();
+ luceneMatchVersion = getLuceneVersion("luceneMatchVersion", Version.LUCENE_24);
defaultIndexConfig = new SolrIndexConfig(this, null, null);
mainIndexConfig = new SolrIndexConfig(this, "mainIndex", defaultIndexConfig);
reopenReaders = getBool("mainIndex/reopenReaders", true);
booleanQueryMaxClauseCount = getInt("query/maxBooleanClauses", BooleanQuery.getMaxClauseCount());
- luceneMatchVersion = getLuceneVersion("luceneMatchVersion", Version.LUCENE_24);
log.info("Using Lucene MatchVersion: " + luceneMatchVersion);
filtOptEnabled = getBool("query/boolTofilterOptimizer/@enabled", false);