You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/04/04 15:03:48 UTC
svn commit: r1088606 [2/5] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/solr/ dev-tools/maven/
dev-tools/maven/solr/ dev-tools/maven/solr/contrib/analysis-extras/
dev-tools/maven/solr/contrib/clustering/ dev-tools/mav...
Modified: lucene/dev/branches/realtime_search/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Mon Apr 4 13:03:42 2011
@@ -65,7 +65,7 @@ public class TestIndicesEquals extends L
// create dir data
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < 20; i++) {
Document document = new Document();
@@ -91,7 +91,7 @@ public class TestIndicesEquals extends L
// create dir data
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
indexWriter.setInfoStream(VERBOSE ? System.out : null);
if (VERBOSE) {
System.out.println("TEST: make test index");
Modified: lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java Mon Apr 4 13:03:42 2011
@@ -61,7 +61,7 @@ public class TestFieldNormModifier exten
super.setUp();
store = newDirectory();
IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();
Modified: lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java Mon Apr 4 13:03:42 2011
@@ -32,7 +32,7 @@ public class TestMultiPassIndexSplitter
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
Document doc;
for (int i = 0; i < NUM_DOCS; i++) {
doc = new Document();
Modified: lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Mon Apr 4 13:03:42 2011
@@ -30,7 +30,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@@ -137,7 +137,7 @@ public class TestAppendingCodec extends
IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer());
cfg.setCodecProvider(new AppendingCodecProvider());
- ((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
+ ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
IndexWriter writer = new IndexWriter(dir, cfg);
Document doc = new Document();
doc.add(newField("f", text, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
Modified: lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java Mon Apr 4 13:03:42 2011
@@ -66,7 +66,7 @@ public class TestLengthNormModifier exte
super.setUp();
store = newDirectory();
IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();
Modified: lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java Mon Apr 4 13:03:42 2011
@@ -43,7 +43,7 @@ public class DuplicateFilterTest extends
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
//Add series of docs with filterable fields : url, text and dates flags
addDoc(writer, "http://lucene.apache.org", "lucene 1.4.3 available", "20040101");
Modified: lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java Mon Apr 4 13:03:42 2011
@@ -40,7 +40,7 @@ public class FuzzyLikeThisQueryTest exte
public void setUp() throws Exception {
super.setUp();
directory = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
//Add series of docs with misspelt names
addDoc(writer, "jonathon smythe","1");
Modified: lucene/dev/branches/realtime_search/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/spellchecker/src/java/org/apache/lucene/search/spell/SpellChecker.java Mon Apr 4 13:03:42 2011
@@ -29,7 +29,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Terms;
@@ -45,7 +45,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.Version;
-import org.apache.lucene.util.VirtualMethod;
/**
* <p>
@@ -508,7 +507,7 @@ public class SpellChecker implements jav
ensureOpen();
final Directory dir = this.spellIndex;
final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, new WhitespaceAnalyzer(Version.LUCENE_CURRENT)).setRAMBufferSizeMB(ramMB));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(mergeFactor);
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
IndexSearcher indexSearcher = obtainSearcher();
final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
Modified: lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/Syns2Index.java Mon Apr 4 13:03:42 2011
@@ -36,7 +36,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
@@ -250,7 +250,7 @@ public class Syns2Index
// override the specific index if it already exists
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
Version.LUCENE_CURRENT, ana).setOpenMode(OpenMode.CREATE));
- ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
+ ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); // why?
Iterator<String> i1 = word2Nums.keySet().iterator();
while (i1.hasNext()) // for each word
{
Modified: lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java (original)
+++ lucene/dev/branches/realtime_search/lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestWordnet.java Mon Apr 4 13:03:42 2011
@@ -29,6 +29,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestWordnet extends LuceneTestCase {
private IndexSearcher searcher;
@@ -42,6 +43,7 @@ public class TestWordnet extends LuceneT
// create a temporary synonym index
File testFile = getDataFile("testSynonyms.txt");
String commandLineArgs[] = { testFile.getAbsolutePath(), storePathName };
+ _TestUtil.rmDir(new File(storePathName));
try {
Syns2Index.main(commandLineArgs);
@@ -71,8 +73,12 @@ public class TestWordnet extends LuceneT
@Override
public void tearDown() throws Exception {
- searcher.close();
- dir.close();
+ if (searcher != null) {
+ searcher.close();
+ }
+ if (dir != null) {
+ dir.close();
+ }
rmDir(storePathName); // delete our temporary synonym index
super.tearDown();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/CheckIndex.java Mon Apr 4 13:03:42 2011
@@ -733,6 +733,29 @@ public class CheckIndex {
throw new RuntimeException("term " + term + " totalTermFreq=" + totalTermFreq2 + " != recomputed totalTermFreq=" + totalTermFreq);
}
}
+
+ // Test skipping
+ if (docFreq >= 16) {
+ for(int idx=0;idx<7;idx++) {
+ final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
+ docs = terms.docs(delDocs, docs);
+ final int docID = docs.advance(skipDocID);
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ } else {
+ if (docID < skipDocID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + ") returned docID=" + docID);
+ }
+ final int nextDocID = docs.nextDoc();
+ if (nextDocID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ if (nextDocID <= docID) {
+ throw new RuntimeException("term " + term + ": advance(docID=" + skipDocID + "), then .next() returned docID=" + nextDocID + " vs prev docID=" + docID);
+ }
+ }
+ }
+ }
}
if (sumTotalTermFreq != 0) {
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfo.java Mon Apr 4 13:03:42 2011
@@ -20,9 +20,11 @@ package org.apache.lucene.index;
/** @lucene.experimental */
public final class FieldInfo {
public static final int UNASSIGNED_CODEC_ID = -1;
- public String name;
+
+ public final String name;
+ public final int number;
+
public boolean isIndexed;
- public int number;
// true if term vector for this field should be stored
boolean storeTermVector;
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/FieldInfos.java Mon Apr 4 13:03:42 2011
@@ -28,6 +28,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import java.util.Map.Entry;
+import org.apache.lucene.index.SegmentCodecs; // Required for Java 1.5 javadocs
import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
@@ -186,7 +187,7 @@ public final class FieldInfos implements
}
// used by assert
- boolean containsConsistent(Integer number, String name) {
+ synchronized boolean containsConsistent(Integer number, String name) {
return name.equals(numberToName.get(number))
&& number.equals(nameToNumber.get(name));
}
@@ -218,12 +219,13 @@ public final class FieldInfos implements
/**
* Creates a new {@link FieldInfos} instance with a private
- * {@link FieldNumberBiMap} and a default {@link SegmentCodecsBuilder}
+ * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder}
* initialized with {@link CodecProvider#getDefault()}.
* <p>
* Note: this ctor should not be used during indexing use
* {@link FieldInfos#FieldInfos(FieldInfos)} or
- * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} instead.
+ * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)}
+ * instead.
*/
public FieldInfos() {
this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault()));
@@ -552,9 +554,10 @@ public final class FieldInfos implements
/**
* Returns <code>true</code> iff this instance is not backed by a
- * {@link FieldNumberBiMap}. Instances read from a directory via
+ * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap}. Instances read from a directory via
* {@link FieldInfos#FieldInfos(Directory, String)} will always be read-only
- * since no {@link FieldNumberBiMap} is supplied, otherwise <code>false</code>.
+ * since no {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} is supplied, otherwise
+ * <code>false</code>.
*/
public final boolean isReadOnly() {
return globalFieldNumbers == null;
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexReader.java Mon Apr 4 13:03:42 2011
@@ -919,6 +919,22 @@ public abstract class IndexReader implem
}
}
+ /**
+ * Returns <code>true</code> if an index exists at the specified directory.
+ * @param directory the directory to check for an index
+ * @param codecProvider provides a CodecProvider in case the index uses non-core codecs
+ * @return <code>true</code> if an index exists; <code>false</code> otherwise
+ * @throws IOException if there is a problem with accessing the index
+ */
+ public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException {
+ try {
+ new SegmentInfos().read(directory, codecProvider);
+ return true;
+ } catch (IOException ioe) {
+ return false;
+ }
+ }
+
/** Returns the number of documents in this index. */
public abstract int numDocs();
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Apr 4 13:03:42 2011
@@ -871,7 +871,7 @@ public class IndexWriter implements Clos
* message when maxFieldLength is reached will be printed
* to this.
*/
- public void setInfoStream(PrintStream infoStream) {
+ public void setInfoStream(PrintStream infoStream) throws IOException {
ensureOpen();
this.infoStream = infoStream;
docWriter.setInfoStream(infoStream);
@@ -881,7 +881,7 @@ public class IndexWriter implements Clos
messageState();
}
- private void messageState() {
+ private void messageState() throws IOException {
message("\ndir=" + directory + "\n" +
"index=" + segString() + "\n" +
"version=" + Constants.LUCENE_VERSION + "\n" +
@@ -1626,6 +1626,8 @@ public class IndexWriter implements Clos
throws CorruptIndexException, IOException {
ensureOpen();
+ flush(true, true);
+
if (infoStream != null)
message("expungeDeletes: index now " + segString());
@@ -1698,6 +1700,10 @@ public class IndexWriter implements Clos
* documents, so you must do so yourself if necessary.
* See also {@link #expungeDeletes(boolean)}
*
+ * <p><b>NOTE</b>: this method first flushes a new
+ * segment (if there are indexed documents), and applies
+ * all buffered deletes.
+ *
* <p><b>NOTE</b>: if this method hits an OutOfMemoryError
* you should immediately close the writer. See <a
* href="#OOME">above</a> for details.</p>
@@ -2644,7 +2650,7 @@ public class IndexWriter implements Clos
return docWriter.getNumDocs();
}
- private void ensureValidMerge(MergePolicy.OneMerge merge) {
+ private void ensureValidMerge(MergePolicy.OneMerge merge) throws IOException {
for(SegmentInfo info : merge.segments) {
if (segmentInfos.indexOf(info) == -1) {
throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
@@ -2935,7 +2941,7 @@ public class IndexWriter implements Clos
* are now participating in a merge, and true is
* returned. Else (the merge conflicts) false is
* returned. */
- final synchronized boolean registerMerge(MergePolicy.OneMerge merge) throws MergePolicy.MergeAbortedException {
+ final synchronized boolean registerMerge(MergePolicy.OneMerge merge) throws MergePolicy.MergeAbortedException, IOException {
if (merge.registerDone)
return true;
@@ -2945,10 +2951,8 @@ public class IndexWriter implements Clos
throw new MergePolicy.MergeAbortedException("merge is aborted: " + merge.segString(directory));
}
- final int count = merge.segments.size();
boolean isExternal = false;
- for(int i=0;i<count;i++) {
- final SegmentInfo info = merge.segments.info(i);
+ for(SegmentInfo info : merge.segments) {
if (mergingSegments.contains(info)) {
return false;
}
@@ -2978,12 +2982,15 @@ public class IndexWriter implements Clos
// is running (while synchronized) to avoid race
// condition where two conflicting merges from different
// threads, start
- for(int i=0;i<count;i++) {
- mergingSegments.add(merge.segments.info(i));
+ message("registerMerge merging=" + mergingSegments);
+ for(SegmentInfo info : merge.segments) {
+ message("registerMerge info=" + info);
+ mergingSegments.add(info);
}
// Merge is now registered
merge.registerDone = true;
+
return true;
}
@@ -3068,6 +3075,10 @@ public class IndexWriter implements Clos
message("merge seg=" + merge.info.name);
}
+ // TODO: I think this should no longer be needed (we
+ // now build CFS before adding segment to the infos);
+ // however, on removing it, tests fail for some reason!
+
// Also enroll the merged segment into mergingSegments;
// this prevents it from getting selected for a merge
// after our merge is done but while we are building the
@@ -3106,10 +3117,11 @@ public class IndexWriter implements Clos
// exception inside mergeInit
if (merge.registerDone) {
final SegmentInfos sourceSegments = merge.segments;
- final int end = sourceSegments.size();
- for(int i=0;i<end;i++) {
- mergingSegments.remove(sourceSegments.info(i));
+ for(SegmentInfo info : sourceSegments) {
+ mergingSegments.remove(info);
}
+ // TODO: if we remove the add in _mergeInit, we should
+ // also remove this:
mergingSegments.remove(merge.info);
merge.registerDone = false;
}
@@ -3188,6 +3200,8 @@ public class IndexWriter implements Clos
merge.readers = new ArrayList<SegmentReader>();
merge.readerClones = new ArrayList<SegmentReader>();
+ merge.estimatedMergeBytes = 0;
+
// This is try/finally to make sure merger's readers are
// closed:
boolean success = false;
@@ -3205,6 +3219,13 @@ public class IndexWriter implements Clos
-config.getReaderTermsIndexDivisor());
merge.readers.add(reader);
+ final int readerMaxDoc = reader.maxDoc();
+ if (readerMaxDoc > 0) {
+ final int delCount = reader.numDeletedDocs();
+ final double delRatio = ((double) delCount)/readerMaxDoc;
+ merge.estimatedMergeBytes += info.sizeInBytes(true) * (1.0 - delRatio);
+ }
+
// We clone the segment readers because other
// deletes may come in while we're merging so we
// need readers that will not change
@@ -3306,8 +3327,11 @@ public class IndexWriter implements Clos
merge.info.setUseCompoundFile(true);
}
- final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
+ if (infoStream != null) {
+ message(String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+ }
+ final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
final int termsIndexDivisor;
final boolean loadDocStores;
@@ -3381,21 +3405,41 @@ public class IndexWriter implements Clos
return segmentInfos.size() > 0 ? segmentInfos.info(segmentInfos.size()-1) : null;
}
- public synchronized String segString() {
+ /** @lucene.internal */
+ public synchronized String segString() throws IOException {
return segString(segmentInfos);
}
- private synchronized String segString(SegmentInfos infos) {
+ /** @lucene.internal */
+ public synchronized String segString(SegmentInfos infos) throws IOException {
StringBuilder buffer = new StringBuilder();
final int count = infos.size();
for(int i = 0; i < count; i++) {
if (i > 0) {
buffer.append(' ');
}
- final SegmentInfo info = infos.info(i);
- buffer.append(info.toString(directory, 0));
- if (info.dir != directory)
- buffer.append("**");
+ buffer.append(segString(infos.info(i)));
+ }
+
+ return buffer.toString();
+ }
+
+ public synchronized String segString(SegmentInfo info) throws IOException {
+ StringBuilder buffer = new StringBuilder();
+ SegmentReader reader = readerPool.getIfExists(info);
+ try {
+ if (reader != null) {
+ buffer.append(reader.toString());
+ } else {
+ buffer.append(info.toString(directory, 0));
+ if (info.dir != directory) {
+ buffer.append("**");
+ }
+ }
+ } finally {
+ if (reader != null) {
+ readerPool.release(reader);
+ }
}
return buffer.toString();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Mon Apr 4 13:03:42 2011
@@ -160,7 +160,7 @@ public final class IndexWriterConfig imp
indexingChain = DocumentsWriterPerThread.defaultIndexingChain;
mergedSegmentWarmer = null;
codecProvider = CodecProvider.getDefault();
- mergePolicy = new LogByteSizeMergePolicy();
+ mergePolicy = new TieredMergePolicy();
readerPooling = DEFAULT_READER_POOLING;
indexerThreadPool = new ThreadAffinityDocumentsWriterThreadPool(DEFAULT_MAX_THREAD_STATES);
readerTermsIndexDivisor = DEFAULT_READER_TERMS_INDEX_DIVISOR;
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Mon Apr 4 13:03:42 2011
@@ -20,7 +20,6 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
@@ -72,7 +71,6 @@ public abstract class LogMergePolicy ext
// out there wrote his own LMP ...
protected long maxMergeSizeForOptimize = Long.MAX_VALUE;
protected int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
- protected boolean requireContiguousMerge = false;
protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
@@ -111,21 +109,6 @@ public abstract class LogMergePolicy ext
writer.get().message("LMP: " + message);
}
- /** If true, merges must be in-order slice of the
- * segments. If false, then the merge policy is free to
- * pick any segments. The default is false, which is
- * in general more efficient than true since it gives the
- * merge policy more freedom to pick closely sized
- * segments. */
- public void setRequireContiguousMerge(boolean v) {
- requireContiguousMerge = v;
- }
-
- /** See {@link #setRequireContiguousMerge}. */
- public boolean getRequireContiguousMerge() {
- return requireContiguousMerge;
- }
-
/** <p>Returns the number of segments that are merged at
* once and also controls the total number of segments
* allowed to accumulate in the index.</p> */
@@ -378,8 +361,6 @@ public abstract class LogMergePolicy ext
return null;
}
- // TODO: handle non-contiguous merge case differently?
-
// Find the newest (rightmost) segment that needs to
// be optimized (other segments may have been flushed
// since optimize started):
@@ -499,14 +480,6 @@ public abstract class LogMergePolicy ext
}
}
- private static class SortByIndex implements Comparator<SegmentInfoAndLevel> {
- public int compare(SegmentInfoAndLevel o1, SegmentInfoAndLevel o2) {
- return o1.index - o2.index;
- }
- }
-
- private static final SortByIndex sortByIndex = new SortByIndex();
-
/** Checks if any merges are now necessary and returns a
* {@link MergePolicy.MergeSpecification} if so. A merge
* is necessary when there are more than {@link
@@ -532,31 +505,24 @@ public abstract class LogMergePolicy ext
final SegmentInfo info = infos.info(i);
long size = size(info);
- // When we require contiguous merge, we still add the
- // segment to levels to avoid merging "across" a set
- // of segment being merged:
- if (!requireContiguousMerge && mergingSegments.contains(info)) {
- if (verbose()) {
- message("seg " + info.name + " already being merged; skip");
- }
- continue;
- }
-
// Floor tiny segments
if (size < 1) {
size = 1;
}
+
final SegmentInfoAndLevel infoLevel = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);
levels.add(infoLevel);
+
if (verbose()) {
- message("seg " + info.name + " level=" + infoLevel.level + " size=" + size);
+ final long segBytes = sizeBytes(info);
+ String extra = mergingSegments.contains(info) ? " [merging]" : "";
+ if (size >= maxMergeSize) {
+ extra += " [skip: too large]";
+ }
+ message("seg=" + writer.get().segString(info) + " level=" + infoLevel.level + " size=" + String.format("%.3f MB", segBytes/1024/1024.) + extra);
}
}
- if (!requireContiguousMerge) {
- Collections.sort(levels);
- }
-
final float levelFloor;
if (minMergeSize <= 0)
levelFloor = (float) 0.0;
@@ -614,23 +580,29 @@ public abstract class LogMergePolicy ext
int end = start + mergeFactor;
while(end <= 1+upto) {
boolean anyTooLarge = false;
+ boolean anyMerging = false;
for(int i=start;i<end;i++) {
final SegmentInfo info = levels.get(i).info;
anyTooLarge |= (size(info) >= maxMergeSize || sizeDocs(info) >= maxMergeDocs);
+ if (mergingSegments.contains(info)) {
+ anyMerging = true;
+ break;
+ }
}
- if (!anyTooLarge) {
+ if (anyMerging) {
+ // skip
+ } else if (!anyTooLarge) {
if (spec == null)
spec = new MergeSpecification();
- if (verbose()) {
- message(" " + start + " to " + end + ": add this merge");
- }
- Collections.sort(levels.subList(start, end), sortByIndex);
final SegmentInfos mergeInfos = new SegmentInfos();
for(int i=start;i<end;i++) {
mergeInfos.add(levels.get(i).info);
assert infos.contains(levels.get(i).info);
}
+ if (verbose()) {
+ message(" add merge=" + writer.get().segString(mergeInfos) + " start=" + start + " end=" + end);
+ }
spec.add(new OneMerge(mergeInfos));
} else if (verbose()) {
message(" " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
@@ -682,7 +654,7 @@ public abstract class LogMergePolicy ext
sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");
sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");
sb.append("useCompoundFile=").append(useCompoundFile).append(", ");
- sb.append("requireContiguousMerge=").append(requireContiguousMerge);
+ sb.append("noCFSRatio=").append(noCFSRatio);
sb.append("]");
return sb.toString();
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MergePolicy.java Mon Apr 4 13:03:42 2011
@@ -72,6 +72,7 @@ public abstract class MergePolicy implem
long mergeGen; // used by IndexWriter
boolean isExternal; // used by IndexWriter
int maxNumSegmentsOptimize; // used by IndexWriter
+ long estimatedMergeBytes; // used by IndexWriter
List<SegmentReader> readers; // used by IndexWriter
List<SegmentReader> readerClones; // used by IndexWriter
public final SegmentInfos segments;
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Mon Apr 4 13:03:42 2011
@@ -670,7 +670,6 @@ public final class SegmentInfo {
StringBuilder s = new StringBuilder();
s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':');
-
char cfs = getUseCompoundFile() ? 'c' : 'C';
s.append(cfs);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon Apr 4 13:03:42 2011
@@ -55,6 +55,9 @@ public class SegmentReader extends Index
AtomicInteger deletedDocsRef = null;
private boolean deletedDocsDirty = false;
private boolean normsDirty = false;
+
+ // TODO: we should move this tracking into SegmentInfo;
+ // this way SegmentInfo.toString shows pending deletes
private int pendingDeleteCount;
private boolean rollbackHasChanges = false;
@@ -803,8 +806,9 @@ public class SegmentReader extends Index
oldRef.decrementAndGet();
}
deletedDocsDirty = true;
- if (!deletedDocs.getAndSet(docNum))
+ if (!deletedDocs.getAndSet(docNum)) {
pendingDeleteCount++;
+ }
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/util/PriorityQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/util/PriorityQueue.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/util/PriorityQueue.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/util/PriorityQueue.java Mon Apr 4 13:03:42 2011
@@ -21,8 +21,10 @@ package org.apache.lucene.util;
* least element can always be found in constant time. Put()'s and pop()'s
* require log(size) time.
*
- * <p><b>NOTE</b>: This class pre-allocates a full array of
- * length <code>maxSize+1</code>, in {@link #initialize}.
+ * <p><b>NOTE</b>: This class will pre-allocate a full array of
+ * length <code>maxSize+1</code> if instantiated via the
+ * {@link #PriorityQueue(int,boolean)} constructor with
+ * <code>prepopulate</code> set to <code>true</code>.
*
* @lucene.internal
*/
@@ -83,9 +85,10 @@ public abstract class PriorityQueue<T> {
/**
* This method can be overridden by extending classes to return a sentinel
- * object which will be used by {@link #initialize(int)} to fill the queue, so
- * that the code which uses that queue can always assume it's full and only
- * change the top without attempting to insert any new object.<br>
+ * object which will be used by the {@link PriorityQueue#PriorityQueue(int,boolean)}
+ * constructor to fill the queue, so that the code which uses that queue can always
+ * assume it's full and only change the top without attempting to insert any new
+ * object.<br>
*
* Those sentinel values should always compare worse than any non-sentinel
* value (i.e., {@link #lessThan} should always favor the
@@ -111,11 +114,11 @@ public abstract class PriorityQueue<T> {
* </pre>
*
* <b>NOTE:</b> if this method returns a non-null value, it will be called by
- * {@link #initialize(int)} {@link #size()} times, relying on a new object to
- * be returned and will not check if it's null again. Therefore you should
- * ensure any call to this method creates a new instance and behaves
- * consistently, e.g., it cannot return null if it previously returned
- * non-null.
+ * the {@link PriorityQueue#PriorityQueue(int,boolean)} constructor
+ * {@link #size()} times, relying on a new object to be returned and will not
+ * check if it's null again. Therefore you should ensure any call to this
+ * method creates a new instance and behaves consistently, e.g., it cannot
+ * return null if it previously returned non-null.
*
* @return the sentinel object to use to pre-populate the queue, or null if
* sentinel objects are not supported.
Modified: lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon Apr 4 13:03:42 2011
@@ -163,7 +163,7 @@ public class RandomIndexWriter implement
System.out.println("RIW.getReader: open new reader");
}
w.commit();
- return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10));
+ return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10), w.getConfig().getCodecProvider());
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Mon Apr 4 13:03:42 2011
@@ -32,6 +32,7 @@ import java.util.Random;
import java.util.Set;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThrottledIndexOutput;
import org.apache.lucene.util._TestUtil;
@@ -413,12 +414,27 @@ public class MockDirectoryWrapper extend
throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause);
}
open = false;
- if (checkIndexOnClose && IndexReader.indexExists(this)) {
- _TestUtil.checkIndex(this);
+ if (checkIndexOnClose) {
+ if (codecProvider != null) {
+ if (IndexReader.indexExists(this, codecProvider)) {
+ _TestUtil.checkIndex(this, codecProvider);
+ }
+ } else {
+ if (IndexReader.indexExists(this)) {
+ _TestUtil.checkIndex(this);
+ }
+ }
}
delegate.close();
}
+ private CodecProvider codecProvider;
+
+ // We pass this CodecProvider to checkIndex when dir is closed...
+ public void setCodecProvider(CodecProvider cp) {
+ codecProvider = cp;
+ }
+
boolean open = true;
public synchronized boolean isOpen() {
Modified: lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java Mon Apr 4 13:03:42 2011
@@ -128,7 +128,7 @@ public class LineFileDocs implements Clo
body = new Field("body", "", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.add(body);
- id = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
+ id = new Field("docid", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
doc.add(id);
date = new Field("date", "", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS);
Modified: lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Mon Apr 4 13:03:42 2011
@@ -142,6 +142,8 @@ public abstract class LuceneTestCase ext
public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random");
/** Get the number of times to run tests */
public static final int TEST_ITER = Integer.parseInt(System.getProperty("tests.iter", "1"));
+ /** Get the minimum number of times to run tests until a failure happens */
+ public static final int TEST_ITER_MIN = Integer.parseInt(System.getProperty("tests.iter.min", Integer.toString(TEST_ITER)));
/** Get the random seed for tests */
public static final String TEST_SEED = System.getProperty("tests.seed", "random");
/** whether or not nightly tests should run */
@@ -765,9 +767,11 @@ public abstract class LuceneTestCase ext
}
if (r.nextBoolean()) {
- c.setMergePolicy(new MockRandomMergePolicy(r));
- } else {
+ c.setMergePolicy(newTieredMergePolicy());
+ } else if (r.nextBoolean()) {
c.setMergePolicy(newLogMergePolicy());
+ } else {
+ c.setMergePolicy(new MockRandomMergePolicy(r));
}
c.setReaderPooling(r.nextBoolean());
@@ -779,6 +783,10 @@ public abstract class LuceneTestCase ext
return newLogMergePolicy(random);
}
+ public static TieredMergePolicy newTieredMergePolicy() {
+ return newTieredMergePolicy(random);
+ }
+
public static LogMergePolicy newLogMergePolicy(Random r) {
LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy();
logmp.setUseCompoundFile(r.nextBoolean());
@@ -791,17 +799,22 @@ public abstract class LuceneTestCase ext
return logmp;
}
- public static LogMergePolicy newInOrderLogMergePolicy() {
- LogMergePolicy logmp = newLogMergePolicy();
- logmp.setRequireContiguousMerge(true);
- return logmp;
- }
-
- public static LogMergePolicy newInOrderLogMergePolicy(int mergeFactor) {
- LogMergePolicy logmp = newLogMergePolicy();
- logmp.setMergeFactor(mergeFactor);
- logmp.setRequireContiguousMerge(true);
- return logmp;
+ public static TieredMergePolicy newTieredMergePolicy(Random r) {
+ TieredMergePolicy tmp = new TieredMergePolicy();
+ if (r.nextInt(3) == 2) {
+ tmp.setMaxMergeAtOnce(2);
+ tmp.setMaxMergeAtOnceExplicit(2);
+ } else {
+ tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 20));
+ tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 30));
+ }
+ tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0);
+ tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0);
+ tmp.setExpungeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0);
+ tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
+ tmp.setUseCompoundFile(r.nextBoolean());
+ tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ return tmp;
}
public static LogMergePolicy newLogMergePolicy(boolean useCFS) {
@@ -1202,11 +1215,24 @@ public abstract class LuceneTestCase ext
if (VERBOSE) {
System.out.println("\nNOTE: running test " + arg0.getName());
}
+
+ // only print iteration info if the user requested more than one iterations
+ boolean verbose = VERBOSE && TEST_ITER > 1;
+ int lastIterFailed = -1;
for (int i = 0; i < TEST_ITER; i++) {
- if (VERBOSE && TEST_ITER > 1) {
+ if (verbose) {
System.out.println("\nNOTE: running iter=" + (1+i) + " of " + TEST_ITER);
}
super.runChild(arg0, arg1);
+ if (testsFailed) {
+ lastIterFailed = i;
+ if (i == TEST_ITER_MIN - 1) {
+ if (verbose) {
+ System.out.println("\nNOTE: iteration " + lastIterFailed + " failed !");
+ }
+ break;
+ }
+ }
}
}
Modified: lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Mon Apr 4 13:03:42 2011
@@ -43,10 +43,13 @@ import org.apache.lucene.index.Concurren
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
+import org.junit.Assert;
public class _TestUtil {
@@ -157,6 +160,19 @@ public class _TestUtil {
return start + r.nextInt(end-start+1);
}
+ public static String randomSimpleString(Random r) {
+ final int end = r.nextInt(10);
+ if (end == 0) {
+ // allow 0 length
+ return "";
+ }
+ final char[] buffer = new char[end];
+ for (int i = 0; i < end; i++) {
+ buffer[i] = (char) _TestUtil.nextInt(r, 97, 102);
+ }
+ return new String(buffer, 0, end);
+ }
+
/** Returns random string, including full unicode range. */
public static String randomUnicodeString(Random r) {
return randomUnicodeString(r, 20);
@@ -294,9 +310,14 @@ public class _TestUtil {
* count lowish */
public static void reduceOpenFiles(IndexWriter w) {
// keep number of open files lowish
- LogMergePolicy lmp = (LogMergePolicy) w.getConfig().getMergePolicy();
- lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
-
+ MergePolicy mp = w.getConfig().getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ LogMergePolicy lmp = (LogMergePolicy) mp;
+ lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
+ } else if (mp instanceof TieredMergePolicy) {
+ TieredMergePolicy tmp = (TieredMergePolicy) mp;
+ tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
+ }
MergeScheduler ms = w.getConfig().getMergeScheduler();
if (ms instanceof ConcurrentMergeScheduler) {
((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Mon Apr 4 13:03:42 2011
@@ -1073,8 +1073,9 @@ public class TestAddIndexes extends Luce
IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) };
Directory dir = new RAMDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy());
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
+ lmp.setUseCompoundFile(true);
lmp.setNoCFSRatio(1.0); // Force creation of CFS
IndexWriter w3 = new IndexWriter(dir, conf);
w3.addIndexes(readers);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java Mon Apr 4 13:03:42 2011
@@ -129,7 +129,7 @@ public class TestAtomicUpdate extends Lu
IndexWriterConfig conf = new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer())
.setMaxBufferedDocs(7);
- ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(3);
+ ((TieredMergePolicy) conf.getMergePolicy()).setMaxMergeAtOnce(3);
IndexWriter writer = new MockIndexWriter(directory, conf);
writer.setInfoStream(VERBOSE ? System.out : null);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Mon Apr 4 13:03:42 2011
@@ -619,7 +619,7 @@ public class TestDeletionPolicy extends
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer())
- .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newInOrderLogMergePolicy());
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
MergePolicy mp = conf.getMergePolicy();
if (mp instanceof LogMergePolicy) {
((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java Mon Apr 4 13:03:42 2011
@@ -116,7 +116,7 @@ public class TestDocsAndPositions extend
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(
- MockTokenizer.WHITESPACE, true, usePayload)).setMergePolicy(newInOrderLogMergePolicy()));
+ MockTokenizer.WHITESPACE, true, usePayload)).setMergePolicy(newLogMergePolicy()));
int numDocs = 131;
int max = 1051;
int term = random.nextInt(max);
@@ -197,7 +197,7 @@ public class TestDocsAndPositions extend
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(
- MockTokenizer.WHITESPACE, true, usePayload)).setMergePolicy(newInOrderLogMergePolicy()));
+ MockTokenizer.WHITESPACE, true, usePayload)).setMergePolicy(newLogMergePolicy()));
int numDocs = 499;
int max = 15678;
int term = random.nextInt(max);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Mon Apr 4 13:03:42 2011
@@ -371,7 +371,7 @@ public class TestIndexReader extends Luc
Directory dir = newDirectory();
byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < 10; i++) {
addDoc(writer, "document number " + (i + 1));
@@ -380,7 +380,7 @@ public class TestIndexReader extends Luc
addDocumentWithTermVectorFields(writer);
}
writer.close();
- writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMergePolicy(newInOrderLogMergePolicy()));
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
doc.add(new Field("bin1", bin));
doc.add(new Field("junk", "junk text", Field.Store.NO, Field.Index.ANALYZED));
@@ -417,7 +417,7 @@ public class TestIndexReader extends Luc
// force optimize
- writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMergePolicy(newInOrderLogMergePolicy()));
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
writer.optimize();
writer.close();
reader = IndexReader.open(dir, false);
@@ -1163,7 +1163,7 @@ public class TestIndexReader extends Luc
public void testMultiReaderDeletes() throws Exception {
Directory dir = newDirectory();
- RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ RandomIndexWriter w= new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
Document doc = new Document();
doc.add(newField("f", "doctor", Field.Store.NO, Field.Index.NOT_ANALYZED));
w.addDocument(doc);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Mon Apr 4 13:03:42 2011
@@ -174,7 +174,7 @@ public class TestIndexReaderReopen exten
private void doTestReopenWithCommit (Random random, Directory dir, boolean withReopen) throws IOException {
IndexWriter iwriter = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(
- OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newInOrderLogMergePolicy()));
+ OpenMode.CREATE).setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(newLogMergePolicy()));
iwriter.commit();
IndexReader reader = IndexReader.open(dir, false);
try {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Mon Apr 4 13:03:42 2011
@@ -68,7 +68,7 @@ public class TestIndexWriterConfig exten
assertNull(conf.getMergedSegmentWarmer());
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
assertEquals(IndexWriterConfig.DEFAULT_READER_TERMS_INDEX_DIVISOR, conf.getReaderTermsIndexDivisor());
- assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass());
+ assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass());
assertNull(conf.getFlushPolicy());
assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB());
@@ -269,7 +269,7 @@ public class TestIndexWriterConfig exten
assertEquals(IndexWriterConfig.DEFAULT_MAX_THREAD_STATES, conf.getMaxThreadStates());
// Test MergePolicy
- assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass());
+ assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
conf.setMergePolicy(new LogDocMergePolicy());
assertEquals(LogDocMergePolicy.class, conf.getMergePolicy().getClass());
conf.setMergePolicy(null);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Mon Apr 4 13:03:42 2011
@@ -104,7 +104,7 @@ public class TestIndexWriterMergePolicy
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
setMaxBufferedDocs(10).
- setMergePolicy(newInOrderLogMergePolicy())
+ setMergePolicy(newLogMergePolicy())
);
for (int i = 0; i < 250; i++) {
@@ -131,7 +131,8 @@ public class TestIndexWriterMergePolicy
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer())
- .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy()));
+ .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
+ .setMergeScheduler(new SerialMergeScheduler()));
// leftmost* segment has 1 doc
// rightmost* segment has 100 docs
@@ -144,8 +145,8 @@ public class TestIndexWriterMergePolicy
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer()).setOpenMode(
- OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(
- new LogDocMergePolicy()));
+ OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
+ .setMergeScheduler(new SerialMergeScheduler()));
}
writer.close();
@@ -153,7 +154,7 @@ public class TestIndexWriterMergePolicy
ldmp.setMergeFactor(10);
writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
new MockAnalyzer()).setOpenMode(
- OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
+ OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new SerialMergeScheduler()));
// merge policy only fixes segments on levels where merges
// have been triggered, so check invariants after all adds
@@ -236,13 +237,13 @@ public class TestIndexWriterMergePolicy
int segmentCount = writer.getSegmentCount();
for (int i = segmentCount - 1; i >= 0; i--) {
int docCount = writer.getDocCount(i);
- assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString(), docCount > lowerBound);
+ assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " upperBound=" + upperBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString() + " config=" + writer.getConfig(), docCount > lowerBound);
if (docCount <= upperBound) {
numSegments++;
} else {
if (upperBound * mergeFactor <= maxMergeDocs) {
- assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString(), numSegments < mergeFactor);
+ assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString() + " config=" + writer.getConfig(), numSegments < mergeFactor);
}
do {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Mon Apr 4 13:03:42 2011
@@ -58,7 +58,7 @@ public class TestIndexWriterMerging exte
IndexWriter writer = new IndexWriter(
merged,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
- setMergePolicy(newInOrderLogMergePolicy(2))
+ setMergePolicy(newLogMergePolicy(2))
);
writer.setInfoStream(VERBOSE ? System.out : null);
writer.addIndexes(indexA, indexB);
@@ -101,7 +101,7 @@ public class TestIndexWriterMerging exte
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
setOpenMode(OpenMode.CREATE).
setMaxBufferedDocs(2).
- setMergePolicy(newInOrderLogMergePolicy(2))
+ setMergePolicy(newLogMergePolicy(2))
);
for (int i = start; i < (start + numDocs); i++)
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Mon Apr 4 13:03:42 2011
@@ -46,7 +46,7 @@ import org.apache.lucene.util.ThreadInte
import java.util.concurrent.atomic.AtomicInteger;
public class TestIndexWriterReader extends LuceneTestCase {
- static PrintStream infoStream;
+ static PrintStream infoStream = VERBOSE ? System.out : null;
public static int count(Term t, IndexReader r) throws IOException {
int count = 0;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Mon Apr 4 13:03:42 2011
@@ -47,7 +47,7 @@ public class TestMaxTermFrequency extend
super.setUp();
dir = newDirectory();
IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new MockAnalyzer(MockTokenizer.SIMPLE, true)).setMergePolicy(newInOrderLogMergePolicy());
+ new MockAnalyzer(MockTokenizer.SIMPLE, true)).setMergePolicy(newLogMergePolicy());
config.setSimilarityProvider(new DefaultSimilarityProvider() {
@Override
public Similarity get(String field) {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Mon Apr 4 13:03:42 2011
@@ -71,7 +71,7 @@ public class TestMultiLevelSkipList exte
public void testSimpleSkip() throws IOException {
Directory dir = new CountingRAMDirectory(new RAMDirectory());
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newInOrderLogMergePolicy()));
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newLogMergePolicy()));
Term term = new Term("test", "a");
for (int i = 0; i < 5000; i++) {
Document d1 = new Document();
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Mon Apr 4 13:03:42 2011
@@ -102,18 +102,7 @@ public class TestNRTThreads extends Luce
if (VERBOSE) {
writer.setInfoStream(System.out);
}
- MergeScheduler ms = writer.getConfig().getMergeScheduler();
- if (ms instanceof ConcurrentMergeScheduler) {
- // try to keep max file open count down
- ((ConcurrentMergeScheduler) ms).setMaxThreadCount(1);
- ((ConcurrentMergeScheduler) ms).setMaxMergeCount(1);
- }
- /*
- LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
- if (lmp.getMergeFactor() > 5) {
- lmp.setMergeFactor(5);
- }
- */
+ _TestUtil.reduceOpenFiles(writer);
final int NUM_INDEX_THREADS = 2;
final int NUM_SEARCH_THREADS = 3;
@@ -147,36 +136,36 @@ public class TestNRTThreads extends Luce
}
if (random.nextBoolean()) {
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": add doc id:" + doc.get("id"));
+ //System.out.println(Thread.currentThread().getName() + ": add doc id:" + doc.get("docid"));
}
writer.addDocument(doc);
} else {
// we use update but it never replaces a
// prior doc
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("id"));
+ //System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid"));
}
- writer.updateDocument(new Term("id", doc.get("id")), doc);
+ writer.updateDocument(new Term("docid", doc.get("docid")), doc);
}
if (random.nextInt(5) == 3) {
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("id"));
+ //System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid"));
}
- toDeleteIDs.add(doc.get("id"));
+ toDeleteIDs.add(doc.get("docid"));
}
if (random.nextInt(50) == 17) {
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes");
+ //System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes");
}
for(String id : toDeleteIDs) {
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": del term=id:" + id);
+ //System.out.println(Thread.currentThread().getName() + ": del term=id:" + id);
}
- writer.deleteDocuments(new Term("id", id));
+ writer.deleteDocuments(new Term("docid", id));
}
final int count = delCount.addAndGet(toDeleteIDs.size());
if (VERBOSE) {
- System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes");
+ //System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes");
}
delIDs.addAll(toDeleteIDs);
toDeleteIDs.clear();
@@ -357,18 +346,18 @@ public class TestNRTThreads extends Luce
final IndexSearcher s = newSearcher(r2);
boolean doFail = false;
for(String id : delIDs) {
- final TopDocs hits = s.search(new TermQuery(new Term("id", id)), 1);
+ final TopDocs hits = s.search(new TermQuery(new Term("docid", id)), 1);
if (hits.totalHits != 0) {
System.out.println("doc id=" + id + " is supposed to be deleted, but got docID=" + hits.scoreDocs[0].doc);
doFail = true;
}
}
- final int endID = Integer.parseInt(docs.nextDoc().get("id"));
+ final int endID = Integer.parseInt(docs.nextDoc().get("docid"));
for(int id=0;id<endID;id++) {
String stringID = ""+id;
if (!delIDs.contains(stringID)) {
- final TopDocs hits = s.search(new TermQuery(new Term("id", stringID)), 1);
+ final TopDocs hits = s.search(new TermQuery(new Term("docid", stringID)), 1);
if (hits.totalHits != 1) {
System.out.println("doc id=" + stringID + " is not supposed to be deleted, but got hitCount=" + hits.totalHits);
doFail = true;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestNorms.java Mon Apr 4 13:03:42 2011
@@ -158,7 +158,7 @@ public class TestNorms extends LuceneTes
private void createIndex(Random random, Directory dir) throws IOException {
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newInOrderLogMergePolicy()));
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
@@ -203,7 +203,7 @@ public class TestNorms extends LuceneTes
private void addDocs(Random random, Directory dir, int ndocs, boolean compound) throws IOException {
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newInOrderLogMergePolicy()));
+ .setMaxBufferedDocs(5).setSimilarityProvider(similarityProviderOne).setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java Mon Apr 4 13:03:42 2011
@@ -264,7 +264,7 @@ public class TestOmitNorms extends Lucen
*/
static byte[] getNorms(String field, Field f1, Field f2) throws IOException {
Directory dir = newDirectory();
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy());
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy());
RandomIndexWriter riw = new RandomIndexWriter(random, dir, iwc);
// add f1
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Mon Apr 4 13:03:42 2011
@@ -257,7 +257,7 @@ public class TestOmitTf extends LuceneTe
newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).
setMaxBufferedDocs(2).
setSimilarityProvider(new SimpleSimilarityProvider()).
- setMergePolicy(newInOrderLogMergePolicy(2))
+ setMergePolicy(newLogMergePolicy(2))
);
writer.setInfoStream(VERBOSE ? System.out : null);
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestPayloads.java Mon Apr 4 13:03:42 2011
@@ -164,7 +164,7 @@ public class TestPayloads extends Lucene
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, analyzer)
.setOpenMode(OpenMode.CREATE)
- .setMergePolicy(newInOrderLogMergePolicy()));
+ .setMergePolicy(newLogMergePolicy()));
// should be in sync with value in TermInfosWriter
final int skipInterval = 16;
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java Mon Apr 4 13:03:42 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.*;
import org.apache.lucene.util.*;
import org.junit.Test;
@@ -48,22 +47,8 @@ public class TestRollingUpdates extends
} else {
id++;
}
- doc.getField("id").setValue(myID);
- int mode = docIter % 3;
- switch (mode) {
- case 0: {
- w.deleteDocuments(new Term("id", myID));
- w.addDocument(doc);
- break;
- }
- case 1: {
- w.deleteDocuments(new TermQuery(new Term("id", myID)));
- w.addDocument(doc);
- break;
- }
- default : w.updateDocument(new Term("id", myID), doc);
- }
-
+ doc.getField("docid").setValue(myID);
+ w.updateDocument(new Term("docid", myID), doc);
if (docIter >= SIZE && random.nextInt(50) == 17) {
if (r != null) {
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Mon Apr 4 13:03:42 2011
@@ -105,7 +105,7 @@ public class TestSegmentTermDocs extends
public void testSkipTo(int indexDivisor) throws IOException {
Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
Term ta = new Term("content","aaa");
for(int i = 0; i < 10; i++)
Modified: lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/test/org/apache/lucene/search/TestBoolean2.java Mon Apr 4 13:03:42 2011
@@ -54,7 +54,7 @@ public class TestBoolean2 extends Lucene
@BeforeClass
public static void beforeClass() throws Exception {
directory = newDirectory();
- RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
+ RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < docFields.length; i++) {
Document doc = new Document();
doc.add(newField(field, docFields[i], Field.Store.NO, Field.Index.ANALYZED));