You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/25 01:07:38 UTC
svn commit: r1139458 [9/24] - in /lucene/dev/branches/solr2193: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/
dev-tools/idea/solr/ dev-tools/idea/solr/contrib/clustering/
dev-tools/maven/ dev-tools/maven/modules/analysis/icu/ dev-t...
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Jun 24 23:07:10 2011
@@ -73,6 +73,7 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
public class TestIndexWriter extends LuceneTestCase {
@@ -133,14 +134,14 @@ public class TestIndexWriter extends Luc
dir.close();
}
- private void addDoc(IndexWriter writer) throws IOException
+ static void addDoc(IndexWriter writer) throws IOException
{
Document doc = new Document();
doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
writer.addDocument(doc);
}
- private void addDocWithIndex(IndexWriter writer, int index) throws IOException
+ static void addDocWithIndex(IndexWriter writer, int index) throws IOException
{
Document doc = new Document();
doc.add(newField("content", "aaa " + index, Field.Store.YES, Field.Index.ANALYZED));
@@ -165,142 +166,6 @@ public class TestIndexWriter extends Luc
}
}
- public void testOptimizeMaxNumSegments() throws IOException {
-
- MockDirectoryWrapper dir = newDirectory();
-
- final Document doc = new Document();
- doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
- final int incrMin = TEST_NIGHTLY ? 15 : 40;
- for(int numDocs=10;numDocs<500;numDocs += _TestUtil.nextInt(random, incrMin, 5*incrMin)) {
- LogDocMergePolicy ldmp = new LogDocMergePolicy();
- ldmp.setMinMergeDocs(1);
- ldmp.setMergeFactor(5);
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(2).setMergePolicy(
- ldmp));
- for(int j=0;j<numDocs;j++)
- writer.addDocument(doc);
- writer.close();
-
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
- final int segCount = sis.size();
-
- ldmp = new LogDocMergePolicy();
- ldmp.setMergeFactor(5);
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMergePolicy(ldmp));
- writer.optimize(3);
- writer.close();
-
- sis = new SegmentInfos();
- sis.read(dir);
- final int optSegCount = sis.size();
-
- if (segCount < 3)
- assertEquals(segCount, optSegCount);
- else
- assertEquals(3, optSegCount);
- }
- dir.close();
- }
-
- public void testOptimizeMaxNumSegments2() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
-
- final Document doc = new Document();
- doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
-
- LogDocMergePolicy ldmp = new LogDocMergePolicy();
- ldmp.setMinMergeDocs(1);
- ldmp.setMergeFactor(4);
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
-
- for(int iter=0;iter<10;iter++) {
- for(int i=0;i<19;i++)
- writer.addDocument(doc);
-
- writer.commit();
- writer.waitForMerges();
- writer.commit();
-
- SegmentInfos sis = new SegmentInfos();
- sis.read(dir);
-
- final int segCount = sis.size();
-
- writer.optimize(7);
- writer.commit();
- writer.waitForMerges();
-
- sis = new SegmentInfos();
- sis.read(dir);
- final int optSegCount = sis.size();
-
- if (segCount < 7)
- assertEquals(segCount, optSegCount);
- else
- assertEquals(7, optSegCount);
- }
- writer.close();
- dir.close();
- }
-
- /**
- * Make sure optimize doesn't use any more than 1X
- * starting index size as its temporary free space
- * required.
- */
- public void testOptimizeTempSpaceUsage() throws IOException {
-
- MockDirectoryWrapper dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10).setMergePolicy(newLogMergePolicy()));
- if (VERBOSE) {
- System.out.println("TEST: config1=" + writer.getConfig());
- }
-
- for(int j=0;j<500;j++) {
- addDocWithIndex(writer, j);
- }
- final int termIndexInterval = writer.getConfig().getTermIndexInterval();
- // force one extra segment w/ different doc store so
- // we see the doc stores get merged
- writer.commit();
- addDocWithIndex(writer, 500);
- writer.close();
-
- if (VERBOSE) {
- System.out.println("TEST: start disk usage");
- }
- long startDiskUsage = 0;
- String[] files = dir.listAll();
- for(int i=0;i<files.length;i++) {
- startDiskUsage += dir.fileLength(files[i]);
- if (VERBOSE) {
- System.out.println(files[i] + ": " + dir.fileLength(files[i]));
- }
- }
-
- dir.resetMaxUsedSizeInBytes();
- dir.setTrackDiskUsage(true);
-
- // Import to use same term index interval else a
- // smaller one here could increase the disk usage and
- // cause a false failure:
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setTermIndexInterval(termIndexInterval).setMergePolicy(newLogMergePolicy()));
- writer.setInfoStream(VERBOSE ? System.out : null);
- writer.optimize();
- writer.close();
- long maxDiskUsage = dir.getMaxUsedSizeInBytes();
- assertTrue("optimize used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4*startDiskUsage) + " (= 4X starting usage)",
- maxDiskUsage <= 4*startDiskUsage);
- dir.close();
- }
-
static String arrayToString(String[] l) {
String s = "";
for(int i=0;i<l.length;i++) {
@@ -361,277 +226,7 @@ public class TestIndexWriter extends Luc
dir.close();
}
- /*
- * Simple test for "commit on close": open writer then
- * add a bunch of docs, making sure reader does not see
- * these docs until writer is closed.
- */
- public void testCommitOnClose() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- for (int i = 0; i < 14; i++) {
- addDoc(writer);
- }
- writer.close();
-
- Term searchTerm = new Term("content", "aaa");
- IndexSearcher searcher = new IndexSearcher(dir, false);
- ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("first number of hits", 14, hits.length);
- searcher.close();
-
- IndexReader reader = IndexReader.open(dir, true);
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- for(int i=0;i<3;i++) {
- for(int j=0;j<11;j++) {
- addDoc(writer);
- }
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
- searcher.close();
- assertTrue("reader should have still been current", reader.isCurrent());
- }
-
- // Now, close the writer:
- writer.close();
- assertFalse("reader should not be current now", reader.isCurrent());
-
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("reader did not see changes after writer was closed", 47, hits.length);
- searcher.close();
- reader.close();
- dir.close();
- }
-
- /*
- * Simple test for "commit on close": open writer, then
- * add a bunch of docs, making sure reader does not see
- * them until writer has closed. Then instead of
- * closing the writer, call abort and verify reader sees
- * nothing was added. Then verify we can open the index
- * and add docs to it.
- */
- public void testCommitOnCloseAbort() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10));
- for (int i = 0; i < 14; i++) {
- addDoc(writer);
- }
- writer.close();
-
- Term searchTerm = new Term("content", "aaa");
- IndexSearcher searcher = new IndexSearcher(dir, false);
- ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("first number of hits", 14, hits.length);
- searcher.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
- for(int j=0;j<17;j++) {
- addDoc(writer);
- }
- // Delete all docs:
- writer.deleteDocuments(searchTerm);
-
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
- searcher.close();
-
- // Now, close the writer:
- writer.rollback();
-
- assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
-
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("saw changes after writer.abort", 14, hits.length);
- searcher.close();
-
- // Now make sure we can re-open the index, add docs,
- // and all is good:
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
-
- // On abort, writer in fact may write to the same
- // segments_N file:
- dir.setPreventDoubleWrite(false);
-
- for(int i=0;i<12;i++) {
- for(int j=0;j<17;j++) {
- addDoc(writer);
- }
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
- searcher.close();
- }
-
- writer.close();
- searcher = new IndexSearcher(dir, false);
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- assertEquals("didn't see changes after close", 218, hits.length);
- searcher.close();
-
- dir.close();
- }
-
- /*
- * Verify that a writer with "commit on close" indeed
- * cleans up the temp segments created after opening
- * that are not referenced by the starting segments
- * file. We check this by using MockDirectoryWrapper to
- * measure max temp disk space used.
- */
- public void testCommitOnCloseDiskUsage() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- Analyzer analyzer;
- if (random.nextBoolean()) {
- // no payloads
- analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
- }
- };
- } else {
- // fixed length payloads
- final int length = random.nextInt(200);
- analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockFixedLengthPayloadFilter(random,
- new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
- length);
- }
- };
- }
-
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).
- setMaxBufferedDocs(10).
- setReaderPooling(false).
- setMergePolicy(newLogMergePolicy(10))
- );
- for(int j=0;j<30;j++) {
- addDocWithIndex(writer, j);
- }
- writer.close();
- dir.resetMaxUsedSizeInBytes();
-
- dir.setTrackDiskUsage(true);
- long startDiskUsage = dir.getMaxUsedSizeInBytes();
- writer = new IndexWriter(
- dir,
- newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
- .setOpenMode(OpenMode.APPEND).
- setMaxBufferedDocs(10).
- setMergeScheduler(new SerialMergeScheduler()).
- setReaderPooling(false).
- setMergePolicy(newLogMergePolicy(10))
-
- );
- for(int j=0;j<1470;j++) {
- addDocWithIndex(writer, j);
- }
- long midDiskUsage = dir.getMaxUsedSizeInBytes();
- dir.resetMaxUsedSizeInBytes();
- writer.optimize();
- writer.close();
-
- IndexReader.open(dir, true).close();
-
- long endDiskUsage = dir.getMaxUsedSizeInBytes();
-
- // Ending index is 50X as large as starting index; due
- // to 3X disk usage normally we allow 150X max
- // transient usage. If something is wrong w/ deleter
- // and it doesn't delete intermediate segments then it
- // will exceed this 150X:
- // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
- assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
- midDiskUsage < 150*startDiskUsage);
- assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
- endDiskUsage < 150*startDiskUsage);
- dir.close();
- }
-
-
- /*
- * Verify that calling optimize when writer is open for
- * "commit on close" works correctly both for rollback()
- * and close().
- */
- public void testCommitOnCloseOptimize() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- // Must disable throwing exc on double-write: this
- // test uses IW.rollback which easily results in
- // writing to same file more than once
- dir.setPreventDoubleWrite(false);
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(10).
- setMergePolicy(newLogMergePolicy(10))
- );
- for(int j=0;j<17;j++) {
- addDocWithIndex(writer, j);
- }
- writer.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
- writer.optimize();
-
- if (VERBOSE) {
- writer.setInfoStream(System.out);
- }
-
- // Open a reader before closing (commiting) the writer:
- IndexReader reader = IndexReader.open(dir, true);
-
- // Reader should see index as unoptimized at this
- // point:
- assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
- reader.close();
-
- // Abort the writer:
- writer.rollback();
- assertNoUnreferencedFiles(dir, "aborted writer after optimize");
-
- // Open a reader after aborting writer:
- reader = IndexReader.open(dir, true);
-
- // Reader should still see index as unoptimized:
- assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
- reader.close();
-
- if (VERBOSE) {
- System.out.println("TEST: do real optimize");
- }
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
- if (VERBOSE) {
- writer.setInfoStream(System.out);
- }
- writer.optimize();
- writer.close();
-
- if (VERBOSE) {
- System.out.println("TEST: writer closed");
- }
- assertNoUnreferencedFiles(dir, "aborted writer after optimize");
-
- // Open a reader after aborting writer:
- reader = IndexReader.open(dir, true);
- // Reader should still see index as unoptimized:
- assertTrue("Reader incorrectly sees that the index is unoptimized", reader.isOptimized());
- reader.close();
- dir.close();
- }
public void testIndexNoDocuments() throws IOException {
MockDirectoryWrapper dir = newDirectory();
@@ -1027,57 +622,13 @@ public class TestIndexWriter extends Luc
}
writer.addDocument(new Document());
writer.close();
- _TestUtil.checkIndex(dir);
IndexReader reader = IndexReader.open(dir, true);
assertEquals(2, reader.numDocs());
reader.close();
dir.close();
}
- // Test calling optimize(false) whereby optimize is kicked
- // off but we don't wait for it to finish (but
- // writer.close()) does wait
- public void testBackgroundOptimize() throws IOException {
-
- Directory dir = newDirectory();
- for(int pass=0;pass<2;pass++) {
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setOpenMode(OpenMode.CREATE).
- setMaxBufferedDocs(2).
- setMergePolicy(newLogMergePolicy(51))
- );
- Document doc = new Document();
- doc.add(newField("field", "aaa", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
- for(int i=0;i<100;i++)
- writer.addDocument(doc);
- writer.optimize(false);
-
- if (0 == pass) {
- writer.close();
- IndexReader reader = IndexReader.open(dir, true);
- assertTrue(reader.isOptimized());
- reader.close();
- } else {
- // Get another segment to flush so we can verify it is
- // NOT included in the optimization
- writer.addDocument(doc);
- writer.addDocument(doc);
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertTrue(!reader.isOptimized());
- reader.close();
-
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- assertEquals(2, infos.size());
- }
- }
- dir.close();
- }
/**
* Test that no NullPointerException will be raised,
@@ -1121,51 +672,6 @@ public class TestIndexWriter extends Luc
}
}
- // Just intercepts all merges & verifies that we are never
- // merging a segment with >= 20 (maxMergeDocs) docs
- private class MyMergeScheduler extends MergeScheduler {
- @Override
- synchronized public void merge(IndexWriter writer)
- throws CorruptIndexException, IOException {
-
- while(true) {
- MergePolicy.OneMerge merge = writer.getNextMerge();
- if (merge == null) {
- break;
- }
- for(int i=0;i<merge.segments.size();i++) {
- assert merge.segments.get(i).docCount < 20;
- }
- writer.merge(merge);
- }
- }
-
- @Override
- public void close() {}
- }
-
- // LUCENE-1013
- public void testSetMaxMergeDocs() throws IOException {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMergeScheduler(new MyMergeScheduler()).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setMaxMergeDocs(20);
- lmp.setMergeFactor(2);
- IndexWriter iw = new IndexWriter(dir, conf);
- iw.setInfoStream(VERBOSE ? System.out : null);
- Document document = new Document();
- document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.YES));
- for(int i=0;i<177;i++)
- iw.addDocument(document);
- iw.close();
- dir.close();
- }
-
-
-
public void testVariableSchema() throws Exception {
Directory dir = newDirectory();
int delID = 0;
@@ -1339,184 +845,7 @@ public class TestIndexWriter extends Luc
dir.close();
}
- // LUCENE-1044: test writer.commit() when ac=false
- public void testForceCommit() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setMergePolicy(newLogMergePolicy(5))
- );
- writer.commit();
-
- for (int i = 0; i < 23; i++)
- addDoc(writer);
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
- writer.commit();
- IndexReader reader2 = reader.reopen();
- assertEquals(0, reader.numDocs());
- assertEquals(23, reader2.numDocs());
- reader.close();
-
- for (int i = 0; i < 17; i++)
- addDoc(writer);
- assertEquals(23, reader2.numDocs());
- reader2.close();
- reader = IndexReader.open(dir, true);
- assertEquals(23, reader.numDocs());
- reader.close();
- writer.commit();
- reader = IndexReader.open(dir, true);
- assertEquals(40, reader.numDocs());
- reader.close();
- writer.close();
- dir.close();
- }
-
- // LUCENE-325: test expungeDeletes, when 2 singular merges
- // are required
- public void testExpungeDeletes() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH));
- writer.setInfoStream(VERBOSE ? System.out : null);
- Document document = new Document();
-
- document = new Document();
- Field storedField = newField("stored", "stored", Field.Store.YES,
- Field.Index.NO);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Field.Store.NO, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS);
- document.add(termVectorField);
- for(int i=0;i<10;i++)
- writer.addDocument(document);
- writer.close();
-
- IndexReader ir = IndexReader.open(dir, false);
- assertEquals(10, ir.maxDoc());
- assertEquals(10, ir.numDocs());
- ir.deleteDocument(0);
- ir.deleteDocument(7);
- assertEquals(8, ir.numDocs());
- ir.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
- assertEquals(8, writer.numDocs());
- assertEquals(10, writer.maxDoc());
- writer.expungeDeletes();
- assertEquals(8, writer.numDocs());
- writer.close();
- ir = IndexReader.open(dir, true);
- assertEquals(8, ir.maxDoc());
- assertEquals(8, ir.numDocs());
- ir.close();
- dir.close();
- }
-
- // LUCENE-325: test expungeDeletes, when many adjacent merges are required
- public void testExpungeDeletes2() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
- setMergePolicy(newLogMergePolicy(50))
- );
-
- Document document = new Document();
-
- document = new Document();
- Field storedField = newField("stored", "stored", Store.YES,
- Index.NO);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Store.NO, Index.NOT_ANALYZED,
- TermVector.WITH_POSITIONS_OFFSETS);
- document.add(termVectorField);
- for(int i=0;i<98;i++)
- writer.addDocument(document);
- writer.close();
-
- IndexReader ir = IndexReader.open(dir, false);
- assertEquals(98, ir.maxDoc());
- assertEquals(98, ir.numDocs());
- for(int i=0;i<98;i+=2)
- ir.deleteDocument(i);
- assertEquals(49, ir.numDocs());
- ir.close();
-
- writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMergePolicy(newLogMergePolicy(3))
- );
- assertEquals(49, writer.numDocs());
- writer.expungeDeletes();
- writer.close();
- ir = IndexReader.open(dir, true);
- assertEquals(49, ir.maxDoc());
- assertEquals(49, ir.numDocs());
- ir.close();
- dir.close();
- }
-
- // LUCENE-325: test expungeDeletes without waiting, when
- // many adjacent merges are required
- public void testExpungeDeletes3() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
- setMergePolicy(newLogMergePolicy(50))
- );
-
- Document document = new Document();
-
- document = new Document();
- Field storedField = newField("stored", "stored", Field.Store.YES,
- Field.Index.NO);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Field.Store.NO, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS);
- document.add(termVectorField);
- for(int i=0;i<98;i++)
- writer.addDocument(document);
- writer.close();
-
- IndexReader ir = IndexReader.open(dir, false);
- assertEquals(98, ir.maxDoc());
- assertEquals(98, ir.numDocs());
- for(int i=0;i<98;i+=2)
- ir.deleteDocument(i);
- assertEquals(49, ir.numDocs());
- ir.close();
-
- writer = new IndexWriter(
- dir,
- newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMergePolicy(newLogMergePolicy(3))
- );
- writer.expungeDeletes(false);
- writer.close();
- ir = IndexReader.open(dir, true);
- assertEquals(49, ir.maxDoc());
- assertEquals(49, ir.numDocs());
- ir.close();
- dir.close();
- }
// LUCENE-1179
public void testEmptyFieldName() throws IOException {
@@ -1554,193 +883,28 @@ public class TestIndexWriter extends Luc
// LUCENE-1222
public void testDoBeforeAfterFlush() throws IOException {
- Directory dir = newDirectory();
- MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- doc.add(newField("field", "a field", Field.Store.YES,
- Field.Index.ANALYZED));
- w.addDocument(doc);
- w.commit();
- assertTrue(w.beforeWasCalled);
- assertTrue(w.afterWasCalled);
- w.beforeWasCalled = false;
- w.afterWasCalled = false;
- w.deleteDocuments(new Term("field", "field"));
- w.commit();
- assertTrue(w.beforeWasCalled);
- assertTrue(w.afterWasCalled);
- w.close();
-
- IndexReader ir = IndexReader.open(dir, true);
- assertEquals(0, ir.numDocs());
- ir.close();
-
- dir.close();
- }
-
-
-
- final String[] utf8Data = new String[] {
- // unpaired low surrogate
- "ab\udc17cd", "ab\ufffdcd",
- "\udc17abcd", "\ufffdabcd",
- "\udc17", "\ufffd",
- "ab\udc17\udc17cd", "ab\ufffd\ufffdcd",
- "\udc17\udc17abcd", "\ufffd\ufffdabcd",
- "\udc17\udc17", "\ufffd\ufffd",
-
- // unpaired high surrogate
- "ab\ud917cd", "ab\ufffdcd",
- "\ud917abcd", "\ufffdabcd",
- "\ud917", "\ufffd",
- "ab\ud917\ud917cd", "ab\ufffd\ufffdcd",
- "\ud917\ud917abcd", "\ufffd\ufffdabcd",
- "\ud917\ud917", "\ufffd\ufffd",
-
- // backwards surrogates
- "ab\udc17\ud917cd", "ab\ufffd\ufffdcd",
- "\udc17\ud917abcd", "\ufffd\ufffdabcd",
- "\udc17\ud917", "\ufffd\ufffd",
- "ab\udc17\ud917\udc17\ud917cd", "ab\ufffd\ud917\udc17\ufffdcd",
- "\udc17\ud917\udc17\ud917abcd", "\ufffd\ud917\udc17\ufffdabcd",
- "\udc17\ud917\udc17\ud917", "\ufffd\ud917\udc17\ufffd"
- };
-
- // LUCENE-510
- public void testInvalidUTF16() throws Throwable {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new StringSplitAnalyzer()));
- Document doc = new Document();
-
- final int count = utf8Data.length/2;
- for(int i=0;i<count;i++)
- doc.add(newField("f" + i, utf8Data[2*i], Field.Store.YES, Field.Index.ANALYZED));
- w.addDocument(doc);
- w.close();
-
- IndexReader ir = IndexReader.open(dir, true);
- Document doc2 = ir.document(0);
- for(int i=0;i<count;i++) {
- assertEquals("field " + i + " was not indexed correctly", 1, ir.docFreq(new Term("f"+i, utf8Data[2*i+1])));
- assertEquals("field " + i + " is incorrect", utf8Data[2*i+1], doc2.getField("f"+i).stringValue());
- }
- ir.close();
- dir.close();
- }
-
- // LUCENE-510
- public void testAllUnicodeChars() throws Throwable {
-
- BytesRef utf8 = new BytesRef(10);
- CharsRef utf16 = new CharsRef(10);
- char[] chars = new char[2];
- for(int ch=0;ch<0x0010FFFF;ch++) {
-
- if (ch == 0xd800)
- // Skip invalid code points
- ch = 0xe000;
-
- int len = 0;
- if (ch <= 0xffff) {
- chars[len++] = (char) ch;
- } else {
- chars[len++] = (char) (((ch-0x0010000) >> 10) + UnicodeUtil.UNI_SUR_HIGH_START);
- chars[len++] = (char) (((ch-0x0010000) & 0x3FFL) + UnicodeUtil.UNI_SUR_LOW_START);
- }
-
- UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8);
-
- String s1 = new String(chars, 0, len);
- String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8");
- assertEquals("codepoint " + ch, s1, s2);
-
- UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
- assertEquals("codepoint " + ch, s1, new String(utf16.chars, 0, utf16.length));
-
- byte[] b = s1.getBytes("UTF-8");
- assertEquals(utf8.length, b.length);
- for(int j=0;j<utf8.length;j++)
- assertEquals(utf8.bytes[j], b[j]);
- }
- }
-
- private int nextInt(int lim) {
- return random.nextInt(lim);
- }
-
- private int nextInt(int start, int end) {
- return start + nextInt(end-start);
- }
-
- private boolean fillUnicode(char[] buffer, char[] expected, int offset, int count) {
- final int len = offset + count;
- boolean hasIllegal = false;
-
- if (offset > 0 && buffer[offset] >= 0xdc00 && buffer[offset] < 0xe000)
- // Don't start in the middle of a valid surrogate pair
- offset--;
-
- for(int i=offset;i<len;i++) {
- int t = nextInt(6);
- if (0 == t && i < len-1) {
- // Make a surrogate pair
- // High surrogate
- expected[i] = buffer[i++] = (char) nextInt(0xd800, 0xdc00);
- // Low surrogate
- expected[i] = buffer[i] = (char) nextInt(0xdc00, 0xe000);
- } else if (t <= 1)
- expected[i] = buffer[i] = (char) nextInt(0x80);
- else if (2 == t)
- expected[i] = buffer[i] = (char) nextInt(0x80, 0x800);
- else if (3 == t)
- expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
- else if (4 == t)
- expected[i] = buffer[i] = (char) nextInt(0xe000, 0xffff);
- else if (5 == t && i < len-1) {
- // Illegal unpaired surrogate
- if (nextInt(10) == 7) {
- if (random.nextBoolean())
- buffer[i] = (char) nextInt(0xd800, 0xdc00);
- else
- buffer[i] = (char) nextInt(0xdc00, 0xe000);
- expected[i++] = 0xfffd;
- expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
- hasIllegal = true;
- } else
- expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
- } else {
- expected[i] = buffer[i] = ' ';
- }
- }
-
- return hasIllegal;
- }
-
- // LUCENE-510
- public void testRandomUnicodeStrings() throws Throwable {
- char[] buffer = new char[20];
- char[] expected = new char[20];
-
- BytesRef utf8 = new BytesRef(20);
- CharsRef utf16 = new CharsRef(20);
-
- int num = 100000 * RANDOM_MULTIPLIER;
- for (int iter = 0; iter < num; iter++) {
- boolean hasIllegal = fillUnicode(buffer, expected, 0, 20);
+ Directory dir = newDirectory();
+ MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document doc = new Document();
+ doc.add(newField("field", "a field", Field.Store.YES,
+ Field.Index.ANALYZED));
+ w.addDocument(doc);
+ w.commit();
+ assertTrue(w.beforeWasCalled);
+ assertTrue(w.afterWasCalled);
+ w.beforeWasCalled = false;
+ w.afterWasCalled = false;
+ w.deleteDocuments(new Term("field", "field"));
+ w.commit();
+ assertTrue(w.beforeWasCalled);
+ assertTrue(w.afterWasCalled);
+ w.close();
- UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
- if (!hasIllegal) {
- byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
- assertEquals(b.length, utf8.length);
- for(int i=0;i<b.length;i++)
- assertEquals(b[i], utf8.bytes[i]);
- }
+ IndexReader ir = IndexReader.open(dir, true);
+ assertEquals(0, ir.numDocs());
+ ir.close();
- UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
- assertEquals(utf16.length, 20);
- for(int i=0;i<20;i++)
- assertEquals(expected[i], utf16.chars[i]);
- }
+ dir.close();
}
// LUCENE-1255
@@ -1792,137 +956,11 @@ public class TestIndexWriter extends Luc
assertEquals(0, tps.nextPosition());
w.close();
- _TestUtil.checkIndex(dir);
s.close();
dir.close();
}
- // LUCENE-1274: test writer.prepareCommit()
- public void testPrepareCommit() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setMergePolicy(newLogMergePolicy(5))
- );
- writer.commit();
-
- for (int i = 0; i < 23; i++)
- addDoc(writer);
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
-
- writer.prepareCommit();
-
- IndexReader reader2 = IndexReader.open(dir, true);
- assertEquals(0, reader2.numDocs());
-
- writer.commit();
-
- IndexReader reader3 = reader.reopen();
- assertEquals(0, reader.numDocs());
- assertEquals(0, reader2.numDocs());
- assertEquals(23, reader3.numDocs());
- reader.close();
- reader2.close();
-
- for (int i = 0; i < 17; i++)
- addDoc(writer);
-
- assertEquals(23, reader3.numDocs());
- reader3.close();
- reader = IndexReader.open(dir, true);
- assertEquals(23, reader.numDocs());
- reader.close();
-
- writer.prepareCommit();
-
- reader = IndexReader.open(dir, true);
- assertEquals(23, reader.numDocs());
- reader.close();
-
- writer.commit();
- reader = IndexReader.open(dir, true);
- assertEquals(40, reader.numDocs());
- reader.close();
- writer.close();
- dir.close();
- }
-
- // LUCENE-1274: test writer.prepareCommit()
- public void testPrepareCommitRollback() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
- dir.setPreventDoubleWrite(false);
-
- IndexWriter writer = new IndexWriter(
- dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDocs(2).
- setMergePolicy(newLogMergePolicy(5))
- );
- writer.commit();
-
- for (int i = 0; i < 23; i++)
- addDoc(writer);
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
-
- writer.prepareCommit();
-
- IndexReader reader2 = IndexReader.open(dir, true);
- assertEquals(0, reader2.numDocs());
-
- writer.rollback();
-
- IndexReader reader3 = reader.reopen();
- assertEquals(0, reader.numDocs());
- assertEquals(0, reader2.numDocs());
- assertEquals(0, reader3.numDocs());
- reader.close();
- reader2.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- for (int i = 0; i < 17; i++)
- addDoc(writer);
-
- assertEquals(0, reader3.numDocs());
- reader3.close();
- reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
- reader.close();
-
- writer.prepareCommit();
-
- reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
- reader.close();
-
- writer.commit();
- reader = IndexReader.open(dir, true);
- assertEquals(17, reader.numDocs());
- reader.close();
- writer.close();
- dir.close();
- }
-
- // LUCENE-1274
- public void testPrepareCommitNoChanges() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- writer.prepareCommit();
- writer.commit();
- writer.close();
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
- reader.close();
- dir.close();
- }
// LUCENE-1219
public void testBinaryFieldOffsetLength() throws IOException {
@@ -1954,45 +992,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- // LUCENE-1382
- public void testCommitUserData() throws IOException {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
- for(int j=0;j<17;j++)
- addDoc(w);
- w.close();
-
- assertEquals(0, IndexReader.getCommitUserData(dir).size());
-
- IndexReader r = IndexReader.open(dir, true);
- // commit(Map) never called for this index
- assertEquals(0, r.getCommitUserData().size());
- r.close();
-
- w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
- for(int j=0;j<17;j++)
- addDoc(w);
- Map<String,String> data = new HashMap<String,String>();
- data.put("label", "test1");
- w.commit(data);
- w.close();
-
- assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
-
- r = IndexReader.open(dir, true);
- assertEquals("test1", r.getCommitUserData().get("label"));
- r.close();
-
- w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- w.optimize();
- w.close();
-
- assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
-
- dir.close();
- }
-
-
// LUCENE-2529
public void testPositionIncrementGapEmptyField() throws Exception {
Directory dir = newDirectory();
@@ -2299,24 +1298,6 @@ public class TestIndexWriter extends Luc
d.close();
}
- public void testEmbeddedFFFF() throws Throwable {
-
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- doc.add(newField("field", "a a\uffffb", Field.Store.NO, Field.Index.ANALYZED));
- w.addDocument(doc);
- doc = new Document();
- doc.add(newField("field", "a", Field.Store.NO, Field.Index.ANALYZED));
- w.addDocument(doc);
- IndexReader r = w.getReader();
- assertEquals(1, r.docFreq(new Term("field", "a\uffffb")));
- r.close();
- w.close();
- _TestUtil.checkIndex(d);
- d.close();
- }
-
public void testNoDocsIndex() throws Throwable {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
@@ -2326,185 +1307,6 @@ public class TestIndexWriter extends Luc
writer.addDocument(new Document());
writer.close();
- _TestUtil.checkIndex(dir);
- dir.close();
- }
-
- // LUCENE-2095: make sure with multiple threads commit
- // doesn't return until all changes are in fact in the
- // index
- public void testCommitThreadSafety() throws Throwable {
- final int NUM_THREADS = 5;
- final double RUN_SEC = 0.5;
- final Directory dir = newDirectory();
- final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
- _TestUtil.reduceOpenFiles(w.w);
- w.commit();
- final AtomicBoolean failed = new AtomicBoolean();
- Thread[] threads = new Thread[NUM_THREADS];
- final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000));
- for(int i=0;i<NUM_THREADS;i++) {
- final int finalI = i;
- threads[i] = new Thread() {
- @Override
- public void run() {
- try {
- final Document doc = new Document();
- IndexReader r = IndexReader.open(dir);
- Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
- doc.add(f);
- int count = 0;
- do {
- if (failed.get()) break;
- for(int j=0;j<10;j++) {
- final String s = finalI + "_" + String.valueOf(count++);
- f.setValue(s);
- w.addDocument(doc);
- w.commit();
- IndexReader r2 = r.reopen();
- assertTrue(r2 != r);
- r.close();
- r = r2;
- assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
- }
- } while(System.currentTimeMillis() < endTime);
- r.close();
- } catch (Throwable t) {
- failed.set(true);
- throw new RuntimeException(t);
- }
- }
- };
- threads[i].start();
- }
- for(int i=0;i<NUM_THREADS;i++) {
- threads[i].join();
- }
- assertFalse(failed.get());
- w.close();
- dir.close();
- }
-
- // both start & end are inclusive
- private final int getInt(Random r, int start, int end) {
- return start + r.nextInt(1+end-start);
- }
-
- private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
- TermsEnum terms = MultiFields.getFields(r).terms("f").iterator();
-
- BytesRef last = new BytesRef();
-
- Set<String> seenTerms = new HashSet<String>();
-
- while(true) {
- final BytesRef term = terms.next();
- if (term == null) {
- break;
- }
-
- assertTrue(last.compareTo(term) < 0);
- last.copy(term);
-
- final String s = term.utf8ToString();
- assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s));
- seenTerms.add(s);
- }
-
- if (isTop) {
- assertTrue(allTerms.equals(seenTerms));
- }
-
- // Test seeking:
- Iterator<String> it = seenTerms.iterator();
- while(it.hasNext()) {
- BytesRef tr = new BytesRef(it.next());
- assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()),
- TermsEnum.SeekStatus.FOUND,
- terms.seek(tr));
- }
- }
-
- private final String asUnicodeChar(char c) {
- return "U+" + Integer.toHexString(c);
- }
-
- private final String termDesc(String s) {
- final String s0;
- assertTrue(s.length() <= 2);
- if (s.length() == 1) {
- s0 = asUnicodeChar(s.charAt(0));
- } else {
- s0 = asUnicodeChar(s.charAt(0)) + "," + asUnicodeChar(s.charAt(1));
- }
- return s0;
- }
-
- // Make sure terms, including ones with surrogate pairs,
- // sort in codepoint sort order by default
- public void testTermUTF16SortOrder() throws Throwable {
- Random rnd = random;
- Directory dir = newDirectory();
- RandomIndexWriter writer = new RandomIndexWriter(rnd, dir);
- Document d = new Document();
- // Single segment
- Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
- d.add(f);
- char[] chars = new char[2];
- final Set<String> allTerms = new HashSet<String>();
-
- int num = 200 * RANDOM_MULTIPLIER;
- for (int i = 0; i < num; i++) {
-
- final String s;
- if (rnd.nextBoolean()) {
- // Single char
- if (rnd.nextBoolean()) {
- // Above surrogates
- chars[0] = (char) getInt(rnd, 1+UnicodeUtil.UNI_SUR_LOW_END, 0xffff);
- } else {
- // Below surrogates
- chars[0] = (char) getInt(rnd, 0, UnicodeUtil.UNI_SUR_HIGH_START-1);
- }
- s = new String(chars, 0, 1);
- } else {
- // Surrogate pair
- chars[0] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_HIGH_START, UnicodeUtil.UNI_SUR_HIGH_END);
- assertTrue(((int) chars[0]) >= UnicodeUtil.UNI_SUR_HIGH_START && ((int) chars[0]) <= UnicodeUtil.UNI_SUR_HIGH_END);
- chars[1] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_LOW_START, UnicodeUtil.UNI_SUR_LOW_END);
- s = new String(chars, 0, 2);
- }
- allTerms.add(s);
- f.setValue(s);
-
- writer.addDocument(d);
-
- if ((1+i) % 42 == 0) {
- writer.commit();
- }
- }
-
- IndexReader r = writer.getReader();
-
- // Test each sub-segment
- final IndexReader[] subs = r.getSequentialSubReaders();
- for(int i=0;i<subs.length;i++) {
- checkTermsOrder(subs[i], allTerms, false);
- }
- checkTermsOrder(r, allTerms, true);
-
- // Test multi segment
- r.close();
-
- writer.optimize();
-
- // Test optimized single segment
- r = writer.getReader();
- checkTermsOrder(r, allTerms, true);
- r.close();
-
- writer.close();
dir.close();
}
@@ -2573,6 +1375,11 @@ public class TestIndexWriter extends Luc
IndexReader r2 = r.reopen();
assertTrue(r != r2);
files = Arrays.asList(dir.listAll());
+
+ // NOTE: here we rely on "Windows" behavior, ie, even
+ // though IW wanted to delete _0.cfs since it was
+ // optimized away, because we have a reader open
+ // against this file, it should still be here:
assertTrue(files.contains("_0.cfs"));
// optimize created this
//assertTrue(files.contains("_2.cfs"));
@@ -2638,67 +1445,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- public void testIndexingThenDeleting() throws Exception {
- final Random r = random;
- Directory dir = newDirectory();
- // note this test explicitly disables payloads
- final Analyzer analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
- }
- };
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
- w.setInfoStream(VERBOSE ? System.out : null);
- Document doc = new Document();
- doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
- int num = TEST_NIGHTLY ? 6 * RANDOM_MULTIPLIER : 3 * RANDOM_MULTIPLIER;
- for (int iter = 0; iter < num; iter++) {
- int count = 0;
-
- final boolean doIndexing = r.nextBoolean();
- if (VERBOSE) {
- System.out.println("TEST: iter doIndexing=" + doIndexing);
- }
- if (doIndexing) {
- // Add docs until a flush is triggered
- final int startFlushCount = w.getFlushCount();
- while(w.getFlushCount() == startFlushCount) {
- w.addDocument(doc);
- count++;
- }
- } else {
- // Delete docs until a flush is triggered
- final int startFlushCount = w.getFlushCount();
- while(w.getFlushCount() == startFlushCount) {
- w.deleteDocuments(new Term("foo", ""+count));
- count++;
- }
- }
- assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 3000);
- }
- w.close();
- dir.close();
- }
-
- public void testNoCommits() throws Exception {
- // Tests that if we don't call commit(), the directory has 0 commits. This has
- // changed since LUCENE-2386, where before IW would always commit on a fresh
- // new index.
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- try {
- IndexReader.listCommits(dir);
- fail("listCommits should have thrown an exception over empty index");
- } catch (IndexNotFoundException e) {
- // that's expected !
- }
- // No changes still should generate a commit, because it's a new index.
- writer.close();
- assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size());
- dir.close();
- }
-
public void testEmptyFSDirWithNoLock() throws Exception {
// Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF),
// then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed
@@ -2775,82 +1521,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- public void testFutureCommit() throws Exception {
- Directory dir = newDirectory();
-
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
- Document doc = new Document();
- w.addDocument(doc);
-
- // commit to "first"
- Map<String,String> commitData = new HashMap<String,String>();
- commitData.put("tag", "first");
- w.commit(commitData);
-
- // commit to "second"
- w.addDocument(doc);
- commitData.put("tag", "second");
- w.commit(commitData);
- w.close();
-
- // open "first" with IndexWriter
- IndexCommit commit = null;
- for(IndexCommit c : IndexReader.listCommits(dir)) {
- if (c.getUserData().get("tag").equals("first")) {
- commit = c;
- break;
- }
- }
-
- assertNotNull(commit);
-
- w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));
-
- assertEquals(1, w.numDocs());
-
- // commit IndexWriter to "third"
- w.addDocument(doc);
- commitData.put("tag", "third");
- w.commit(commitData);
- w.close();
-
- // make sure "second" commit is still there
- commit = null;
- for(IndexCommit c : IndexReader.listCommits(dir)) {
- if (c.getUserData().get("tag").equals("second")) {
- commit = c;
- break;
- }
- }
-
- assertNotNull(commit);
-
- IndexReader r = IndexReader.open(commit, true);
- assertEquals(2, r.numDocs());
- r.close();
-
- // open "second", w/ writeable IndexReader & commit
- r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);
- assertEquals(2, r.numDocs());
- r.deleteDocument(0);
- r.deleteDocument(1);
- commitData.put("tag", "fourth");
- r.commit(commitData);
- r.close();
-
- // make sure "third" commit is still there
- commit = null;
- for(IndexCommit c : IndexReader.listCommits(dir)) {
- if (c.getUserData().get("tag").equals("third")) {
- commit = c;
- break;
- }
- }
- assertNotNull(commit);
-
- dir.close();
- }
-
public void testRandomStoredFields() throws IOException {
Directory dir = newDirectory();
Random rand = random;
@@ -2860,7 +1530,7 @@ public class TestIndexWriter extends Luc
if (VERBOSE) {
w.w.setInfoStream(System.out);
}
- final int docCount = 200*RANDOM_MULTIPLIER;
+ final int docCount = atLeast(200);
final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
final List<Integer> fieldIDs = new ArrayList<Integer>();
@@ -2925,7 +1595,8 @@ public class TestIndexWriter extends Luc
System.out.println("TEST: cycle x=" + x + " r=" + r);
}
- for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
+ int num = atLeast(1000);
+ for(int iter=0;iter<num;iter++) {
String testID = idsList[rand.nextInt(idsList.length)];
if (VERBOSE) {
System.out.println("TEST: test id=" + testID);
@@ -2980,39 +1651,7 @@ public class TestIndexWriter extends Luc
dir.close();
}
- public void testDeleteAllSlowly() throws Exception {
- final Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random, dir);
- final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER;
- final List<Integer> ids = new ArrayList<Integer>(NUM_DOCS);
- for(int id=0;id<NUM_DOCS;id++) {
- ids.add(id);
- }
- Collections.shuffle(ids, random);
- for(int id : ids) {
- Document doc = new Document();
- doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
- w.addDocument(doc);
- }
- Collections.shuffle(ids, random);
- int upto = 0;
- while(upto < ids.size()) {
- final int left = ids.size() - upto;
- final int inc = Math.min(left, _TestUtil.nextInt(random, 1, 20));
- final int limit = upto + inc;
- while(upto < limit) {
- w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
- }
- final IndexReader r = w.getReader();
- assertEquals(NUM_DOCS - upto, r.numDocs());
- r.close();
- }
-
- w.close();
- dir.close();
- }
-
- private static class StringSplitAnalyzer extends Analyzer {
+ static final class StringSplitAnalyzer extends Analyzer {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
return new StringSplitTokenizer(reader);
@@ -3130,4 +1769,18 @@ public class TestIndexWriter extends Luc
reader.close();
dir.close();
}
+
+ // LUCENE-3183
+ public void testEmptyFieldNameTIIOne() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ iwc.setTermIndexInterval(1);
+ iwc.setReaderTermsIndexDivisor(1);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(newField("", "a b c", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ writer.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Jun 24 23:07:10 2011
@@ -18,9 +18,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
@@ -860,4 +867,79 @@ public class TestIndexWriterDelete exten
modifier.close();
dir.close();
}
+
+ public void testDeleteAllSlowly() throws Exception {
+ final Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random, dir);
+ final int NUM_DOCS = atLeast(1000);
+ final List<Integer> ids = new ArrayList<Integer>(NUM_DOCS);
+ for(int id=0;id<NUM_DOCS;id++) {
+ ids.add(id);
+ }
+ Collections.shuffle(ids, random);
+ for(int id : ids) {
+ Document doc = new Document();
+ doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
+ w.addDocument(doc);
+ }
+ Collections.shuffle(ids, random);
+ int upto = 0;
+ while(upto < ids.size()) {
+ final int left = ids.size() - upto;
+ final int inc = Math.min(left, _TestUtil.nextInt(random, 1, 20));
+ final int limit = upto + inc;
+ while(upto < limit) {
+ w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
+ }
+ final IndexReader r = w.getReader();
+ assertEquals(NUM_DOCS - upto, r.numDocs());
+ r.close();
+ }
+
+ w.close();
+ dir.close();
+ }
+
+ public void testIndexingThenDeleting() throws Exception {
+ final Random r = random;
+ Directory dir = newDirectory();
+ // note this test explicitly disables payloads
+ final Analyzer analyzer = new Analyzer() {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ }
+ };
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
+ w.setInfoStream(VERBOSE ? System.out : null);
+ Document doc = new Document();
+ doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
+ int num = atLeast(3);
+ for (int iter = 0; iter < num; iter++) {
+ int count = 0;
+
+ final boolean doIndexing = r.nextBoolean();
+ if (VERBOSE) {
+ System.out.println("TEST: iter doIndexing=" + doIndexing);
+ }
+ if (doIndexing) {
+ // Add docs until a flush is triggered
+ final int startFlushCount = w.getFlushCount();
+ while(w.getFlushCount() == startFlushCount) {
+ w.addDocument(doc);
+ count++;
+ }
+ } else {
+ // Delete docs until a flush is triggered
+ final int startFlushCount = w.getFlushCount();
+ while(w.getFlushCount() == startFlushCount) {
+ w.deleteDocuments(new Term("foo", ""+count));
+ count++;
+ }
+ }
+ assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 2500);
+ }
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Jun 24 23:07:10 2011
@@ -236,7 +236,6 @@ public class TestIndexWriterExceptions e
assertEquals(count, count2);
r2.close();
- _TestUtil.checkIndex(dir);
dir.close();
}
@@ -287,7 +286,6 @@ public class TestIndexWriterExceptions e
assertEquals(count, count2);
r2.close();
- _TestUtil.checkIndex(dir);
dir.close();
}
@@ -349,7 +347,6 @@ public class TestIndexWriterExceptions e
// expected
}
w.close();
- _TestUtil.checkIndex(dir);
dir.close();
}
@@ -933,7 +930,8 @@ public class TestIndexWriterExceptions e
addDoc(w);
w.close();
- for(int i=0;i<200;i++) {
+ int iter = TEST_NIGHTLY ? 200 : 20;
+ for(int i=0;i<iter;i++) {
if (VERBOSE) {
System.out.println("TEST: iter " + i);
}
@@ -1225,7 +1223,8 @@ public class TestIndexWriterExceptions e
FailOnTermVectors[] failures = new FailOnTermVectors[] {
new FailOnTermVectors(FailOnTermVectors.AFTER_INIT_STAGE),
new FailOnTermVectors(FailOnTermVectors.INIT_STAGE), };
- for (int j = 0; j < 3 * RANDOM_MULTIPLIER; j++) {
+ int num = atLeast(3);
+ for (int j = 0; j < num; j++) {
for (FailOnTermVectors failure : failures) {
MockDirectoryWrapper dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -104,9 +104,12 @@ public class TestIndexWriterMergePolicy
dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setMaxBufferedDocs(10).
- setMergePolicy(newLogMergePolicy())
+ setMergePolicy(newLogMergePolicy()).
+ setMergeScheduler(new SerialMergeScheduler())
);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
for (int i = 0; i < 250; i++) {
addDoc(writer);
checkInvariants(writer);
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Fri Jun 24 23:07:10 2011
@@ -19,6 +19,9 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.util.LuceneTestCase;
@@ -113,4 +116,187 @@ public class TestIndexWriterMerging exte
}
writer.close();
}
+
+ // LUCENE-325: test expungeDeletes, when 2 singular merges
+ // are required
+ public void testExpungeDeletes() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMaxBufferedDocs(2).setRAMBufferSizeMB(
+ IndexWriterConfig.DISABLE_AUTO_FLUSH));
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ Document document = new Document();
+
+ document = new Document();
+ Field storedField = newField("stored", "stored", Field.Store.YES,
+ Field.Index.NO);
+ document.add(storedField);
+ Field termVectorField = newField("termVector", "termVector",
+ Field.Store.NO, Field.Index.NOT_ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS);
+ document.add(termVectorField);
+ for(int i=0;i<10;i++)
+ writer.addDocument(document);
+ writer.close();
+
+ IndexReader ir = IndexReader.open(dir, false);
+ assertEquals(10, ir.maxDoc());
+ assertEquals(10, ir.numDocs());
+ ir.deleteDocument(0);
+ ir.deleteDocument(7);
+ assertEquals(8, ir.numDocs());
+ ir.close();
+
+ writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+ assertEquals(8, writer.numDocs());
+ assertEquals(10, writer.maxDoc());
+ writer.expungeDeletes();
+ assertEquals(8, writer.numDocs());
+ writer.close();
+ ir = IndexReader.open(dir, true);
+ assertEquals(8, ir.maxDoc());
+ assertEquals(8, ir.numDocs());
+ ir.close();
+ dir.close();
+ }
+
+ // LUCENE-325: test expungeDeletes, when many adjacent merges are required
+ public void testExpungeDeletes2() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDocs(2).
+ setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
+ setMergePolicy(newLogMergePolicy(50))
+ );
+
+ Document document = new Document();
+
+ document = new Document();
+ Field storedField = newField("stored", "stored", Store.YES,
+ Index.NO);
+ document.add(storedField);
+ Field termVectorField = newField("termVector", "termVector",
+ Store.NO, Index.NOT_ANALYZED,
+ TermVector.WITH_POSITIONS_OFFSETS);
+ document.add(termVectorField);
+ for(int i=0;i<98;i++)
+ writer.addDocument(document);
+ writer.close();
+
+ IndexReader ir = IndexReader.open(dir, false);
+ assertEquals(98, ir.maxDoc());
+ assertEquals(98, ir.numDocs());
+ for(int i=0;i<98;i+=2)
+ ir.deleteDocument(i);
+ assertEquals(49, ir.numDocs());
+ ir.close();
+
+ writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMergePolicy(newLogMergePolicy(3))
+ );
+ assertEquals(49, writer.numDocs());
+ writer.expungeDeletes();
+ writer.close();
+ ir = IndexReader.open(dir, true);
+ assertEquals(49, ir.maxDoc());
+ assertEquals(49, ir.numDocs());
+ ir.close();
+ dir.close();
+ }
+
+ // LUCENE-325: test expungeDeletes without waiting, when
+ // many adjacent merges are required
+ public void testExpungeDeletes3() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMaxBufferedDocs(2).
+ setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
+ setMergePolicy(newLogMergePolicy(50))
+ );
+
+ Document document = new Document();
+
+ document = new Document();
+ Field storedField = newField("stored", "stored", Field.Store.YES,
+ Field.Index.NO);
+ document.add(storedField);
+ Field termVectorField = newField("termVector", "termVector",
+ Field.Store.NO, Field.Index.NOT_ANALYZED,
+ Field.TermVector.WITH_POSITIONS_OFFSETS);
+ document.add(termVectorField);
+ for(int i=0;i<98;i++)
+ writer.addDocument(document);
+ writer.close();
+
+ IndexReader ir = IndexReader.open(dir, false);
+ assertEquals(98, ir.maxDoc());
+ assertEquals(98, ir.numDocs());
+ for(int i=0;i<98;i+=2)
+ ir.deleteDocument(i);
+ assertEquals(49, ir.numDocs());
+ ir.close();
+
+ writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setMergePolicy(newLogMergePolicy(3))
+ );
+ writer.expungeDeletes(false);
+ writer.close();
+ ir = IndexReader.open(dir, true);
+ assertEquals(49, ir.maxDoc());
+ assertEquals(49, ir.numDocs());
+ ir.close();
+ dir.close();
+ }
+
+ // Just intercepts all merges & verifies that we are never
+ // merging a segment with >= 20 (maxMergeDocs) docs
+ private class MyMergeScheduler extends MergeScheduler {
+ @Override
+ synchronized public void merge(IndexWriter writer)
+ throws CorruptIndexException, IOException {
+
+ while(true) {
+ MergePolicy.OneMerge merge = writer.getNextMerge();
+ if (merge == null) {
+ break;
+ }
+ for(int i=0;i<merge.segments.size();i++) {
+ assert merge.segments.get(i).docCount < 20;
+ }
+ writer.merge(merge);
+ }
+ }
+
+ @Override
+ public void close() {}
+ }
+
+ // LUCENE-1013
+ public void testSetMaxMergeDocs() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setMergeScheduler(new MyMergeScheduler()).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
+ LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
+ lmp.setMaxMergeDocs(20);
+ lmp.setMergeFactor(2);
+ IndexWriter iw = new IndexWriter(dir, conf);
+ iw.setInfoStream(VERBOSE ? System.out : null);
+ Document document = new Document();
+ document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
+ Field.TermVector.YES));
+ for(int i=0;i<177;i++)
+ iw.addDocument(document);
+ iw.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Fri Jun 24 23:07:10 2011
@@ -114,7 +114,6 @@ public class TestIndexWriterOnDiskFull e
// Make sure reader can open the index:
IndexReader.open(dir, true).close();
- _TestUtil.checkIndex(dir);
}
dir.close();
@@ -491,7 +490,6 @@ public class TestIndexWriterOnDiskFull e
w.addDocument(doc);
w.close();
- _TestUtil.checkIndex(dir);
dir.close();
}
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Fri Jun 24 23:07:10 2011
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util._TestUtil;
@@ -63,7 +64,7 @@ public class TestIndexWriterOnJRECrash e
}
} else {
// we are the fork, setup a crashing thread
- final int crashTime = _TestUtil.nextInt(random, 500, 4000);
+ final int crashTime = _TestUtil.nextInt(random, 3000, 4000);
Thread t = new Thread() {
@Override
public void run() {
@@ -92,6 +93,8 @@ public class TestIndexWriterOnJRECrash e
+ "java");
cmd.add("-Xmx512m");
cmd.add("-Dtests.crashmode=true");
+ // passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act...
+ cmd.add("-Dtests.nightly=true");
cmd.add("-DtempDir=" + tempDir.getPath());
cmd.add("-Dtests.seed=" + random.nextLong() + ":" + random.nextLong());
cmd.add("-ea");
@@ -123,7 +126,8 @@ public class TestIndexWriterOnJRECrash e
*/
public boolean checkIndexes(File file) throws IOException {
if (file.isDirectory()) {
- Directory dir = newFSDirectory(file);
+ MockDirectoryWrapper dir = newFSDirectory(file);
+ dir.setCheckIndexOnClose(false); // don't double-checkindex
if (IndexReader.indexExists(dir)) {
if (VERBOSE) {
System.err.println("Checking index: " + file);
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Fri Jun 24 23:07:10 2011
@@ -658,7 +658,7 @@ public class TestIndexWriterReader exten
((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
- int num = 100 * RANDOM_MULTIPLIER;
+ int num = atLeast(100);
for (int i = 0; i < num; i++) {
writer.addDocument(createDocument(i, "test", 4));
}
@@ -821,7 +821,6 @@ public class TestIndexWriterReader exten
writer.close();
- _TestUtil.checkIndex(dir1);
dir1.close();
}
@@ -908,7 +907,6 @@ public class TestIndexWriterReader exten
assertEquals(0, excs.size());
writer.close();
- _TestUtil.checkIndex(dir1);
r.close();
dir1.close();
}
@@ -1015,6 +1013,7 @@ public class TestIndexWriterReader exten
HashSet<String> illegalCodecs = new HashSet<String>();
illegalCodecs.add("PreFlex");
illegalCodecs.add("SimpleText");
+ illegalCodecs.add("Memory");
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1);
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java Fri Jun 24 23:07:10 2011
@@ -25,6 +25,8 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
/**
@@ -33,8 +35,8 @@ import org.apache.lucene.util.LuceneTest
*/
public class TestLazyBug extends LuceneTestCase {
- public static int NUM_DOCS = 500;
- public static int NUM_FIELDS = 100;
+ public static int NUM_DOCS = TEST_NIGHTLY ? 500 : 50;
+ public static int NUM_FIELDS = TEST_NIGHTLY ? 100 : 10;
private static String[] data = new String[] {
"now",
@@ -49,6 +51,19 @@ public class TestLazyBug extends LuceneT
private static Set<String> dataset = asSet(data);
private static String MAGIC_FIELD = "f"+(NUM_FIELDS/3);
+
+ private static Directory directory;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ directory = makeIndex();
+ }
+
+ @AfterClass
+ public static void afterClass() throws Exception {
+ directory.close();
+ directory = null;
+ }
private static FieldSelector SELECTOR = new FieldSelector() {
public FieldSelectorResult accept(String f) {
@@ -59,7 +74,7 @@ public class TestLazyBug extends LuceneT
}
};
- private Directory makeIndex() throws Exception {
+ private static Directory makeIndex() throws Exception {
Directory dir = newDirectory();
try {
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
@@ -72,7 +87,7 @@ public class TestLazyBug extends LuceneT
doc.add(newField("f"+f,
data[f % data.length]
+ '#' + data[random.nextInt(data.length)],
- Field.Store.YES,
+ Field.Store.NO,
Field.Index.ANALYZED));
}
writer.addDocument(doc);
@@ -85,8 +100,7 @@ public class TestLazyBug extends LuceneT
}
public void doTest(int[] docs) throws Exception {
- Directory dir = makeIndex();
- IndexReader reader = IndexReader.open(dir, true);
+ IndexReader reader = IndexReader.open(directory, true);
for (int i = 0; i < docs.length; i++) {
Document d = reader.document(docs[i], SELECTOR);
d.get(MAGIC_FIELD);
@@ -109,19 +123,18 @@ public class TestLazyBug extends LuceneT
}
}
reader.close();
- dir.close();
}
public void testLazyWorks() throws Exception {
- doTest(new int[] { 399 });
+ doTest(new int[] { NUM_DOCS-1 });
}
public void testLazyAlsoWorks() throws Exception {
- doTest(new int[] { 399, 150 });
+ doTest(new int[] { NUM_DOCS-1, NUM_DOCS/2 });
}
public void testLazyBroken() throws Exception {
- doTest(new int[] { 150, 399 });
+ doTest(new int[] { NUM_DOCS/2, NUM_DOCS-1 });
}
}
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Fri Jun 24 23:07:10 2011
@@ -133,6 +133,7 @@ public class TestLazyProxSkipping extend
public void testLazySkipping() throws IOException {
assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText"));
+ assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory"));
// test whether only the minimum amount of seeks()
// are performed
performTest(5);
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Fri Jun 24 23:07:10 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.File;
import java.io.IOException;
import java.io.StringReader;
@@ -65,12 +64,13 @@ public class TestLongPostings extends Lu
public void testLongPostings() throws Exception {
assumeFalse("Too slow with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+ assumeFalse("Too slow with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
// Don't use _TestUtil.getTempDir so that we own the
// randomness (ie same seed will point to same dir):
Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong()));
- final int NUM_DOCS = (int) ((TEST_NIGHTLY ? 4e6 : (RANDOM_MULTIPLIER*2e4)) * (1+random.nextDouble()));
+ final int NUM_DOCS = atLeast(2000);
if (VERBOSE) {
System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
@@ -145,7 +145,8 @@ public class TestLongPostings extends Lu
assertTrue(r.docFreq(new Term("field", s1)) > 0);
assertTrue(r.docFreq(new Term("field", s2)) > 0);
- for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
+ int num = atLeast(1000);
+ for(int iter=0;iter<num;iter++) {
final String term;
final boolean doS1;
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java Fri Jun 24 23:07:10 2011
@@ -27,7 +27,7 @@ public class TestMultiFields extends Luc
public void testRandom() throws Exception {
- int num = 2 * RANDOM_MULTIPLIER;
+ int num = atLeast(2);
for (int iter = 0; iter < num; iter++) {
Directory dir = newDirectory();
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Fri Jun 24 23:07:10 2011
@@ -43,7 +43,6 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -97,14 +96,16 @@ public class TestNRTThreads extends Luce
final long t0 = System.currentTimeMillis();
- if (CodecProvider.getDefault().getDefaultFieldCodec().equals("SimpleText")) {
+ final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec();
+ if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) {
// no
CodecProvider.getDefault().setDefaultFieldCodec("Standard");
}
final LineFileDocs docs = new LineFileDocs(random);
final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
- final MockDirectoryWrapper dir = new MockDirectoryWrapper(random, FSDirectory.open(tempDir));
+ final MockDirectoryWrapper dir = newFSDirectory(tempDir);
+ dir.setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves.
final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
if (LuceneTestCase.TEST_NIGHTLY) {
@@ -157,7 +158,7 @@ public class TestNRTThreads extends Luce
final int NUM_INDEX_THREADS = 2;
final int NUM_SEARCH_THREADS = 3;
- final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
+ final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
final AtomicBoolean failed = new AtomicBoolean();
final AtomicInteger addCount = new AtomicInteger();
@@ -328,11 +329,11 @@ public class TestNRTThreads extends Luce
if (addedField != null) {
doc.removeField(addedField);
}
- } catch (Exception exc) {
+ } catch (Throwable t) {
System.out.println(Thread.currentThread().getName() + ": hit exc");
- exc.printStackTrace();
+ t.printStackTrace();
failed.set(true);
- throw new RuntimeException(exc);
+ throw new RuntimeException(t);
}
}
if (VERBOSE) {
@@ -405,30 +406,30 @@ public class TestNRTThreads extends Luce
for(int thread=0;thread<NUM_SEARCH_THREADS;thread++) {
searchThreads[thread] = new Thread() {
@Override
- public void run() {
+ public void run() {
try {
TermsEnum termsEnum = MultiFields.getTerms(s.getIndexReader(), "body").iterator();
int seenTermCount = 0;
int shift;
int trigger;
- if (totTermCount.get() == 0) {
+ if (totTermCount.get() < 10) {
shift = 0;
trigger = 1;
} else {
- shift = random.nextInt(totTermCount.get()/10);
trigger = totTermCount.get()/10;
+ shift = random.nextInt(trigger);
}
while(System.currentTimeMillis() < searchStopTime) {
BytesRef term = termsEnum.next();
if (term == null) {
- if (seenTermCount == 0) {
+ if (seenTermCount < 10) {
break;
}
totTermCount.set(seenTermCount);
seenTermCount = 0;
trigger = totTermCount.get()/10;
//System.out.println("trigger " + trigger);
- shift = random.nextInt(totTermCount.get()/10);
+ shift = random.nextInt(trigger);
termsEnum.seek(new BytesRef(""));
continue;
}
@@ -448,6 +449,7 @@ public class TestNRTThreads extends Luce
System.out.println(Thread.currentThread().getName() + ": search done");
}
} catch (Throwable t) {
+ System.out.println(Thread.currentThread().getName() + ": hit exc");
failed.set(true);
t.printStackTrace(System.out);
throw new RuntimeException(t);
@@ -577,7 +579,7 @@ public class TestNRTThreads extends Luce
private int runQuery(IndexSearcher s, Query q) throws Exception {
s.search(q, 10);
- return s.search(q, null, 10, new Sort(new SortField("title", SortField.STRING))).totalHits;
+ return s.search(q, null, 10, new Sort(new SortField("title", SortField.Type.STRING))).totalHits;
}
private void smokeTestReader(IndexReader r) throws Exception {
Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java Fri Jun 24 23:07:10 2011
@@ -143,7 +143,8 @@ public class TestNorms extends LuceneTes
}
private void doTestNorms(Random random, Directory dir) throws IOException {
- for (int i=0; i<5; i++) {
+ int num = atLeast(1);
+ for (int i=0; i<num; i++) {
addDocs(random, dir,12,true);
verifyIndex(dir);
modifyNormsForF1(dir);