You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Robert Muir <rc...@gmail.com> on 2011/01/26 18:52:34 UTC
Re: svn commit: r1063384 - in /lucene/dev/branches/branch_3x: ./
lucene/ lucene/backwards/src/test/org/apache/lucene/index/
lucene/backwards/src/test/org/apache/lucene/search/ lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene
Mike, looks like this one is the culprit causing the recent test fails
(ant test-core -Dtestcase=TestIndexReader -Dtestmethod=testDiskFull
-Dtests.seed=-3874264177341011095:-5876795174467582038
-Dtests.multiplier=5)
On Tue, Jan 25, 2011 at 1:05 PM, <mi...@apache.org> wrote:
> Author: mikemccand
> Date: Tue Jan 25 18:05:44 2011
> New Revision: 1063384
>
> URL: http://svn.apache.org/viewvc?rev=1063384&view=rev
> Log:
> LUCENE-2010: drop segments that are 100% deleted docs in IW/IR commit
>
> Modified:
> lucene/dev/branches/branch_3x/ (props changed)
> lucene/dev/branches/branch_3x/lucene/ (props changed)
> lucene/dev/branches/branch_3x/lucene/CHANGES.txt
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java
> lucene/dev/branches/branch_3x/solr/ (props changed)
> lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
>
> Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
> +++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Jan 25 18:05:44 2011
> @@ -153,6 +153,9 @@ Changes in runtime behavior
> * LUCENE-2829: Improve the performance of "primary key" lookup use
> case (running a TermQuery that matches one document) on a
> multi-segment index. (Robert Muir, Mike McCandless)
> +
> +* LUCENE-2010: Segments with 100% deleted documents are now removed on
> + IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
>
> API Changes
>
> @@ -578,6 +581,9 @@ Optimizations
> * LUCENE-2824: Optimize BufferedIndexInput to do less bounds checks.
> (Robert Muir)
>
> +* LUCENE-2010: Segments with 100% deleted documents are now removed on
> + IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
> +
> Build
>
> * LUCENE-2124: Moved the JDK-based collation support from contrib/collation
>
> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java Tue Jan 25 18:05:44 2011
> @@ -371,7 +371,7 @@ public class TestAddIndexesNoOptimize ex
> writer.setMergeFactor(4);
>
> writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) });
> - assertEquals(1060, writer.maxDoc());
> + assertEquals(1020, writer.maxDoc());
> assertEquals(1000, writer.getDocCount(0));
> writer.close();
> }
> @@ -413,7 +413,7 @@ public class TestAddIndexesNoOptimize ex
> writer.setMergeFactor(4);
>
> writer.addIndexesNoOptimize(new Directory[] { aux, aux2 });
> - assertEquals(1060, writer.maxDoc());
> + assertEquals(1040, writer.maxDoc());
> assertEquals(1000, writer.getDocCount(0));
> writer.close();
> }
>
> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 18:05:44 2011
> @@ -340,7 +340,7 @@ public class TestIndexReader extends Luc
>
> // CREATE A NEW READER and re-test
> reader = IndexReader.open(dir, false);
> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
> + assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
> reader.close();
> reader2.close();
> @@ -688,7 +688,6 @@ public class TestIndexReader extends Luc
>
> // CREATE A NEW READER and re-test
> reader = IndexReader.open(dir, false);
> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
> assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
> assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
> @@ -839,7 +838,6 @@ public class TestIndexReader extends Luc
> writer.close();
> IndexReader reader = IndexReader.open(dir, false);
> reader.deleteDocument(0);
> - reader.deleteDocument(1);
> reader.close();
> reader = IndexReader.open(dir, false);
> reader.undeleteAll();
> @@ -856,7 +854,6 @@ public class TestIndexReader extends Luc
> writer.close();
> IndexReader reader = IndexReader.open(dir, false);
> reader.deleteDocument(0);
> - reader.deleteDocument(1);
> reader.close();
> reader = IndexReader.open(dir, false);
> reader.undeleteAll();
> @@ -1254,9 +1251,6 @@ public class TestIndexReader extends Luc
>
> // Open another reader to confirm that everything is deleted
> reader2 = IndexReader.open(dir, false);
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
> assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
> assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
> assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
>
> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Jan 25 18:05:44 2011
> @@ -1214,7 +1214,6 @@ public class TestIndexReaderReopen exten
>
> IndexReader r = IndexReader.open(dir, false);
> assertEquals(0, r.numDocs());
> - assertEquals(4, r.maxDoc());
>
> Iterator it = IndexReader.listCommits(dir).iterator();
> while(it.hasNext()) {
>
> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jan 25 18:05:44 2011
> @@ -3173,7 +3173,6 @@ public class TestIndexWriter extends Luc
> w.close();
>
> IndexReader ir = IndexReader.open(dir, true);
> - assertEquals(1, ir.maxDoc());
> assertEquals(0, ir.numDocs());
> ir.close();
>
>
> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java Tue Jan 25 18:05:44 2011
> @@ -7,8 +7,8 @@ import java.io.ObjectInputStream;
> import java.io.ObjectOutputStream;
> import java.util.ArrayList;
> import java.util.List;
> -
> import junit.framework.Assert;
> +import java.lang.reflect.Method;
>
> import org.apache.lucene.analysis.WhitespaceAnalyzer;
> import org.apache.lucene.document.Document;
> @@ -209,6 +209,18 @@ public class QueryUtils {
> }
> w.commit();
> w.deleteDocuments( new MatchAllDocsQuery() );
> + try {
> + // Carefully invoke what is a package-private (test
> + // only, internal) method on IndexWriter:
> + Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
> + if (m != null) {
> + m.setAccessible(true);
> + m.invoke(w);
> + }
> + } catch (Exception e) {
> + // Should not happen?
> + throw new RuntimeException(e);
> + }
> w.commit();
>
> if (0 < numDeletedDocs)
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Tue Jan 25 18:05:44 2011
> @@ -270,9 +270,9 @@ class BufferedDeletes {
> }
>
> private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool,
> - SegmentInfo info,
> - SegmentDeletes coalescedDeletes,
> - SegmentDeletes segmentDeletes) throws IOException {
> + SegmentInfo info,
> + SegmentDeletes coalescedDeletes,
> + SegmentDeletes segmentDeletes) throws IOException {
> assert readerPool.infoIsLive(info);
>
> assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0;
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Jan 25 18:05:44 2011
> @@ -773,6 +773,9 @@ class DirectoryReader extends IndexReade
> for (int i = 0; i < subReaders.length; i++)
> subReaders[i].commit();
>
> + // Remove segments that contain only 100% deleted docs:
> + segmentInfos.pruneDeletedSegments();
> +
> // Sync all files we just wrote
> directory.sync(segmentInfos.files(directory, false));
> segmentInfos.commit(directory);
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Jan 25 18:05:44 2011
> @@ -1031,7 +1031,14 @@ public abstract class IndexReader implem
> return n;
> }
>
> - /** Undeletes all documents currently marked as deleted in this index.
> + /** Undeletes all documents currently marked as deleted in
> + * this index.
> + *
> + * <p>NOTE: this is only a best-effort process. For
> + * example, if all documents in a given segment were
> + * deleted, Lucene now drops that segment from the index,
> + * which means its documents will not be recovered by this
> + * method.
> *
> * @throws StaleReaderException if the index has changed
> * since this reader was opened
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jan 25 18:05:44 2011
> @@ -4121,6 +4121,15 @@ public class IndexWriter implements Clos
> }
> }
>
> + private boolean keepFullyDeletedSegments;
> +
> + /** Only for testing.
> + *
> + * @lucene.internal */
> + void keepFullyDeletedSegments() {
> + keepFullyDeletedSegments = true;
> + }
> +
> // called only from assert
> private boolean filesExist(SegmentInfos toSync) throws IOException {
> Collection<String> files = toSync.files(directory, false);
> @@ -4179,6 +4188,10 @@ public class IndexWriter implements Clos
> readerPool.commit();
>
> toSync = (SegmentInfos) segmentInfos.clone();
> + if (!keepFullyDeletedSegments) {
> + toSync.pruneDeletedSegments();
> + }
> +
> assert filesExist(toSync);
>
> if (commitUserData != null)
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Tue Jan 25 18:05:44 2011
> @@ -404,6 +404,19 @@ public final class SegmentInfos extends
> }
> }
>
> + /** Prunes any segment whose docs are all deleted. */
> + public void pruneDeletedSegments() throws IOException {
> + int segIdx = 0;
> + while(segIdx < size()) {
> + final SegmentInfo info = info(segIdx);
> + if (info.getDelCount() == info.docCount) {
> + remove(segIdx);
> + } else {
> + segIdx++;
> + }
> + }
> + }
> +
> /**
> * Returns a copy of this instance, also copying each
> * SegmentInfo.
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Jan 25 18:05:44 2011
> @@ -394,7 +394,7 @@ public class TestAddIndexes extends Luce
> ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4);
>
> writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(random, new RAMDirectory(aux)) });
> - assertEquals(1060, writer.maxDoc());
> + assertEquals(1020, writer.maxDoc());
> assertEquals(1000, writer.getDocCount(0));
> writer.close();
> dir.close();
> @@ -438,7 +438,7 @@ public class TestAddIndexes extends Luce
> ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4);
>
> writer.addIndexes(new Directory[] { aux, aux2 });
> - assertEquals(1060, writer.maxDoc());
> + assertEquals(1040, writer.maxDoc());
> assertEquals(1000, writer.getDocCount(0));
> writer.close();
> dir.close();
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 18:05:44 2011
> @@ -350,7 +350,7 @@ public class TestIndexReader extends Luc
>
> // CREATE A NEW READER and re-test
> reader = IndexReader.open(dir, false);
> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
> + assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
> reader.close();
> reader2.close();
> @@ -682,7 +682,6 @@ public class TestIndexReader extends Luc
>
> // CREATE A NEW READER and re-test
> reader = IndexReader.open(dir, false);
> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
> assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
> assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
> @@ -823,7 +822,6 @@ public class TestIndexReader extends Luc
> writer.close();
> IndexReader reader = IndexReader.open(dir, false);
> reader.deleteDocument(0);
> - reader.deleteDocument(1);
> reader.close();
> reader = IndexReader.open(dir, false);
> reader.undeleteAll();
> @@ -840,7 +838,6 @@ public class TestIndexReader extends Luc
> writer.close();
> IndexReader reader = IndexReader.open(dir, false);
> reader.deleteDocument(0);
> - reader.deleteDocument(1);
> reader.close();
> reader = IndexReader.open(dir, false);
> reader.undeleteAll();
> @@ -1228,9 +1225,6 @@ public class TestIndexReader extends Luc
>
> // Open another reader to confirm that everything is deleted
> reader2 = IndexReader.open(dir, false);
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
> assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
> assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
> assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Jan 25 18:05:44 2011
> @@ -1200,7 +1200,6 @@ public class TestIndexReaderReopen exten
>
> IndexReader r = IndexReader.open(dir, false);
> assertEquals(0, r.numDocs());
> - assertEquals(4, r.maxDoc());
>
> Collection<IndexCommit> commits = IndexReader.listCommits(dir);
> for (final IndexCommit commit : commits) {
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jan 25 18:05:44 2011
> @@ -101,19 +101,12 @@ public class TestIndexWriter extends Luc
> }
> reader.close();
>
> - // test doc count before segments are merged/index is optimized
> - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
> - assertEquals(100, writer.maxDoc());
> - writer.close();
> -
> reader = IndexReader.open(dir, true);
> - assertEquals(100, reader.maxDoc());
> assertEquals(60, reader.numDocs());
> reader.close();
>
> // optimize the index and check that the new doc count is correct
> writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
> - assertEquals(100, writer.maxDoc());
> assertEquals(60, writer.numDocs());
> writer.optimize();
> assertEquals(60, writer.maxDoc());
> @@ -1601,7 +1594,6 @@ public class TestIndexWriter extends Luc
> w.close();
>
> IndexReader ir = IndexReader.open(dir, true);
> - assertEquals(1, ir.maxDoc());
> assertEquals(0, ir.numDocs());
> ir.close();
>
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Jan 25 18:05:44 2011
> @@ -562,22 +562,23 @@ public class TestIndexWriterExceptions e
> System.out.println("TEST: open reader");
> }
> IndexReader reader = IndexReader.open(dir, true);
> - int expected = 3+(1-i)*2;
> - assertEquals(expected, reader.docFreq(new Term("contents", "here")));
> - assertEquals(expected, reader.maxDoc());
> - int numDel = 0;
> - for(int j=0;j<reader.maxDoc();j++) {
> - if (reader.isDeleted(j))
> - numDel++;
> - else {
> - reader.document(j);
> - reader.getTermFreqVectors(j);
> + if (i == 0) {
> + int expected = 5;
> + assertEquals(expected, reader.docFreq(new Term("contents", "here")));
> + assertEquals(expected, reader.maxDoc());
> + int numDel = 0;
> + for(int j=0;j<reader.maxDoc();j++) {
> + if (reader.isDeleted(j))
> + numDel++;
> + else {
> + reader.document(j);
> + reader.getTermFreqVectors(j);
> + }
> }
> + assertEquals(1, numDel);
> }
> reader.close();
>
> - assertEquals(1, numDel);
> -
> writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
> analyzer).setMaxBufferedDocs(10));
> doc = new Document();
> @@ -589,10 +590,10 @@ public class TestIndexWriterExceptions e
> writer.close();
>
> reader = IndexReader.open(dir, true);
> - expected = 19+(1-i)*2;
> + int expected = 19+(1-i)*2;
> assertEquals(expected, reader.docFreq(new Term("contents", "here")));
> assertEquals(expected, reader.maxDoc());
> - numDel = 0;
> + int numDel = 0;
> for(int j=0;j<reader.maxDoc();j++) {
> if (reader.isDeleted(j))
> numDel++;
>
> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java (original)
> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java Tue Jan 25 18:05:44 2011
> @@ -6,6 +6,7 @@ import java.io.IOException;
> import java.io.ObjectInputStream;
> import java.io.ObjectOutputStream;
> import java.util.Random;
> +import java.lang.reflect.Method;
>
> import junit.framework.Assert;
>
> @@ -214,6 +215,16 @@ public class QueryUtils {
> }
> w.commit();
> w.deleteDocuments( new MatchAllDocsQuery() );
> + try {
> + // Carefully invoke what is a package-private (test
> + // only, internal) method on IndexWriter:
> + Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
> + m.setAccessible(true);
> + m.invoke(w);
> + } catch (Exception e) {
> + // Should not happen?
> + throw new RuntimeException(e);
> + }
> w.commit();
>
> if (0 < numDeletedDocs)
>
> Modified: lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java?rev=1063384&r1=1063383&r2=1063384&view=diff
> ==============================================================================
> --- lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java (original)
> +++ lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java Tue Jan 25 18:05:44 2011
> @@ -12,6 +12,7 @@ public class DistributedTermsComponentTe
>
> @Override
> public void doTest() throws Exception {
> + del("*:*");
> index(id, 18, "b_t", "snake spider shark snail slug seal");
> index(id, 19, "b_t", "snake spider shark snail slug");
> index(id, 20, "b_t", "snake spider shark snail");
>
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org
Re: svn commit: r1063384 - in /lucene/dev/branches/branch_3x: ./
lucene/ lucene/backwards/src/test/org/apache/lucene/index/
lucene/backwards/src/test/org/apache/lucene/search/ lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene
Posted by Michael McCandless <lu...@mikemccandless.com>.
Argh! I will dig.
Mike
On Wed, Jan 26, 2011 at 12:52 PM, Robert Muir <rc...@gmail.com> wrote:
> Mike, looks like this one is the culprit causing the recent test fails
> (ant test-core -Dtestcase=TestIndexReader -Dtestmethod=testDiskFull
> -Dtests.seed=-3874264177341011095:-5876795174467582038
> -Dtests.multiplier=5)
>
> On Tue, Jan 25, 2011 at 1:05 PM, <mi...@apache.org> wrote:
>> Author: mikemccand
>> Date: Tue Jan 25 18:05:44 2011
>> New Revision: 1063384
>>
>> URL: http://svn.apache.org/viewvc?rev=1063384&view=rev
>> Log:
>> LUCENE-2010: drop segments that are 100% deleted docs in IW/IR commit
>>
>> Modified:
>> lucene/dev/branches/branch_3x/ (props changed)
>> lucene/dev/branches/branch_3x/lucene/ (props changed)
>> lucene/dev/branches/branch_3x/lucene/CHANGES.txt
>> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java
>> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
>> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
>> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
>> lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java
>> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
>> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
>> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
>> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
>> lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
>> lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java
>> lucene/dev/branches/branch_3x/solr/ (props changed)
>> lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
>> +++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Tue Jan 25 18:05:44 2011
>> @@ -153,6 +153,9 @@ Changes in runtime behavior
>> * LUCENE-2829: Improve the performance of "primary key" lookup use
>> case (running a TermQuery that matches one document) on a
>> multi-segment index. (Robert Muir, Mike McCandless)
>> +
>> +* LUCENE-2010: Segments with 100% deleted documents are now removed on
>> + IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
>>
>> API Changes
>>
>> @@ -578,6 +581,9 @@ Optimizations
>> * LUCENE-2824: Optimize BufferedIndexInput to do less bounds checks.
>> (Robert Muir)
>>
>> +* LUCENE-2010: Segments with 100% deleted documents are now removed on
>> + IndexReader or IndexWriter commit. (Uwe Schindler, Mike McCandless)
>> +
>> Build
>>
>> * LUCENE-2124: Moved the JDK-based collation support from contrib/collation
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java Tue Jan 25 18:05:44 2011
>> @@ -371,7 +371,7 @@ public class TestAddIndexesNoOptimize ex
>> writer.setMergeFactor(4);
>>
>> writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) });
>> - assertEquals(1060, writer.maxDoc());
>> + assertEquals(1020, writer.maxDoc());
>> assertEquals(1000, writer.getDocCount(0));
>> writer.close();
>> }
>> @@ -413,7 +413,7 @@ public class TestAddIndexesNoOptimize ex
>> writer.setMergeFactor(4);
>>
>> writer.addIndexesNoOptimize(new Directory[] { aux, aux2 });
>> - assertEquals(1060, writer.maxDoc());
>> + assertEquals(1040, writer.maxDoc());
>> assertEquals(1000, writer.getDocCount(0));
>> writer.close();
>> }
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 18:05:44 2011
>> @@ -340,7 +340,7 @@ public class TestIndexReader extends Luc
>>
>> // CREATE A NEW READER and re-test
>> reader = IndexReader.open(dir, false);
>> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
>> + assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
>> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
>> reader.close();
>> reader2.close();
>> @@ -688,7 +688,6 @@ public class TestIndexReader extends Luc
>>
>> // CREATE A NEW READER and re-test
>> reader = IndexReader.open(dir, false);
>> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
>> assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
>> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
>> assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
>> @@ -839,7 +838,6 @@ public class TestIndexReader extends Luc
>> writer.close();
>> IndexReader reader = IndexReader.open(dir, false);
>> reader.deleteDocument(0);
>> - reader.deleteDocument(1);
>> reader.close();
>> reader = IndexReader.open(dir, false);
>> reader.undeleteAll();
>> @@ -856,7 +854,6 @@ public class TestIndexReader extends Luc
>> writer.close();
>> IndexReader reader = IndexReader.open(dir, false);
>> reader.deleteDocument(0);
>> - reader.deleteDocument(1);
>> reader.close();
>> reader = IndexReader.open(dir, false);
>> reader.undeleteAll();
>> @@ -1254,9 +1251,6 @@ public class TestIndexReader extends Luc
>>
>> // Open another reader to confirm that everything is deleted
>> reader2 = IndexReader.open(dir, false);
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
>> assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
>> assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
>> assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Jan 25 18:05:44 2011
>> @@ -1214,7 +1214,6 @@ public class TestIndexReaderReopen exten
>>
>> IndexReader r = IndexReader.open(dir, false);
>> assertEquals(0, r.numDocs());
>> - assertEquals(4, r.maxDoc());
>>
>> Iterator it = IndexReader.listCommits(dir).iterator();
>> while(it.hasNext()) {
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jan 25 18:05:44 2011
>> @@ -3173,7 +3173,6 @@ public class TestIndexWriter extends Luc
>> w.close();
>>
>> IndexReader ir = IndexReader.open(dir, true);
>> - assertEquals(1, ir.maxDoc());
>> assertEquals(0, ir.numDocs());
>> ir.close();
>>
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/search/QueryUtils.java Tue Jan 25 18:05:44 2011
>> @@ -7,8 +7,8 @@ import java.io.ObjectInputStream;
>> import java.io.ObjectOutputStream;
>> import java.util.ArrayList;
>> import java.util.List;
>> -
>> import junit.framework.Assert;
>> +import java.lang.reflect.Method;
>>
>> import org.apache.lucene.analysis.WhitespaceAnalyzer;
>> import org.apache.lucene.document.Document;
>> @@ -209,6 +209,18 @@ public class QueryUtils {
>> }
>> w.commit();
>> w.deleteDocuments( new MatchAllDocsQuery() );
>> + try {
>> + // Carefully invoke what is a package-private (test
>> + // only, internal) method on IndexWriter:
>> + Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
>> + if (m != null) {
>> + m.setAccessible(true);
>> + m.invoke(w);
>> + }
>> + } catch (Exception e) {
>> + // Should not happen?
>> + throw new RuntimeException(e);
>> + }
>> w.commit();
>>
>> if (0 < numDeletedDocs)
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Tue Jan 25 18:05:44 2011
>> @@ -270,9 +270,9 @@ class BufferedDeletes {
>> }
>>
>> private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool,
>> - SegmentInfo info,
>> - SegmentDeletes coalescedDeletes,
>> - SegmentDeletes segmentDeletes) throws IOException {
>> + SegmentInfo info,
>> + SegmentDeletes coalescedDeletes,
>> + SegmentDeletes segmentDeletes) throws IOException {
>> assert readerPool.infoIsLive(info);
>>
>> assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0;
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Jan 25 18:05:44 2011
>> @@ -773,6 +773,9 @@ class DirectoryReader extends IndexReade
>> for (int i = 0; i < subReaders.length; i++)
>> subReaders[i].commit();
>>
>> + // Remove segments that contain only 100% deleted docs:
>> + segmentInfos.pruneDeletedSegments();
>> +
>> // Sync all files we just wrote
>> directory.sync(segmentInfos.files(directory, false));
>> segmentInfos.commit(directory);
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Jan 25 18:05:44 2011
>> @@ -1031,7 +1031,14 @@ public abstract class IndexReader implem
>> return n;
>> }
>>
>> - /** Undeletes all documents currently marked as deleted in this index.
>> + /** Undeletes all documents currently marked as deleted in
>> + * this index.
>> + *
>> + * <p>NOTE: this is only a best-effort process. For
>> + * example, if all documents in a given segment were
>> + * deleted, Lucene now drops that segment from the index,
>> + * which means its documents will not be recovered by this
>> + * method.
>> *
>> * @throws StaleReaderException if the index has changed
>> * since this reader was opened
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jan 25 18:05:44 2011
>> @@ -4121,6 +4121,15 @@ public class IndexWriter implements Clos
>> }
>> }
>>
>> + private boolean keepFullyDeletedSegments;
>> +
>> + /** Only for testing.
>> + *
>> + * @lucene.internal */
>> + void keepFullyDeletedSegments() {
>> + keepFullyDeletedSegments = true;
>> + }
>> +
>> // called only from assert
>> private boolean filesExist(SegmentInfos toSync) throws IOException {
>> Collection<String> files = toSync.files(directory, false);
>> @@ -4179,6 +4188,10 @@ public class IndexWriter implements Clos
>> readerPool.commit();
>>
>> toSync = (SegmentInfos) segmentInfos.clone();
>> + if (!keepFullyDeletedSegments) {
>> + toSync.pruneDeletedSegments();
>> + }
>> +
>> assert filesExist(toSync);
>>
>> if (commitUserData != null)
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Tue Jan 25 18:05:44 2011
>> @@ -404,6 +404,19 @@ public final class SegmentInfos extends
>> }
>> }
>>
>> + /** Prunes any segment whose docs are all deleted. */
>> + public void pruneDeletedSegments() throws IOException {
>> + int segIdx = 0;
>> + while(segIdx < size()) {
>> + final SegmentInfo info = info(segIdx);
>> + if (info.getDelCount() == info.docCount) {
>> + remove(segIdx);
>> + } else {
>> + segIdx++;
>> + }
>> + }
>> + }
>> +
>> /**
>> * Returns a copy of this instance, also copying each
>> * SegmentInfo.
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Tue Jan 25 18:05:44 2011
>> @@ -394,7 +394,7 @@ public class TestAddIndexes extends Luce
>> ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4);
>>
>> writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(random, new RAMDirectory(aux)) });
>> - assertEquals(1060, writer.maxDoc());
>> + assertEquals(1020, writer.maxDoc());
>> assertEquals(1000, writer.getDocCount(0));
>> writer.close();
>> dir.close();
>> @@ -438,7 +438,7 @@ public class TestAddIndexes extends Luce
>> ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4);
>>
>> writer.addIndexes(new Directory[] { aux, aux2 });
>> - assertEquals(1060, writer.maxDoc());
>> + assertEquals(1040, writer.maxDoc());
>> assertEquals(1000, writer.getDocCount(0));
>> writer.close();
>> dir.close();
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jan 25 18:05:44 2011
>> @@ -350,7 +350,7 @@ public class TestIndexReader extends Luc
>>
>> // CREATE A NEW READER and re-test
>> reader = IndexReader.open(dir, false);
>> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
>> + assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm));
>> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
>> reader.close();
>> reader2.close();
>> @@ -682,7 +682,6 @@ public class TestIndexReader extends Luc
>>
>> // CREATE A NEW READER and re-test
>> reader = IndexReader.open(dir, false);
>> - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm));
>> assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2));
>> assertTermDocsCount("deleted termDocs", reader, searchTerm, 0);
>> assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100);
>> @@ -823,7 +822,6 @@ public class TestIndexReader extends Luc
>> writer.close();
>> IndexReader reader = IndexReader.open(dir, false);
>> reader.deleteDocument(0);
>> - reader.deleteDocument(1);
>> reader.close();
>> reader = IndexReader.open(dir, false);
>> reader.undeleteAll();
>> @@ -840,7 +838,6 @@ public class TestIndexReader extends Luc
>> writer.close();
>> IndexReader reader = IndexReader.open(dir, false);
>> reader.deleteDocument(0);
>> - reader.deleteDocument(1);
>> reader.close();
>> reader = IndexReader.open(dir, false);
>> reader.undeleteAll();
>> @@ -1228,9 +1225,6 @@ public class TestIndexReader extends Luc
>>
>> // Open another reader to confirm that everything is deleted
>> reader2 = IndexReader.open(dir, false);
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1));
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2));
>> - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3));
>> assertTermDocsCount("reopened 2", reader2, searchTerm1, 0);
>> assertTermDocsCount("reopened 2", reader2, searchTerm2, 0);
>> assertTermDocsCount("reopened 2", reader2, searchTerm3, 100);
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Tue Jan 25 18:05:44 2011
>> @@ -1200,7 +1200,6 @@ public class TestIndexReaderReopen exten
>>
>> IndexReader r = IndexReader.open(dir, false);
>> assertEquals(0, r.numDocs());
>> - assertEquals(4, r.maxDoc());
>>
>> Collection<IndexCommit> commits = IndexReader.listCommits(dir);
>> for (final IndexCommit commit : commits) {
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jan 25 18:05:44 2011
>> @@ -101,19 +101,12 @@ public class TestIndexWriter extends Luc
>> }
>> reader.close();
>>
>> - // test doc count before segments are merged/index is optimized
>> - writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
>> - assertEquals(100, writer.maxDoc());
>> - writer.close();
>> -
>> reader = IndexReader.open(dir, true);
>> - assertEquals(100, reader.maxDoc());
>> assertEquals(60, reader.numDocs());
>> reader.close();
>>
>> // optimize the index and check that the new doc count is correct
>> writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
>> - assertEquals(100, writer.maxDoc());
>> assertEquals(60, writer.numDocs());
>> writer.optimize();
>> assertEquals(60, writer.maxDoc());
>> @@ -1601,7 +1594,6 @@ public class TestIndexWriter extends Luc
>> w.close();
>>
>> IndexReader ir = IndexReader.open(dir, true);
>> - assertEquals(1, ir.maxDoc());
>> assertEquals(0, ir.numDocs());
>> ir.close();
>>
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue Jan 25 18:05:44 2011
>> @@ -562,22 +562,23 @@ public class TestIndexWriterExceptions e
>> System.out.println("TEST: open reader");
>> }
>> IndexReader reader = IndexReader.open(dir, true);
>> - int expected = 3+(1-i)*2;
>> - assertEquals(expected, reader.docFreq(new Term("contents", "here")));
>> - assertEquals(expected, reader.maxDoc());
>> - int numDel = 0;
>> - for(int j=0;j<reader.maxDoc();j++) {
>> - if (reader.isDeleted(j))
>> - numDel++;
>> - else {
>> - reader.document(j);
>> - reader.getTermFreqVectors(j);
>> + if (i == 0) {
>> + int expected = 5;
>> + assertEquals(expected, reader.docFreq(new Term("contents", "here")));
>> + assertEquals(expected, reader.maxDoc());
>> + int numDel = 0;
>> + for(int j=0;j<reader.maxDoc();j++) {
>> + if (reader.isDeleted(j))
>> + numDel++;
>> + else {
>> + reader.document(j);
>> + reader.getTermFreqVectors(j);
>> + }
>> }
>> + assertEquals(1, numDel);
>> }
>> reader.close();
>>
>> - assertEquals(1, numDel);
>> -
>> writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
>> analyzer).setMaxBufferedDocs(10));
>> doc = new Document();
>> @@ -589,10 +590,10 @@ public class TestIndexWriterExceptions e
>> writer.close();
>>
>> reader = IndexReader.open(dir, true);
>> - expected = 19+(1-i)*2;
>> + int expected = 19+(1-i)*2;
>> assertEquals(expected, reader.docFreq(new Term("contents", "here")));
>> assertEquals(expected, reader.maxDoc());
>> - numDel = 0;
>> + int numDel = 0;
>> for(int j=0;j<reader.maxDoc();j++) {
>> if (reader.isDeleted(j))
>> numDel++;
>>
>> Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java (original)
>> +++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/QueryUtils.java Tue Jan 25 18:05:44 2011
>> @@ -6,6 +6,7 @@ import java.io.IOException;
>> import java.io.ObjectInputStream;
>> import java.io.ObjectOutputStream;
>> import java.util.Random;
>> +import java.lang.reflect.Method;
>>
>> import junit.framework.Assert;
>>
>> @@ -214,6 +215,16 @@ public class QueryUtils {
>> }
>> w.commit();
>> w.deleteDocuments( new MatchAllDocsQuery() );
>> + try {
>> + // Carefully invoke what is a package-private (test
>> + // only, internal) method on IndexWriter:
>> + Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
>> + m.setAccessible(true);
>> + m.invoke(w);
>> + } catch (Exception e) {
>> + // Should not happen?
>> + throw new RuntimeException(e);
>> + }
>> w.commit();
>>
>> if (0 < numDeletedDocs)
>>
>> Modified: lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
>> URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java?rev=1063384&r1=1063383&r2=1063384&view=diff
>> ==============================================================================
>> --- lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java (original)
>> +++ lucene/dev/branches/branch_3x/solr/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java Tue Jan 25 18:05:44 2011
>> @@ -12,6 +12,7 @@ public class DistributedTermsComponentTe
>>
>> @Override
>> public void doTest() throws Exception {
>> + del("*:*");
>> index(id, 18, "b_t", "snake spider shark snail slug seal");
>> index(id, 19, "b_t", "snake spider shark snail slug");
>> index(id, 20, "b_t", "snake spider shark snail");
>>
>>
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org