svn commit: r1504860 - in /lucene/dev/trunk/lucene: CHANGES.txt core/src/java/org/apache/lucene/index/IndexWriter.java core/src/test/org/apache/lucene/index/TestAddIndexes.java

[sh...@apache.org]

Author: shaie
Date: Fri Jul 19 12:43:11 2013
New Revision: 1504860

URL: http://svn.apache.org/r1504860
Log:
LUCENE-5116: IW.addIndexes doesn't prune all deleted segments

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1504860&r1=1504859&r2=1504860&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Jul 19 12:43:11 2013
@@ -64,6 +64,11 @@ New features
   (default is false).  If true then edits are measured in Unicode code
   points instead of UTF8 bytes.  (Artem Lukanin via Mike McCandless)
 
+Bug Fixes
+
+* LUCENE-5116: IndexWriter.addIndexes(IndexReader...) should drop empty (or all
+  deleted) segments. (Robert Muir, Shai Erera)
+
 API Changes
 
 * LUCENE-5094: Add ramBytesUsed() to MultiDocValues.OrdinalMap.

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1504860&r1=1504859&r2=1504860&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Fri Jul 19 12:43:11 2013
@@ -2429,6 +2429,10 @@ public class IndexWriter implements Clos
    * close the writer. See <a href="#OOME">above</a> for details.
    * 
    * <p>
+   * <b>NOTE:</b> empty segments are dropped by this method and not added to this
+   * index.
+   * 
+   * <p>
    * <b>NOTE:</b> this method merges all given {@link IndexReader}s in one
    * merge. If you intend to merge a large number of readers, it may be better
    * to call this method multiple times, each time with a small set of readers.
@@ -2462,11 +2466,20 @@ public class IndexWriter implements Clos
       String mergedName = newSegmentName();
       final List<AtomicReader> mergeReaders = new ArrayList<AtomicReader>();
       for (IndexReader indexReader : readers) {
-        numDocs += indexReader.numDocs();
-        for (AtomicReaderContext ctx : indexReader.leaves()) {
-          mergeReaders.add(ctx.reader());
+        if (indexReader.numDocs() > 0) {
+          numDocs += indexReader.numDocs();
+          for (AtomicReaderContext ctx : indexReader.leaves()) {
+            if (ctx.reader().numDocs() > 0) { // drop empty (or all deleted) segments
+              mergeReaders.add(ctx.reader());
+            }
+          }
         }
       }
+      
+      if (mergeReaders.isEmpty()) { // no segments with documents to add
+        return;
+      }
+      
       final IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, -1));
 
       // TODO: somehow we should fix this merge so it's

Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1504860&r1=1504859&r2=1504860&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Jul 19 12:43:11 2013
@@ -43,6 +43,7 @@ import org.apache.lucene.store.BaseDirec
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -1205,4 +1206,53 @@ public class TestAddIndexes extends Luce
     r3.close();
     d3.close();
   }
+  
+  public void testAddEmpty() throws Exception {
+    Directory d1 = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random(), d1);
+    MultiReader empty = new MultiReader();
+    w.addIndexes(empty);
+    w.close();
+    DirectoryReader dr = DirectoryReader.open(d1);
+    for (AtomicReaderContext ctx : dr.leaves()) {
+      assertTrue("empty segments should be dropped by addIndexes", ctx.reader().maxDoc() > 0);
+    }
+    dr.close();
+    d1.close();
+  }
+
+  // Currently it's impossible to end up with a segment with all documents
+  // deleted, as such segments are dropped. Still, to validate that addIndexes
+  // works with such segments, or readers that end up in such state, we fake an
+  // all deleted segment.
+  public void testFakeAllDeleted() throws Exception {
+    Directory src = newDirectory(), dest = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random(), src);
+    w.addDocument(new Document());
+    IndexReader allDeletedReader = new FilterAtomicReader(w.getReader().leaves().get(0).reader()) {
+      @Override
+      public Bits getLiveDocs() {
+        return new Bits() {
+          @Override public int length() { return 1; }
+          @Override public boolean get(int index) { return false; }
+        };
+      }
+      @Override public boolean hasDeletions() { return true; }
+      @Override public int numDocs() { return 0; }
+    };
+    w.close();
+    
+    w = new RandomIndexWriter(random(), dest);
+    w.addIndexes(allDeletedReader);
+    w.close();
+    DirectoryReader dr = DirectoryReader.open(src);
+    for (AtomicReaderContext ctx : dr.leaves()) {
+      assertTrue("empty segments should be dropped by addIndexes", ctx.reader().maxDoc() > 0);
+    }
+    dr.close();
+    allDeletedReader.close();
+    src.close();
+    dest.close();
+  }
+
 }