You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/05/05 11:09:43 UTC

svn commit: r1099728 [1/2] - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test/org/apache/lucene/index/ lucene/contrib/queries/src/test/org/apache/lucene/search/ lucene/src/java/org/apache/lucene/index/ lucene/sr...

Author: mikemccand
Date: Thu May  5 09:09:41 2011
New Revision: 1099728

URL: http://svn.apache.org/viewvc?rev=1099728&view=rev
Log:
LUCENE-1076: allow non-contiguous merges; improve handling of buffered deletes (merge from trunk, part 1)

Added:
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
      - copied, changed from r1065095, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
    lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java
      - copied unchanged from r1065095, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/MockRandomMergePolicy.java
Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
    lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
    lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
    lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDocBoost.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermScorer.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Thu May  5 09:09:41 2011
@@ -70,12 +70,12 @@ public class TestPerSegmentDeletes exten
     // flushing without applying deletes means 
     // there will still be deletes in the segment infos
     writer.flush(false, false);
-    assertTrue(writer.bufferedDeletes.any());
+    //assertTrue(writer.bufferedDeletes.any());
     
     // get reader flushes pending deletes
     // so there should not be anymore
     IndexReader r1 = writer.getReader();
-    assertFalse(writer.bufferedDeletes.any());
+    //assertFalse(writer.bufferedDeletes.any());
     r1.close();
     
     // delete id:2 from the first segment

Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java Thu May  5 09:09:41 2011
@@ -42,7 +42,6 @@ public class FuzzyLikeThisQueryTest exte
 	  super.setUp();
 		directory = newDirectory();
 		RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
-		
 		//Add series of docs with misspelt names
 		addDoc(writer, "jonathon smythe","1");
 		addDoc(writer, "jonathan smith","2");

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Thu May  5 09:09:41 2011
@@ -17,410 +17,165 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import java.io.IOException;
-import java.io.PrintStream;
+import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.Date;
-import java.util.Map.Entry;
+import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
-
-/** Holds a {@link SegmentDeletes} for each segment in the
- *  index. */
+import org.apache.lucene.util.RamUsageEstimator;
 
+/** Holds buffered deletes, by docID, term or query for a
+ *  single segment. This is used to hold buffered pending
+ *  deletes against the to-be-flushed segment as well as
+ *  per-segment deletes for each segment in the index. */
+
+// NOTE: we are sync'd by BufferedDeletes, ie, all access to
+// instances of this class is via sync'd methods on
+// BufferedDeletes
 class BufferedDeletes {
 
-  // Deletes for all flushed/merged segments:
-  private final Map<SegmentInfo,SegmentDeletes> deletesMap = new HashMap<SegmentInfo,SegmentDeletes>();
-
-  // used only by assert
-  private Term lastDeleteTerm;
-  
-  private PrintStream infoStream;
-  private final AtomicLong bytesUsed = new AtomicLong();
-  private final AtomicInteger numTerms = new AtomicInteger();
-  private final int messageID;
-
-  public BufferedDeletes(int messageID) {
-    this.messageID = messageID;
-  }
-
-  private synchronized void message(String message) {
-    if (infoStream != null) {
-      infoStream.println("BD " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: BD " + message);
-    }
-  }
-  
-  public synchronized void setInfoStream(PrintStream infoStream) {
-    this.infoStream = infoStream;
-  }
-
-  public synchronized void pushDeletes(SegmentDeletes newDeletes, SegmentInfo info) {
-    pushDeletes(newDeletes, info, false);
-  }
-
-  // Moves all pending deletes onto the provided segment,
-  // then clears the pending deletes
-  public synchronized void pushDeletes(SegmentDeletes newDeletes, SegmentInfo info, boolean noLimit) {
-    assert newDeletes.any();
-    numTerms.addAndGet(newDeletes.numTermDeletes.get());
-
-    if (!noLimit) {
-      assert !deletesMap.containsKey(info);
-      assert info != null;
-      deletesMap.put(info, newDeletes);
-      bytesUsed.addAndGet(newDeletes.bytesUsed.get());
+  /* Rough logic: HashMap has an array[Entry] w/ varying
+     load factor (say 2 * POINTER).  Entry is object w/ Term
+     key, Integer val, int hash, Entry next
+     (OBJ_HEADER + 3*POINTER + INT).  Term is object w/
+     String field and String text (OBJ_HEADER + 2*POINTER).
+     We don't count Term's field since it's interned.
+     Term's text is String (OBJ_HEADER + 4*INT + POINTER +
+     OBJ_HEADER + string.length*CHAR).  Integer is
+     OBJ_HEADER + INT. */
+  final static int BYTES_PER_DEL_TERM = 8*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
+
+  /* Rough logic: del docIDs are List<Integer>.  Say list
+     allocates ~2X size (2*POINTER).  Integer is OBJ_HEADER
+     + int */
+  final static int BYTES_PER_DEL_DOCID = 2*RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT;
+
+  /* Rough logic: HashMap has an array[Entry] w/ varying
+     load factor (say 2 * POINTER).  Entry is object w/
+     Query key, Integer val, int hash, Entry next
+     (OBJ_HEADER + 3*POINTER + INT).  Query we often
+     undercount (say 24 bytes).  Integer is OBJ_HEADER + INT. */
+  final static int BYTES_PER_DEL_QUERY = 5*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 2*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2*RamUsageEstimator.NUM_BYTES_INT + 24;
+
+  // TODO: many of the deletes stored here will map to
+  // Integer.MAX_VALUE; we could be more efficient for this
+  // case ie use a SortedSet not a SortedMap.  But: Java's
+  // SortedSet impls are simply backed by a Map so we won't
+  // save anything unless we do something custom...
+  final AtomicInteger numTermDeletes = new AtomicInteger();
+  final SortedMap<Term,Integer> terms = new TreeMap<Term,Integer>();
+  final Map<Query,Integer> queries = new HashMap<Query,Integer>();
+  final List<Integer> docIDs = new ArrayList<Integer>();
+
+  public static final Integer MAX_INT = Integer.valueOf(Integer.MAX_VALUE);
+
+  final AtomicLong bytesUsed = new AtomicLong();
+
+  private final static boolean VERBOSE_DELETES = false;
+
+  long gen;
+
+  @Override
+  public String toString() {
+    if (VERBOSE_DELETES) {
+      return "gen=" + gen + " numTerms=" + numTermDeletes + ", terms=" + terms
+        + ", queries=" + queries + ", docIDs=" + docIDs + ", bytesUsed="
+        + bytesUsed;
     } else {
-      final SegmentDeletes deletes = getDeletes(info);
-      bytesUsed.addAndGet(-deletes.bytesUsed.get());
-      deletes.update(newDeletes, noLimit);
-      bytesUsed.addAndGet(deletes.bytesUsed.get());
-    }    
-    if (infoStream != null) {
-      message("push deletes seg=" + info + " dels=" + getDeletes(info));
-    }
-    assert checkDeleteStats();    
-  }
-
-  public synchronized void clear() {
-    deletesMap.clear();
-    numTerms.set(0);
-    bytesUsed.set(0);
-  }
-
-  synchronized boolean any() {
-    return bytesUsed.get() != 0;
-  }
-
-  public int numTerms() {
-    return numTerms.get();
-  }
-
-  public long bytesUsed() {
-    return bytesUsed.get();
-  }
-
-  // IW calls this on finishing a merge.  While the merge
-  // was running, it's possible new deletes were pushed onto
-  // our last (and only our last) segment.  In this case we
-  // must carry forward those deletes onto the merged
-  // segment.
-  synchronized void commitMerge(MergePolicy.OneMerge merge) {
-    assert checkDeleteStats();
-    if (infoStream != null) {
-      message("commitMerge merge.info=" + merge.info + " merge.segments=" + merge.segments);
-    }
-    final SegmentInfo lastInfo = merge.segments.lastElement();
-    final SegmentDeletes lastDeletes = deletesMap.get(lastInfo);
-    if (lastDeletes != null) {
-      deletesMap.remove(lastInfo);
-      assert !deletesMap.containsKey(merge.info);
-      deletesMap.put(merge.info, lastDeletes);
-      // don't need to update numTerms/bytesUsed since we
-      // are just moving the deletes from one info to
-      // another
-      if (infoStream != null) {
-        message("commitMerge done: new deletions=" + lastDeletes);
+      String s = "gen=" + gen;
+      if (numTermDeletes.get() != 0) {
+        s += " " + numTermDeletes.get() + " deleted terms (unique count=" + terms.size() + ")";
       }
-    } else if (infoStream != null) {
-      message("commitMerge done: no new deletions");
-    }
-    assert !anyDeletes(merge.segments.range(0, merge.segments.size()-1));
-    assert checkDeleteStats();
-  }
-
-  synchronized void clear(SegmentDeletes deletes) {
-    deletes.clear();
-  }
-  
-  public synchronized boolean applyDeletes(IndexWriter.ReaderPool readerPool, SegmentInfos segmentInfos, SegmentInfos applyInfos) throws IOException {
-    if (!any()) {
-      return false;
-    }
-    final long t0 = System.currentTimeMillis();
-
-    if (infoStream != null) {
-      message("applyDeletes: applyInfos=" + applyInfos + "; index=" + segmentInfos);
-    }
-
-    assert checkDeleteStats();
-
-    assert applyInfos.size() > 0;
-
-    boolean any = false;
-    
-    final SegmentInfo lastApplyInfo = applyInfos.lastElement();
-    final int lastIdx = segmentInfos.indexOf(lastApplyInfo);
-    
-    final SegmentInfo firstInfo = applyInfos.firstElement();
-    final int firstIdx = segmentInfos.indexOf(firstInfo);
-
-    // applyInfos must be a slice of segmentInfos
-    assert lastIdx - firstIdx + 1 == applyInfos.size();
-    
-    // iterate over all segment infos backwards
-    // coalesceing deletes along the way 
-    // when we're at or below the last of the 
-    // segments to apply to, start applying the deletes
-    // we traverse up to the first apply infos
-    SegmentDeletes coalescedDeletes = null;
-    boolean hasDeletes = false;
-    for (int segIdx=segmentInfos.size()-1; segIdx >= firstIdx; segIdx--) {
-      final SegmentInfo info = segmentInfos.info(segIdx);
-      final SegmentDeletes deletes = deletesMap.get(info);
-      assert deletes == null || deletes.any();
-
-      if (deletes == null && coalescedDeletes == null) {
-        continue;
+      if (queries.size() != 0) {
+        s += " " + queries.size() + " deleted queries";
       }
-
-      if (infoStream != null) {
-        message("applyDeletes: seg=" + info + " segment's deletes=[" + (deletes == null ? "null" : deletes) + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "]");
+      if (docIDs.size() != 0) {
+        s += " " + docIDs.size() + " deleted docIDs";
       }
-
-      hasDeletes |= deletes != null;
-
-      if (segIdx <= lastIdx && hasDeletes) {
-
-        final long delCountInc = applyDeletes(readerPool, info, coalescedDeletes, deletes);
-
-        if (delCountInc != 0) {
-          any = true;
-        }
-
-        if (infoStream != null) {
-          message("deletes touched " + delCountInc + " docIDs");
-        }
-      
-        if (deletes != null) {
-          // we've applied doc ids, and they're only applied
-          // on the current segment
-          bytesUsed.addAndGet(-deletes.docIDs.size() * SegmentDeletes.BYTES_PER_DEL_DOCID);
-          deletes.clearDocIDs();
-        }
-      }
-      
-      // now coalesce at the max limit
-      if (deletes != null) {
-        if (coalescedDeletes == null) {
-          coalescedDeletes = new SegmentDeletes();
-        }
-        // TODO: we could make this single pass (coalesce as
-        // we apply the deletes
-        coalescedDeletes.update(deletes, true);
-      }
-    }
-
-    // move all deletes to segment just before our merge.
-    if (firstIdx > 0) {
-
-      SegmentDeletes mergedDeletes = null;
-      // TODO: we could also make this single pass
-      for (SegmentInfo info : applyInfos) {
-        final SegmentDeletes deletes = deletesMap.get(info);
-        if (deletes != null) {
-          assert deletes.any();
-          if (mergedDeletes == null) {
-            mergedDeletes = getDeletes(segmentInfos.info(firstIdx-1));
-            numTerms.addAndGet(-mergedDeletes.numTermDeletes.get());
-            assert numTerms.get() >= 0;
-            bytesUsed.addAndGet(-mergedDeletes.bytesUsed.get());
-            assert bytesUsed.get() >= 0;
-          }
-
-          mergedDeletes.update(deletes, true);
-        }
+      if (bytesUsed.get() != 0) {
+        s += " bytesUsed=" + bytesUsed.get();
       }
 
-      if (mergedDeletes != null) {
-        numTerms.addAndGet(mergedDeletes.numTermDeletes.get());
-        bytesUsed.addAndGet(mergedDeletes.bytesUsed.get());
-      }
-
-      if (infoStream != null) {
-        if (mergedDeletes != null) {
-          message("applyDeletes: merge all deletes into seg=" + segmentInfos.info(firstIdx-1) + ": " + mergedDeletes);
-        } else {
-          message("applyDeletes: no deletes to merge");
-        }
-      }
-    } else {
-      // We drop the deletes in this case, because we've
-      // applied them to segment infos starting w/ the first
-      // segment.  There are no prior segments so there's no
-      // reason to keep them around.  When the applyInfos ==
-      // segmentInfos this means all deletes have been
-      // removed:
-    }
-    remove(applyInfos);
-
-    assert checkDeleteStats();
-    assert applyInfos != segmentInfos || !any();
-    
-    if (infoStream != null) {
-      message("applyDeletes took " + (System.currentTimeMillis()-t0) + " msec");
+      return s;
     }
-    return any;
   }
   
-  private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool,
-                                         SegmentInfo info, 
-                                         SegmentDeletes coalescedDeletes,
-                                         SegmentDeletes segmentDeletes) throws IOException {    
-    assert readerPool.infoIsLive(info);
-    
-    assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0;
-    
-    long delCount = 0;
-
-    // Lock order: IW -> BD -> RP
-    SegmentReader reader = readerPool.get(info, false);
-    try {
-      if (coalescedDeletes != null) {
-        delCount += applyDeletes(coalescedDeletes, reader);
-      }
-      if (segmentDeletes != null) {
-        delCount += applyDeletes(segmentDeletes, reader);
+  void update(BufferedDeletes in) {
+    numTermDeletes.addAndGet(in.numTermDeletes.get());
+    for (Map.Entry<Term,Integer> ent : in.terms.entrySet()) {
+      final Term term = ent.getKey();
+      if (!terms.containsKey(term)) {
+        // only incr bytesUsed if this term wasn't already buffered:
+        bytesUsed.addAndGet(BYTES_PER_DEL_TERM);
       }
-    } finally {
-      readerPool.release(reader);
+      terms.put(term, MAX_INT);
     }
-    return delCount;
-  }
-  
-  private synchronized long applyDeletes(SegmentDeletes deletes, SegmentReader reader) throws IOException {
-
-    long delCount = 0;
 
-    assert checkDeleteTerm(null);
-    
-    if (deletes.terms.size() > 0) {
-      final TermDocs docs = reader.termDocs();
-
-      for (Entry<Term,Integer> entry: deletes.terms.entrySet()) {
-        Term term = entry.getKey();
-        // Since we visit terms sorted, we gain performance
-        // by re-using the same TermsEnum and seeking only
-        // forwards
-        assert checkDeleteTerm(term);
-        docs.seek(term);
-          
-        final int limit = entry.getValue();
-        while (docs.next()) {
-          final int docID = docs.doc();
-          if (docID >= limit) {
-            break;
-          }
-          // TODO: we could/should change
-          // reader.deleteDocument to return boolean
-          // true if it did in fact delete, because here
-          // we could be deleting an already-deleted doc
-          // which makes this an upper bound:
-          delCount++;
-
-          reader.deleteDocument(docID);
-        }
+    for (Map.Entry<Query,Integer> ent : in.queries.entrySet()) {
+      final Query query = ent.getKey();
+      if (!queries.containsKey(query)) {
+        // only incr bytesUsed if this query wasn't already buffered:
+        bytesUsed.addAndGet(BYTES_PER_DEL_QUERY);
       }
+      queries.put(query, MAX_INT);
     }
 
-    // Delete by docID
-    for (Integer docIdInt : deletes.docIDs) {
-      int docID = docIdInt.intValue();
-      reader.deleteDocument(docID);
-      delCount++;
-    }
-
-    // Delete by query
-    if (deletes.queries.size() > 0) {
-      IndexSearcher searcher = new IndexSearcher(reader);
-      try {
-        for (Entry<Query, Integer> entry : deletes.queries.entrySet()) {
-          Query query = entry.getKey();
-          int limit = entry.getValue().intValue();
-          Weight weight = query.weight(searcher);
-          Scorer scorer = weight.scorer(reader, true, false);
-          if (scorer != null) {
-            while(true)  {
-              int doc = scorer.nextDoc();
-              if (doc >= limit)
-                break;
-
-              reader.deleteDocument(doc);
-              // TODO: we could/should change
-              // reader.deleteDocument to return boolean
-              // true if it did in fact delete, because here
-              // we could be deleting an already-deleted doc
-              // which makes this an upper bound:
-              delCount++;
-            }
-          }
-        }
-      } finally {
-        searcher.close();
-      }
-    }
-
-    return delCount;
+    // docIDs never move across segments and the docIDs
+    // should already be cleared
   }
-  
-  public synchronized SegmentDeletes getDeletes(SegmentInfo info) {
-    SegmentDeletes deletes = deletesMap.get(info);
-    if (deletes == null) {
-      deletes = new SegmentDeletes();
-      deletesMap.put(info, deletes);
+
+  public void addQuery(Query query, int docIDUpto) {
+    Integer current = queries.put(query, docIDUpto);
+    // increment bytes used only if the query wasn't added so far.
+    if (current == null) {
+      bytesUsed.addAndGet(BYTES_PER_DEL_QUERY);
     }
-    return deletes;
   }
-  
-  public synchronized void remove(SegmentInfos infos) {
-    assert infos.size() > 0;
-    for (SegmentInfo info : infos) {
-      SegmentDeletes deletes = deletesMap.get(info);
-      if (deletes != null) {
-        bytesUsed.addAndGet(-deletes.bytesUsed.get());
-        assert bytesUsed.get() >= 0: "bytesUsed=" + bytesUsed;
-        numTerms.addAndGet(-deletes.numTermDeletes.get());
-        assert numTerms.get() >= 0: "numTerms=" + numTerms;
-        deletesMap.remove(info);
-      }
-    }
+
+  public void addDocID(int docID) {
+    docIDs.add(Integer.valueOf(docID));
+    bytesUsed.addAndGet(BYTES_PER_DEL_DOCID);
   }
 
-  // used only by assert
-  private boolean anyDeletes(SegmentInfos infos) {
-    for(SegmentInfo info : infos) {
-      if (deletesMap.containsKey(info)) {
-        return true;
-      }
+  public void addTerm(Term term, int docIDUpto) {
+    Integer current = terms.get(term);
+    if (current != null && docIDUpto < current) {
+      // Only record the new number if it's greater than the
+      // current one.  This is important because if multiple
+      // threads are replacing the same doc at nearly the
+      // same time, it's possible that one thread that got a
+      // higher docID is scheduled before the other
+      // threads.  If we blindly replace then we can
+      // incorrectly get both docs indexed.
+      return;
     }
-    return false;
-  }
 
-  // used only by assert
-  private boolean checkDeleteTerm(Term term) {
-    if (term != null) {
-      assert lastDeleteTerm == null || term.compareTo(lastDeleteTerm) > 0: "lastTerm=" + lastDeleteTerm + " vs term=" + term;
+    terms.put(term, Integer.valueOf(docIDUpto));
+    numTermDeletes.incrementAndGet();
+    if (current == null) {
+      bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.text.length() * RamUsageEstimator.NUM_BYTES_CHAR);
     }
-    lastDeleteTerm = term;
-    return true;
+  }
+    
+  void clear() {
+    terms.clear();
+    queries.clear();
+    docIDs.clear();
+    numTermDeletes.set(0);
+    bytesUsed.set(0);
   }
   
-  // only for assert
-  private boolean checkDeleteStats() {
-    int numTerms2 = 0;
-    long bytesUsed2 = 0;
-    for(SegmentDeletes deletes : deletesMap.values()) {
-      numTerms2 += deletes.numTermDeletes.get();
-      bytesUsed2 += deletes.bytesUsed.get();
-    }
-    assert numTerms2 == numTerms.get(): "numTerms2=" + numTerms2 + " vs " + numTerms.get();
-    assert bytesUsed2 == bytesUsed.get(): "bytesUsed2=" + bytesUsed2 + " vs " + bytesUsed;
-    return true;
+  void clearDocIDs() {
+    bytesUsed.addAndGet(-docIDs.size()*BYTES_PER_DEL_DOCID);
+    docIDs.clear();
+  }
+  
+  boolean any() {
+    return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0;
   }
 }

Copied: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (from r1065095, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?p2=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java&r1=1065095&r2=1099728&rev=1099728&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Thu May  5 09:09:41 2011
@@ -28,7 +28,6 @@ import java.util.Collections;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
@@ -130,7 +129,7 @@ class BufferedDeletesStream {
 
   // Sorts SegmentInfos from smallest to biggest bufferedDelGen:
   private static final Comparator<SegmentInfo> sortByDelGen = new Comparator<SegmentInfo>() {
-    @Override
+    // @Override -- not until Java 1.6
     public int compare(SegmentInfo si1, SegmentInfo si2) {
       final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
       if (cmp > 0) {
@@ -318,58 +317,30 @@ class BufferedDeletesStream {
     assert checkDeleteTerm(null);
     
     if (deletes.terms.size() > 0) {
-      Fields fields = reader.fields();
-      if (fields == null) {
-        // This reader has no postings
-        return 0;
-      }
-
-      TermsEnum termsEnum = null;
-        
-      String currentField = null;
-      DocsEnum docs = null;
+      final TermDocs docs = reader.termDocs();
         
       for (Entry<Term,Integer> entry: deletes.terms.entrySet()) {
         Term term = entry.getKey();
         // Since we visit terms sorted, we gain performance
         // by re-using the same TermsEnum and seeking only
         // forwards
-        if (term.field() != currentField) {
-          assert currentField == null || currentField.compareTo(term.field()) < 0;
-          currentField = term.field();
-          Terms terms = fields.terms(currentField);
-          if (terms != null) {
-            termsEnum = terms.iterator();
-          } else {
-            termsEnum = null;
-          }
-        }
-          
-        if (termsEnum == null) {
-          continue;
-        }
         assert checkDeleteTerm(term);
+        docs.seek(term);
           
-        if (termsEnum.seek(term.bytes(), false) == TermsEnum.SeekStatus.FOUND) {
-          DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
-            
-          if (docsEnum != null) {
-            docs = docsEnum;
-            final int limit = entry.getValue();
-            while (true) {
-              final int docID = docs.nextDoc();
-              if (docID == DocsEnum.NO_MORE_DOCS || docID >= limit) {
-                break;
-              }
-              reader.deleteDocument(docID);
-              // TODO: we could/should change
-              // reader.deleteDocument to return boolean
-              // true if it did in fact delete, because here
-              // we could be deleting an already-deleted doc
-              // which makes this an upper bound:
-              delCount++;
-            }
+        final int limit = entry.getValue();
+        while (docs.next()) {
+          final int docID = docs.doc();
+          if (docID >= limit) {
+            break;
           }
+          // TODO: we could/should change
+          // reader.deleteDocument to return boolean
+          // true if it did in fact delete, because here
+          // we could be deleting an already-deleted doc
+          // which makes this an upper bound:
+          delCount++;
+
+          reader.deleteDocument(docID);
         }
       }
     }
@@ -384,14 +355,12 @@ class BufferedDeletesStream {
     // Delete by query
     if (deletes.queries.size() > 0) {
       IndexSearcher searcher = new IndexSearcher(reader);
-      assert searcher.getTopReaderContext().isAtomic;
-      final AtomicReaderContext readerContext = (AtomicReaderContext) searcher.getTopReaderContext();
       try {
         for (Entry<Query, Integer> entry : deletes.queries.entrySet()) {
           Query query = entry.getKey();
           int limit = entry.getValue().intValue();
           Weight weight = query.weight(searcher);
-          Scorer scorer = weight.scorer(readerContext, Weight.ScorerContext.def());
+          Scorer scorer = weight.scorer(reader, true, false);
           if (scorer != null) {
             while(true)  {
               int doc = scorer.nextDoc();

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu May  5 09:09:41 2011
@@ -132,7 +132,7 @@ final class DocumentsWriter {
   private final int maxThreadStates;
 
   // Deletes for our still-in-RAM (to be flushed next) segment
-  private SegmentDeletes pendingDeletes = new SegmentDeletes();
+  private BufferedDeletes pendingDeletes = new BufferedDeletes();
   
   static class DocState {
     DocumentsWriter docWriter;
@@ -264,16 +264,16 @@ final class DocumentsWriter {
   private boolean closed;
   private final FieldInfos fieldInfos;
 
-  private final BufferedDeletes bufferedDeletes;
+  private final BufferedDeletesStream bufferedDeletesStream;
   private final IndexWriter.FlushControl flushControl;
 
-  DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldInfos fieldInfos, BufferedDeletes bufferedDeletes) throws IOException {
+  DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldInfos fieldInfos, BufferedDeletesStream bufferedDeletesStream) throws IOException {
     this.directory = directory;
     this.writer = writer;
     this.similarity = config.getSimilarity();
     this.maxThreadStates = config.getMaxThreadStates();
     this.fieldInfos = fieldInfos;
-    this.bufferedDeletes = bufferedDeletes;
+    this.bufferedDeletesStream = bufferedDeletesStream;
     flushControl = writer.flushControl;
 
     consumer = config.getIndexingChain().getChain(this);
@@ -463,23 +463,24 @@ final class DocumentsWriter {
   }
 
   // for testing
-  public SegmentDeletes getPendingDeletes() {
+  public BufferedDeletes getPendingDeletes() {
     return pendingDeletes;
   }
 
   private void pushDeletes(SegmentInfo newSegment, SegmentInfos segmentInfos) {
     // Lock order: DW -> BD
     if (pendingDeletes.any()) {
-      if (newSegment != null) {
+      if (segmentInfos.size() > 0 || newSegment != null) {
         if (infoStream != null) {
-          message("flush: push buffered deletes to newSegment");
+          message("flush: push buffered deletes");
         }
-        bufferedDeletes.pushDeletes(pendingDeletes, newSegment);
-      } else if (segmentInfos.size() > 0) {
+        bufferedDeletesStream.push(pendingDeletes);
         if (infoStream != null) {
-          message("flush: push buffered deletes to previously flushed segment " + segmentInfos.lastElement());
+          message("flush: delGen=" + pendingDeletes.gen);
+        }
+        if (newSegment != null) {
+          newSegment.setBufferedDeletesGen(pendingDeletes.gen);
         }
-        bufferedDeletes.pushDeletes(pendingDeletes, segmentInfos.lastElement(), true);
       } else {
         if (infoStream != null) {
           message("flush: drop buffered deletes: no segments");
@@ -488,7 +489,9 @@ final class DocumentsWriter {
         // there are no segments, the deletions cannot
         // affect anything.
       }
-      pendingDeletes = new SegmentDeletes();
+      pendingDeletes = new BufferedDeletes();
+    } else if (newSegment != null) {
+      newSegment.setBufferedDeletesGen(bufferedDeletesStream.getNextGen());
     }
   }
 
@@ -599,7 +602,6 @@ final class DocumentsWriter {
 
     // Lock order: IW -> DW -> BD
     pushDeletes(newSegment, segmentInfos);
-
     if (infoStream != null) {
       message("flush time " + (System.currentTimeMillis()-startTime) + " msec");
     }
@@ -1013,7 +1015,7 @@ final class DocumentsWriter {
     final boolean doBalance;
     final long deletesRAMUsed;
 
-    deletesRAMUsed = bufferedDeletes.bytesUsed();
+    deletesRAMUsed = bufferedDeletesStream.bytesUsed();
 
     final long ramBufferSize;
     final double mb = config.getRAMBufferSizeMB();

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu May  5 09:09:41 2011
@@ -309,7 +309,7 @@ public class IndexWriter implements Clos
   private final AtomicInteger flushDeletesCount = new AtomicInteger();
 
   final ReaderPool readerPool = new ReaderPool();
-  final BufferedDeletes bufferedDeletes;
+  final BufferedDeletesStream bufferedDeletesStream;
   
   // This is a "write once" variable (like the organic dye
   // on a DVD-R that may or may not be heated by a laser and
@@ -1077,8 +1077,8 @@ public class IndexWriter implements Clos
     mergePolicy = conf.getMergePolicy();
     mergePolicy.setIndexWriter(this);
     mergeScheduler = conf.getMergeScheduler();
-    bufferedDeletes = new BufferedDeletes(messageID);
-    bufferedDeletes.setInfoStream(infoStream);
+    bufferedDeletesStream = new BufferedDeletesStream(messageID);
+    bufferedDeletesStream.setInfoStream(infoStream);
     poolReaders = conf.getReaderPooling();
 
     OpenMode mode = conf.getOpenMode();
@@ -1146,7 +1146,7 @@ public class IndexWriter implements Clos
 
       setRollbackSegmentInfos(segmentInfos);
 
-      docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletes);
+      docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletesStream);
       docWriter.setInfoStream(infoStream);
       docWriter.setMaxFieldLength(maxFieldLength);
 
@@ -1646,7 +1646,7 @@ public class IndexWriter implements Clos
     this.infoStream = infoStream;
     docWriter.setInfoStream(infoStream);
     deleter.setInfoStream(infoStream);
-    bufferedDeletes.setInfoStream(infoStream);
+    bufferedDeletesStream.setInfoStream(infoStream);
     if (infoStream != null)
       messageState();
   }
@@ -1938,7 +1938,7 @@ public class IndexWriter implements Clos
 
   public synchronized boolean hasDeletions() throws IOException {
     ensureOpen();
-    if (bufferedDeletes.any()) {
+    if (bufferedDeletesStream.any()) {
       return true;
     }
     if (docWriter.anyDeletions()) {
@@ -2678,7 +2678,7 @@ public class IndexWriter implements Clos
       mergePolicy.close();
       mergeScheduler.close();
 
-      bufferedDeletes.clear();
+      bufferedDeletesStream.clear();
 
       synchronized(this) {
 
@@ -3349,10 +3349,10 @@ public class IndexWriter implements Clos
         // tiny segments:
         if (flushControl.getFlushDeletes() ||
             (config.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH &&
-             bufferedDeletes.bytesUsed() > (1024*1024*config.getRAMBufferSizeMB()/2))) {
+             bufferedDeletesStream.bytesUsed() > (1024*1024*config.getRAMBufferSizeMB()/2))) {
           applyAllDeletes = true;
           if (infoStream != null) {
-            message("force apply deletes bytesUsed=" + bufferedDeletes.bytesUsed() + " vs ramBuffer=" + (1024*1024*config.getRAMBufferSizeMB()));
+            message("force apply deletes bytesUsed=" + bufferedDeletesStream.bytesUsed() + " vs ramBuffer=" + (1024*1024*config.getRAMBufferSizeMB()));
           }
         }
       }
@@ -3362,12 +3362,15 @@ public class IndexWriter implements Clos
           message("apply all deletes during flush");
         }
         flushDeletesCount.incrementAndGet();
-        if (bufferedDeletes.applyDeletes(readerPool, segmentInfos, segmentInfos)) {
+        final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos);
+        if (result.anyDeletes) {
           checkpoint();
         }
+        bufferedDeletesStream.prune(segmentInfos);
+        assert !bufferedDeletesStream.any();
         flushControl.clearDeletes();
       } else if (infoStream != null) {
-        message("don't apply deletes now delTermCount=" + bufferedDeletes.numTerms() + " bytesUsed=" + bufferedDeletes.bytesUsed());
+        message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
       }
 
       doAfterFlush();
@@ -3393,7 +3396,7 @@ public class IndexWriter implements Clos
    */
   public final long ramSizeInBytes() {
     ensureOpen();
-    return docWriter.bytesUsed() + bufferedDeletes.bytesUsed();
+    return docWriter.bytesUsed() + bufferedDeletesStream.bytesUsed();
   }
 
   /** Expert:  Return the number of documents currently
@@ -3403,28 +3406,12 @@ public class IndexWriter implements Clos
     return docWriter.getNumDocs();
   }
 
-  private int ensureContiguousMerge(MergePolicy.OneMerge merge) {
-
-    int first = segmentInfos.indexOf(merge.segments.info(0));
-    if (first == -1)
-      throw new MergePolicy.MergeException("could not find segment " + merge.segments.info(0).name + " in current index " + segString(), directory);
-
-    final int numSegments = segmentInfos.size();
-    
-    final int numSegmentsToMerge = merge.segments.size();
-    for(int i=0;i<numSegmentsToMerge;i++) {
-      final SegmentInfo info = merge.segments.info(i);
-
-      if (first + i >= numSegments || !segmentInfos.info(first+i).equals(info)) {
-        if (segmentInfos.indexOf(info) == -1)
-          throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
-        else
-          throw new MergePolicy.MergeException("MergePolicy selected non-contiguous segments to merge (" + merge.segString(directory) + " vs " + segString() + "), which IndexWriter (currently) cannot handle",
-                                               directory);
+  private void ensureValidMerge(MergePolicy.OneMerge merge) {
+    for(SegmentInfo info : merge.segments) {
+      if (segmentInfos.indexOf(info) == -1) {
+        throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
       }
     }
-
-    return first;
   }
 
   /** Carefully merges deletes for the segments we just
@@ -3449,9 +3436,11 @@ public class IndexWriter implements Clos
     // started merging:
     int docUpto = 0;
     int delCount = 0;
+    long minGen = Long.MAX_VALUE;
 
     for(int i=0; i < sourceSegments.size(); i++) {
       SegmentInfo info = sourceSegments.info(i);
+      minGen = Math.min(info.getBufferedDeletesGen(), minGen);
       int docCount = info.docCount;
       SegmentReader previousReader = merge.readersClone[i];
       SegmentReader currentReader = merge.readers[i];
@@ -3500,9 +3489,17 @@ public class IndexWriter implements Clos
     assert mergedReader.numDeletedDocs() == delCount;
 
     mergedReader.hasChanges = delCount > 0;
+
+    // If new deletes were applied while we were merging
+    // (which happens if eg commit() or getReader() is
+    // called during our merge), then it better be the case
+    // that the delGen has increased for all our merged
+    // segments:
+    assert !mergedReader.hasChanges || minGen > mergedReader.getSegmentInfo().getBufferedDeletesGen();
+
+    mergedReader.getSegmentInfo().setBufferedDeletesGen(minGen);
   }
 
-  /* FIXME if we want to support non-contiguous segment merges */
   synchronized private boolean commitMerge(MergePolicy.OneMerge merge, SegmentReader mergedReader) throws IOException {
 
     assert testPoint("startCommitMerge");
@@ -3528,7 +3525,7 @@ public class IndexWriter implements Clos
       return false;
     }
 
-    final int start = ensureContiguousMerge(merge);
+    ensureValidMerge(merge);
 
     commitMergedDeletes(merge, mergedReader);
       
@@ -3538,10 +3535,32 @@ public class IndexWriter implements Clos
     // format as well:
     setMergeDocStoreIsCompoundFile(merge);
 
-    segmentInfos.subList(start, start + merge.segments.size()).clear();
     assert !segmentInfos.contains(merge.info);
-    segmentInfos.add(start, merge.info);
-    
+
+    final Set mergedAway = new HashSet<SegmentInfo>(merge.segments);
+    int segIdx = 0;
+    int newSegIdx = 0;
+    boolean inserted = false;
+    final int curSegCount = segmentInfos.size();
+    while(segIdx < curSegCount) {
+      final SegmentInfo info = segmentInfos.info(segIdx++);
+      if (mergedAway.contains(info)) {
+        if (!inserted) {
+          segmentInfos.set(segIdx-1, merge.info);
+          inserted = true;
+          newSegIdx++;
+        }
+      } else {
+        segmentInfos.set(newSegIdx++, info);
+      }
+    }
+    assert newSegIdx == curSegCount - merge.segments.size() + 1;
+    segmentInfos.subList(newSegIdx, segmentInfos.size()).clear();
+
+    if (infoStream != null) {
+      message("after commit: " + segString());
+    }
+
     closeMergeReaders(merge, false);
 
     // Must note the change to segmentInfos so any commits
@@ -3553,16 +3572,12 @@ public class IndexWriter implements Clos
     // disk, updating SegmentInfo, etc.:
     readerPool.clear(merge.segments);
     
-    // remove pending deletes of the segments 
-    // that were merged, moving them onto the segment just
-    // before the merged segment
-    // Lock order: IW -> BD
-    bufferedDeletes.commitMerge(merge);
-
     if (merge.optimize) {
       // cascade the optimize:
       segmentsToOptimize.add(merge.info);
     }
+
+    
     return true;
   }
   
@@ -3691,7 +3706,7 @@ public class IndexWriter implements Clos
       }
     }
 
-    ensureContiguousMerge(merge);
+    ensureValidMerge(merge);
 
     pendingMerges.add(merge);
 
@@ -3741,6 +3756,9 @@ public class IndexWriter implements Clos
       throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot merge");
     }
 
+    // TODO: is there any perf benefit to sorting
+    // merged segments?  eg biggest to smallest?
+
     if (merge.info != null)
       // mergeInit already done
       return;
@@ -3748,11 +3766,6 @@ public class IndexWriter implements Clos
     if (merge.isAborted())
       return;
 
-    // Lock order: IW -> BD
-    if (bufferedDeletes.applyDeletes(readerPool, segmentInfos, merge.segments)) {
-      checkpoint();
-    }
-
     boolean hasVectors = false;
     for (SegmentInfo sourceSegment : merge.segments) {
       if (sourceSegment.getHasVectors()) {
@@ -3765,6 +3778,17 @@ public class IndexWriter implements Clos
     // names.
     merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, true, false, hasVectors);
 
+    // Lock order: IW -> BD
+    final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments);
+    if (result.anyDeletes) {
+      checkpoint();
+    }
+
+    merge.info.setBufferedDeletesGen(result.gen);
+
+    // Lock order: IW -> BD
+    bufferedDeletesStream.prune(segmentInfos);
+
     Map<String,String> details = new HashMap<String,String>();
     details.put("optimize", Boolean.toString(merge.optimize));
     details.put("mergeFactor", Integer.toString(merge.segments.size()));
@@ -4402,7 +4426,7 @@ public class IndexWriter implements Clos
   }
 
   synchronized boolean nrtIsCurrent(SegmentInfos infos) {
-    return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletes.any();
+    return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
   }
 
   synchronized boolean isClosed() {
@@ -4569,7 +4593,7 @@ public class IndexWriter implements Clos
       final double ramBufferSizeMB = config.getRAMBufferSizeMB();
       if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
         final long limit = (long) (ramBufferSizeMB*1024*1024);
-        long used = bufferedDeletes.bytesUsed() + docWriter.bytesUsed();
+        long used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed();
         if (used >= limit) {
           
           // DocumentsWriter may be able to free up some
@@ -4577,7 +4601,7 @@ public class IndexWriter implements Clos
           // Lock order: FC -> DW
           docWriter.balanceRAM();
 
-          used = bufferedDeletes.bytesUsed() + docWriter.bytesUsed();
+          used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed();
           if (used >= limit) {
             return setFlushPending("ram full: " + reason, false);
           }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Thu May  5 09:09:41 2011
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
 
 import java.io.IOException;
 import java.util.Set;
+import java.util.Arrays;
+import java.util.Comparator;
 
 /** <p>This class implements a {@link MergePolicy} that tries
  *  to merge segments into levels of exponentially
@@ -67,6 +69,7 @@ public abstract class LogMergePolicy ext
   // out there wrote his own LMP ...
   protected long maxMergeSizeForOptimize = Long.MAX_VALUE;
   protected int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
+  protected boolean requireContiguousMerge = true;
 
   protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
 
@@ -105,6 +108,21 @@ public abstract class LogMergePolicy ext
       writer.get().message("LMP: " + message);
   }
 
+  /** If true, merges must be in-order slice of the
+   *  segments.  If false, then the merge policy is free to
+   *  pick any segments.  The default is false, which is
+   *  in general more efficient than true since it gives the
+   *  merge policy more freedom to pick closely sized
+   *  segments. */
+  public void setRequireContiguousMerge(boolean v) {
+    requireContiguousMerge = v;
+  }
+
+  /** See {@link #setRequireContiguousMerge}. */
+  public boolean getRequireContiguousMerge() {
+    return requireContiguousMerge;
+  }
+
   /** <p>Returns the number of segments that are merged at
    * once and also controls the total number of segments
    * allowed to accumulate in the index.</p> */
@@ -348,6 +366,8 @@ public abstract class LogMergePolicy ext
     // If the segments are already optimized (e.g. there's only 1 segment), or
     // there are <maxNumSegements, all optimized, nothing to do.
     if (isOptimized(infos, maxNumSegments, segmentsToOptimize)) return null;
+
+    // TODO: handle non-contiguous merge case differently?
     
     // Find the newest (rightmost) segment that needs to
     // be optimized (other segments may have been flushed
@@ -436,6 +456,37 @@ public abstract class LogMergePolicy ext
     return spec;
   }
 
+  private static class SegmentInfoAndLevel implements Comparable {
+    SegmentInfo info;
+    float level;
+    int index;
+    
+    public SegmentInfoAndLevel(SegmentInfo info, float level, int index) {
+      this.info = info;
+      this.level = level;
+      this.index = index;
+    }
+
+    // Sorts largest to smallest
+    public int compareTo(Object o) {
+      SegmentInfoAndLevel other = (SegmentInfoAndLevel) o;
+      if (level < other.level)
+        return 1;
+      else if (level > other.level)
+        return -1;
+      else
+        return 0;
+    }
+  }
+
+  private static class SortByIndex implements Comparator<SegmentInfoAndLevel> {
+    public int compare(SegmentInfoAndLevel o1, SegmentInfoAndLevel o2) {
+      return o1.index - o2.index;
+    }
+  }
+
+  private static final SortByIndex sortByIndex = new SortByIndex();
+
   /** Checks if any merges are now necessary and returns a
    *  {@link MergePolicy.MergeSpecification} if so.  A merge
    *  is necessary when there are more than {@link
@@ -452,7 +503,7 @@ public abstract class LogMergePolicy ext
 
     // Compute levels, which is just log (base mergeFactor)
     // of the size of each segment
-    float[] levels = new float[numSegments];
+    SegmentInfoAndLevel[] levels = new SegmentInfoAndLevel[numSegments];
     final float norm = (float) Math.log(mergeFactor);
 
     for(int i=0;i<numSegments;i++) {
@@ -462,7 +513,12 @@ public abstract class LogMergePolicy ext
       // Floor tiny segments
       if (size < 1)
         size = 1;
-      levels[i] = (float) Math.log(size)/norm;
+      levels[i] = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);
+      message("seg " + info.name + " level=" + levels[i].level + " size=" + size);
+    }
+
+    if (!requireContiguousMerge) {
+      Arrays.sort(levels);
     }
 
     final float levelFloor;
@@ -485,9 +541,9 @@ public abstract class LogMergePolicy ext
 
       // Find max level of all segments not already
       // quantized.
-      float maxLevel = levels[start];
+      float maxLevel = levels[start].level;
       for(int i=1+start;i<numSegments;i++) {
-        final float level = levels[i];
+        final float level = levels[i].level;
         if (level > maxLevel)
           maxLevel = level;
       }
@@ -508,7 +564,7 @@ public abstract class LogMergePolicy ext
 
       int upto = numSegments-1;
       while(upto >= start) {
-        if (levels[upto] >= levelBottom) {
+        if (levels[upto].level >= levelBottom) {
           break;
         }
         upto--;
@@ -521,18 +577,26 @@ public abstract class LogMergePolicy ext
       while(end <= 1+upto) {
         boolean anyTooLarge = false;
         for(int i=start;i<end;i++) {
-          final SegmentInfo info = infos.info(i);
+          final SegmentInfo info = levels[i].info;
           anyTooLarge |= (size(info) >= maxMergeSize || sizeDocs(info) >= maxMergeDocs);
         }
 
         if (!anyTooLarge) {
           if (spec == null)
             spec = new MergeSpecification();
-          if (verbose())
+          if (verbose()) {
             message("    " + start + " to " + end + ": add this merge");
-          spec.add(new OneMerge(infos.range(start, end)));
-        } else if (verbose())
+          }
+          Arrays.sort(levels, start, end, sortByIndex);
+          final SegmentInfos mergeInfos = new SegmentInfos();
+          for(int i=start;i<end;i++) {
+            mergeInfos.add(levels[i].info);
+            assert infos.contains(levels[i].info);
+          }
+          spec.add(new OneMerge(mergeInfos));
+        } else if (verbose()) {
           message("    " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
+        }
 
         start = end;
         end = start + mergeFactor;
@@ -580,6 +644,7 @@ public abstract class LogMergePolicy ext
     sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");
     sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");
     sb.append("useCompoundFile=").append(useCompoundFile);
+    sb.append("requireContiguousMerge=").append(requireContiguousMerge);
     sb.append("]");
     return sb.toString();
   }

Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Thu May  5 09:09:41 2011
@@ -102,6 +102,10 @@ public final class SegmentInfo {
   // afterwards ("3.0", "3.1" etc.).
   // see Constants.LUCENE_MAIN_VERSION.
   private String version;
+
+  // NOTE: only used in-RAM by IW to track buffered deletes;
+  // this is never written to/read from the Directory
+  private long bufferedDeletesGen;
   
   public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile,
                      boolean hasProx, boolean hasVectors) { 
@@ -856,5 +860,12 @@ public final class SegmentInfo {
   public String getVersion() {
     return version;
   }
-  
+
+  long getBufferedDeletesGen() {
+    return bufferedDeletesGen;
+  }
+
+  void setBufferedDeletesGen(long v) {
+    bufferedDeletesGen = v;
+  }
 }

Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Thu May  5 09:09:41 2011
@@ -31,8 +31,6 @@ import java.util.*;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Field.Index;
 import org.apache.lucene.document.Field.Store;
@@ -43,6 +41,7 @@ import org.apache.lucene.index.IndexWrit
 import org.apache.lucene.index.LogByteSizeMergePolicy;
 import org.apache.lucene.index.LogDocMergePolicy;
 import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MockRandomMergePolicy;
 import org.apache.lucene.index.SerialMergeScheduler;
 import org.apache.lucene.index.SlowMultiReaderWrapper;
 import org.apache.lucene.search.BooleanQuery;
@@ -652,15 +651,11 @@ public abstract class LuceneTestCase ext
       c.setMaxThreadStates(_TestUtil.nextInt(r, 1, 20));
     }
     
-    if (c.getMergePolicy() instanceof LogMergePolicy) {
-      LogMergePolicy logmp = (LogMergePolicy) c.getMergePolicy();
-      logmp.setUseCompoundFile(r.nextBoolean());
-      logmp.setCalibrateSizeByDeletes(r.nextBoolean());
-      if (r.nextInt(3) == 2) {
-        logmp.setMergeFactor(2);
-      } else {
-        logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20));
-      }
+    // TODO: turn this back on after all backports
+    if (false && r.nextBoolean()) {
+      c.setMergePolicy(new MockRandomMergePolicy(r));
+    } else {
+      c.setMergePolicy(newLogMergePolicy());
     }
     
     c.setReaderPooling(r.nextBoolean());

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Thu May  5 09:09:41 2011
@@ -95,8 +95,8 @@ public class TestMergeSchedulerExternal 
     
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new MyMergeScheduler())
-        .setMaxBufferedDocs(2).setRAMBufferSizeMB(
-            IndexWriterConfig.DISABLE_AUTO_FLUSH));
+        .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+        .setMergePolicy(newLogMergePolicy()));
     LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy();
     logMP.setMergeFactor(10);
     for(int i=0;i<20;i++)

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java Thu May  5 09:09:41 2011
@@ -75,8 +75,11 @@ public class TestSearch extends LuceneTe
       Directory directory = newDirectory();
       Analyzer analyzer = new MockAnalyzer(random);
       IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
+      
       IndexWriter writer = new IndexWriter(directory, conf);
 
       String[] docs = {
@@ -91,6 +94,7 @@ public class TestSearch extends LuceneTe
       for (int j = 0; j < docs.length; j++) {
         Document d = new Document();
         d.add(newField("contents", docs[j], Field.Store.YES, Field.Index.ANALYZED));
+        d.add(newField("id", ""+j, Field.Index.NOT_ANALYZED_NO_NORMS));
         writer.addDocument(d);
       }
       writer.close();
@@ -107,18 +111,17 @@ public class TestSearch extends LuceneTe
       };
       ScoreDoc[] hits = null;
 
+      Sort sort = new Sort(new SortField[] {
+          SortField.FIELD_SCORE,
+          new SortField("id", SortField.INT)});
+
       QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "contents", analyzer);
       parser.setPhraseSlop(4);
       for (int j = 0; j < queries.length; j++) {
         Query query = parser.parse(queries[j]);
         out.println("Query: " + query.toString("contents"));
 
-      //DateFilter filter =
-      //  new DateFilter("modified", Time(1997,0,1), Time(1998,0,1));
-      //DateFilter filter = DateFilter.Before("modified", Time(1997,00,01));
-      //System.out.println(filter);
-
-        hits = searcher.search(query, null, 1000).scoreDocs;
+        hits = searcher.search(query, null, 1000, sort).scoreDocs;
 
         out.println(hits.length + " total results");
         for (int i = 0 ; i < hits.length && i < 10; i++) {

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Thu May  5 09:09:41 2011
@@ -80,8 +80,10 @@ public class TestSearchForDuplicates ext
       Directory directory = newDirectory();
       Analyzer analyzer = new MockAnalyzer(random);
       IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFiles);
+      final MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFiles);
+      }
       IndexWriter writer = new IndexWriter(directory, conf);
       if (VERBOSE) {
         System.out.println("TEST: now build index");
@@ -106,7 +108,11 @@ public class TestSearchForDuplicates ext
       Query query = parser.parse(HIGH_PRIORITY);
       out.println("Query: " + query.toString(PRIORITY_FIELD));
 
-      ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS).scoreDocs;
+      final Sort sort = new Sort(new SortField[] {
+          SortField.FIELD_SCORE,
+          new SortField(ID_FIELD, SortField.INT)});
+
+      ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
       printHits(out, hits, searcher);
       checkHits(hits, MAX_DOCS, searcher);
 
@@ -121,7 +127,7 @@ public class TestSearchForDuplicates ext
       query = parser.parse(HIGH_PRIORITY + " OR " + MED_PRIORITY);
       out.println("Query: " + query.toString(PRIORITY_FIELD));
 
-      hits = searcher.search(query, null, MAX_DOCS).scoreDocs;
+      hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
       printHits(out, hits, searcher);
       checkHits(hits, MAX_DOCS, searcher);
 
@@ -143,7 +149,7 @@ public class TestSearchForDuplicates ext
   private void checkHits(ScoreDoc[] hits, int expectedCount, Searcher searcher) throws IOException {
     assertEquals("total results", expectedCount, hits.length);
     for (int i = 0 ; i < hits.length; i++) {
-      if ( i < 10 || (i > 94 && i < 105) ) {
+      if (i < 10 || (i > 94 && i < 105) ) {
       Document d = searcher.doc(hits[i].doc);
         assertEquals("check " + i, String.valueOf(i), d.get(ID_FIELD));
       }

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java Thu May  5 09:09:41 2011
@@ -20,7 +20,6 @@ import org.apache.lucene.util.*;
 import org.apache.lucene.store.*;
 import org.apache.lucene.document.*;
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
 
 import java.util.Random;
 import java.io.File;
@@ -132,6 +131,7 @@ public class TestAtomicUpdate extends Lu
         .setMaxBufferedDocs(7);
     ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(3);
     IndexWriter writer = new MockIndexWriter(directory, conf);
+    writer.setInfoStream(VERBOSE ? System.out : null);
 
     // Establish a base index of 100 docs:
     for(int i=0;i<100;i++) {

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Thu May  5 09:09:41 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -133,11 +132,15 @@ public class TestConcurrentMergeSchedule
     IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random))
         .setMergePolicy(mp));
+    writer.setInfoStream(VERBOSE ? System.out : null);
 
     Document doc = new Document();
     Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
     doc.add(idField);
     for(int i=0;i<10;i++) {
+      if (VERBOSE) {
+        System.out.println("\nTEST: cycle");
+      }
       for(int j=0;j<100;j++) {
         idField.setValue(Integer.toString(i*100+j));
         writer.addDocument(doc);
@@ -145,6 +148,9 @@ public class TestConcurrentMergeSchedule
 
       int delID = i;
       while(delID < 100*(1+i)) {
+        if (VERBOSE) {
+          System.out.println("TEST: del " + delID);
+        }
         writer.deleteDocuments(new Term("id", ""+delID));
         delID += 10;
       }

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Thu May  5 09:09:41 2011
@@ -120,6 +120,9 @@ public class TestDeletionPolicy extends 
     }
 
     public void onInit(List<? extends IndexCommit> commits) throws IOException {
+      if (VERBOSE) {
+        System.out.println("TEST: onInit");
+      }
       verifyCommitOrder(commits);
       numOnInit++;
       // do no deletions on init
@@ -127,6 +130,9 @@ public class TestDeletionPolicy extends 
     }
 
     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+      if (VERBOSE) {
+        System.out.println("TEST: onCommit");
+      }
       verifyCommitOrder(commits);
       doDeletes(commits, true);
     }
@@ -201,8 +207,10 @@ public class TestDeletionPolicy extends 
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
         new MockAnalyzer(random))
         .setIndexDeletionPolicy(policy);
-    LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-    lmp.setUseCompoundFile(true);
+    MergePolicy mp = conf.getMergePolicy();
+    if (mp instanceof LogMergePolicy) {
+      ((LogMergePolicy) mp).setUseCompoundFile(true);
+    }
     IndexWriter writer = new IndexWriter(dir, conf);
     writer.close();
 
@@ -216,8 +224,10 @@ public class TestDeletionPolicy extends 
       conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
           new MockAnalyzer(random)).setOpenMode(
           OpenMode.APPEND).setIndexDeletionPolicy(policy);
-      lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(true);
+      mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(true);
+      }
       writer = new IndexWriter(dir, conf);
       for(int j=0;j<17;j++) {
         addDoc(writer);
@@ -281,6 +291,10 @@ public class TestDeletionPolicy extends 
   public void testKeepAllDeletionPolicy() throws IOException {
     for(int pass=0;pass<2;pass++) {
 
+      if (VERBOSE) {
+        System.out.println("TEST: cycle pass=" + pass);
+      }
+
       boolean useCompoundFile = (pass % 2) != 0;
 
       // Never deletes a commit
@@ -293,34 +307,49 @@ public class TestDeletionPolicy extends 
           TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
           .setMergeScheduler(new SerialMergeScheduler());
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
-      lmp.setMergeFactor(10);
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
       IndexWriter writer = new IndexWriter(dir, conf);
       for(int i=0;i<107;i++) {
         addDoc(writer);
       }
       writer.close();
 
-      conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
-          new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
-          OpenMode.APPEND).setIndexDeletionPolicy(policy);
-      lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
-      writer = new IndexWriter(dir, conf);
-      writer.optimize();
-      writer.close();
-
-      assertEquals(1, policy.numOnInit);
+      final boolean isOptimized;
+      {
+        IndexReader r = IndexReader.open(dir);
+        isOptimized = r.isOptimized();
+        r.close();
+      }
+      if (!isOptimized) {
+        conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+                                    new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
+                                                                                              OpenMode.APPEND).setIndexDeletionPolicy(policy)
+          .setMergePolicy(newLogMergePolicy());
+        mp = conf.getMergePolicy();
+        if (mp instanceof LogMergePolicy) {
+          ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+        }
+        if (VERBOSE) {
+          System.out.println("TEST: open writer for optimize");
+        }
+        writer = new IndexWriter(dir, conf);
+        writer.setInfoStream(VERBOSE ? System.out : null);
+        writer.optimize();
+        writer.close();
+      }
+      assertEquals(isOptimized ? 0:1, policy.numOnInit);
 
       // If we are not auto committing then there should
       // be exactly 2 commits (one per close above):
-      assertEquals(2, policy.numOnCommit);
+      assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit);
 
       // Test listCommits
       Collection<IndexCommit> commits = IndexReader.listCommits(dir);
       // 2 from closing writer
-      assertEquals(2, commits.size());
+      assertEquals(1 + (isOptimized ? 0:1), commits.size());
 
       // Make sure we can open a reader on each commit:
       for (final IndexCommit commit : commits) {
@@ -481,8 +510,10 @@ public class TestDeletionPolicy extends 
           TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
           .setMaxBufferedDocs(10);
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
       IndexWriter writer = new IndexWriter(dir, conf);
       for(int i=0;i<107;i++) {
         addDoc(writer);
@@ -491,8 +522,10 @@ public class TestDeletionPolicy extends 
 
       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
-      lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(true);
+      }
       writer = new IndexWriter(dir, conf);
       writer.optimize();
       writer.close();
@@ -530,8 +563,10 @@ public class TestDeletionPolicy extends 
             TEST_VERSION_CURRENT, new MockAnalyzer(random))
             .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
             .setMaxBufferedDocs(10);
-        LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-        lmp.setUseCompoundFile(useCompoundFile);
+        MergePolicy mp = conf.getMergePolicy();
+        if (mp instanceof LogMergePolicy) {
+          ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+        }
         IndexWriter writer = new IndexWriter(dir, conf);
         for(int i=0;i<17;i++) {
           addDoc(writer);
@@ -587,24 +622,34 @@ public class TestDeletionPolicy extends 
       IndexWriterConfig conf = newIndexWriterConfig(
           TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy);
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
       IndexWriter writer = new IndexWriter(dir, conf);
       writer.close();
       Term searchTerm = new Term("content", "aaa");        
       Query query = new TermQuery(searchTerm);
 
       for(int i=0;i<N+1;i++) {
+        if (VERBOSE) {
+          System.out.println("\nTEST: cycle i=" + i);
+        }
         conf = newIndexWriterConfig(
             TEST_VERSION_CURRENT, new MockAnalyzer(random))
             .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
-        lmp = (LogMergePolicy) conf.getMergePolicy();
-        lmp.setUseCompoundFile(useCompoundFile);
+        mp = conf.getMergePolicy();
+        if (mp instanceof LogMergePolicy) {
+          ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+        }
         writer = new IndexWriter(dir, conf);
         for(int j=0;j<17;j++) {
           addDoc(writer);
         }
         // this is a commit
+        if (VERBOSE) {
+          System.out.println("TEST: close writer");
+        }
         writer.close();
         IndexReader reader = IndexReader.open(dir, policy, false);
         reader.deleteDocument(3*i+1);
@@ -613,20 +658,28 @@ public class TestDeletionPolicy extends 
         ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
         assertEquals(16*(1+i), hits.length);
         // this is a commit
+        if (VERBOSE) {
+          System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
+        }
         reader.close();
         searcher.close();
       }
       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
-      lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
+      IndexReader r = IndexReader.open(dir);
+      final boolean wasOptimized = r.isOptimized();
+      r.close();
       writer = new IndexWriter(dir, conf);
       writer.optimize();
       // this is a commit
       writer.close();
 
       assertEquals(2*(N+1)+1, policy.numOnInit);
-      assertEquals(2*(N+2), policy.numOnCommit);
+      assertEquals(2*(N+2) - (wasOptimized ? 1:0), policy.numOnCommit);
 
       IndexSearcher searcher = new IndexSearcher(dir, false);
       ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
@@ -693,8 +746,10 @@ public class TestDeletionPolicy extends 
           TEST_VERSION_CURRENT, new MockAnalyzer(random))
           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
           .setMaxBufferedDocs(10);
-      LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-      lmp.setUseCompoundFile(useCompoundFile);
+      MergePolicy mp = conf.getMergePolicy();
+      if (mp instanceof LogMergePolicy) {
+        ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+      }
       IndexWriter writer = new IndexWriter(dir, conf);
       writer.close();
       Term searchTerm = new Term("content", "aaa");        
@@ -706,8 +761,10 @@ public class TestDeletionPolicy extends 
             TEST_VERSION_CURRENT, new MockAnalyzer(random))
             .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
             .setMaxBufferedDocs(10);
-        lmp = (LogMergePolicy) conf.getMergePolicy();
-        lmp.setUseCompoundFile(useCompoundFile);
+        mp = conf.getMergePolicy();
+        if (mp instanceof LogMergePolicy) {
+          ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+        }
         writer = new IndexWriter(dir, conf);
         for(int j=0;j<17;j++) {
           addDoc(writer);

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Thu May  5 09:09:41 2011
@@ -107,8 +107,7 @@ public class TestIndexReaderCloneNorms e
     createIndex(random, dir3);
     IndexWriter iw = new IndexWriter(dir3, newIndexWriterConfig(
         TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
-        .setMaxBufferedDocs(5));
-    ((LogMergePolicy) iw.getConfig().getMergePolicy()).setMergeFactor(3);
+                                     .setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
     iw.addIndexes(new Directory[] { dir1, dir2 });
     iw.optimize();
     iw.close();
@@ -125,8 +124,7 @@ public class TestIndexReaderCloneNorms e
 
     // now with optimize
     iw = new IndexWriter(dir3, newIndexWriterConfig( TEST_VERSION_CURRENT,
-        anlzr).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(5));
-    ((LogMergePolicy) iw.getConfig().getMergePolicy()).setMergeFactor(3);
+                                                     anlzr).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
     iw.optimize();
     iw.close();
     verifyIndex(dir3);
@@ -230,7 +228,7 @@ public class TestIndexReaderCloneNorms e
   private void createIndex(Random random, Directory dir) throws IOException {
     IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
-        .setMaxBufferedDocs(5).setSimilarity(similarityOne));
+        .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy()));
     LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(true);
@@ -282,7 +280,7 @@ public class TestIndexReaderCloneNorms e
       throws IOException {
     IndexWriterConfig conf = newIndexWriterConfig(
             TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
-            .setMaxBufferedDocs(5).setSimilarity(similarityOne);
+      .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy());
     LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
     lmp.setMergeFactor(3);
     lmp.setUseCompoundFile(compound);

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu May  5 09:09:41 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
  */
 
 import java.io.ByteArrayOutputStream;
-import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.io.Reader;
@@ -35,7 +34,6 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
 import org.apache.lucene.analysis.TokenStream;
@@ -1345,9 +1343,9 @@ public class TestIndexWriter extends Luc
       }
       IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()));
       writer.setInfoStream(VERBOSE ? System.out : null);
-      LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
-      lmp.setMergeFactor(2);
-      lmp.setUseCompoundFile(false);
+      //LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
+      //lmp.setMergeFactor(2);
+      //lmp.setUseCompoundFile(false);
       Document doc = new Document();
       String contents = "aa bb cc dd ee ff gg hh ii jj kk";
 
@@ -1404,7 +1402,7 @@ public class TestIndexWriter extends Luc
 
       IndexWriterConfig conf = newIndexWriterConfig(
           TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)
-          .setMaxBufferedDocs(2);
+          .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
       if (pass == 2) {
         conf.setMergeScheduler(new SerialMergeScheduler());
       }
@@ -1580,8 +1578,8 @@ public class TestIndexWriter extends Luc
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( 
         TEST_VERSION_CURRENT, new MockAnalyzer(random))
         .setMaxBufferedDocs(2).setRAMBufferSizeMB(
-            IndexWriterConfig.DISABLE_AUTO_FLUSH));
-
+                                                  IndexWriterConfig.DISABLE_AUTO_FLUSH));
+    writer.setInfoStream(VERBOSE ? System.out : null);
     Document document = new Document();
 
     document = new Document();
@@ -2634,11 +2632,6 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  // both start & end are inclusive
-  private final int getInt(Random r, int start, int end) {
-    return start + r.nextInt(1+end-start);
-  }
-
   public void testDeleteUnusedFiles() throws Exception {
 
     for(int iter=0;iter<2;iter++) {
@@ -2827,6 +2820,8 @@ public class TestIndexWriter extends Luc
                                          .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()));
     String[] files = dir.listAll();
 
+    writer.setInfoStream(VERBOSE ? System.out : null);
+
     // Creating over empty dir should not create any files,
     // or, at most the write.lock file
     final int extraFileCount;
@@ -2848,9 +2843,10 @@ public class TestIndexWriter extends Luc
     doc = new Document();
     doc.add(newField("c", "val", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
     writer.addDocument(doc);
+
     // The second document should cause a flush.
-    assertTrue("flush should have occurred and files created", dir.listAll().length > 5 + extraFileCount);
-   
+    assertTrue("flush should have occurred and files should have been created", dir.listAll().length > 5 + extraFileCount);
+
     // After rollback, IW should remove all files
     writer.rollback();
     assertEquals("no files should exist in the directory after rollback", 0, dir.listAll().length);

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu May  5 09:09:41 2011
@@ -286,6 +286,7 @@ public class TestIndexWriterExceptions e
   public void testExceptionDocumentsWriterInit() throws IOException {
     Directory dir = newDirectory();
     MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    w.setInfoStream(VERBOSE ? System.out : null);
     Document doc = new Document();
     doc.add(newField("field", "a field", Field.Store.YES,
                       Field.Index.ANALYZED));
@@ -357,7 +358,7 @@ public class TestIndexWriterExceptions e
   public void testExceptionOnMergeInit() throws IOException {
     Directory dir = newDirectory();
     IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
-      .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler());
+      .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
     ((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
     MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
     w.doFail = true;
@@ -521,7 +522,7 @@ public class TestIndexWriterExceptions e
         System.out.println("TEST: cycle i=" + i);
       }
       MockDirectoryWrapper dir = newDirectory();
-      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
+      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
       writer.setInfoStream(VERBOSE ? System.out : null);
 
       // don't allow a sudden merge to clean up the deleted