You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/06/25 01:07:38 UTC

svn commit: r1139458 [9/24] - in /lucene/dev/branches/solr2193: ./ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/ dev-tools/idea/solr/ dev-tools/idea/solr/contrib/clustering/ dev-tools/maven/ dev-tools/maven/modules/analysis/icu/ dev-t...

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Jun 24 23:07:10 2011
@@ -73,6 +73,7 @@ import org.apache.lucene.util.LuceneTest
 import org.apache.lucene.util.ThreadInterruptedException;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
 
 public class TestIndexWriter extends LuceneTestCase {
 
@@ -133,14 +134,14 @@ public class TestIndexWriter extends Luc
         dir.close();
     }
 
-    private void addDoc(IndexWriter writer) throws IOException
+    static void addDoc(IndexWriter writer) throws IOException
     {
         Document doc = new Document();
         doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
         writer.addDocument(doc);
     }
 
-    private void addDocWithIndex(IndexWriter writer, int index) throws IOException
+    static void addDocWithIndex(IndexWriter writer, int index) throws IOException
     {
         Document doc = new Document();
         doc.add(newField("content", "aaa " + index, Field.Store.YES, Field.Index.ANALYZED));
@@ -165,142 +166,6 @@ public class TestIndexWriter extends Luc
       }
     }
 
-    public void testOptimizeMaxNumSegments() throws IOException {
-
-      MockDirectoryWrapper dir = newDirectory();
-
-      final Document doc = new Document();
-      doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
-      final int incrMin = TEST_NIGHTLY ? 15 : 40;
-      for(int numDocs=10;numDocs<500;numDocs += _TestUtil.nextInt(random, incrMin, 5*incrMin)) {
-        LogDocMergePolicy ldmp = new LogDocMergePolicy();
-        ldmp.setMinMergeDocs(1);
-        ldmp.setMergeFactor(5);
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-          TEST_VERSION_CURRENT, new MockAnalyzer(random))
-          .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(2).setMergePolicy(
-              ldmp));
-        for(int j=0;j<numDocs;j++)
-          writer.addDocument(doc);
-        writer.close();
-
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
-        final int segCount = sis.size();
-
-        ldmp = new LogDocMergePolicy();
-        ldmp.setMergeFactor(5);
-        writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
-          new MockAnalyzer(random)).setMergePolicy(ldmp));
-        writer.optimize(3);
-        writer.close();
-
-        sis = new SegmentInfos();
-        sis.read(dir);
-        final int optSegCount = sis.size();
-
-        if (segCount < 3)
-          assertEquals(segCount, optSegCount);
-        else
-          assertEquals(3, optSegCount);
-      }
-      dir.close();
-    }
-
-    public void testOptimizeMaxNumSegments2() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-
-      final Document doc = new Document();
-      doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED));
-
-      LogDocMergePolicy ldmp = new LogDocMergePolicy();
-      ldmp.setMinMergeDocs(1);
-      ldmp.setMergeFactor(4);
-      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random))
-        .setMaxBufferedDocs(2).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
-
-      for(int iter=0;iter<10;iter++) {
-        for(int i=0;i<19;i++)
-          writer.addDocument(doc);
-
-        writer.commit();
-        writer.waitForMerges();
-        writer.commit();
-
-        SegmentInfos sis = new SegmentInfos();
-        sis.read(dir);
-
-        final int segCount = sis.size();
-
-        writer.optimize(7);
-        writer.commit();
-        writer.waitForMerges();
-
-        sis = new SegmentInfos();
-        sis.read(dir);
-        final int optSegCount = sis.size();
-
-        if (segCount < 7)
-          assertEquals(segCount, optSegCount);
-        else
-          assertEquals(7, optSegCount);
-      }
-      writer.close();
-      dir.close();
-    }
-
-    /**
-     * Make sure optimize doesn't use any more than 1X
-     * starting index size as its temporary free space
-     * required.
-     */
-    public void testOptimizeTempSpaceUsage() throws IOException {
-
-      MockDirectoryWrapper dir = newDirectory();
-      IndexWriter writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10).setMergePolicy(newLogMergePolicy()));
-      if (VERBOSE) {
-        System.out.println("TEST: config1=" + writer.getConfig());
-      }
-
-      for(int j=0;j<500;j++) {
-        addDocWithIndex(writer, j);
-      }
-      final int termIndexInterval = writer.getConfig().getTermIndexInterval();
-      // force one extra segment w/ different doc store so
-      // we see the doc stores get merged
-      writer.commit();
-      addDocWithIndex(writer, 500);
-      writer.close();
-
-      if (VERBOSE) {
-        System.out.println("TEST: start disk usage");
-      }
-      long startDiskUsage = 0;
-      String[] files = dir.listAll();
-      for(int i=0;i<files.length;i++) {
-        startDiskUsage += dir.fileLength(files[i]);
-        if (VERBOSE) {
-          System.out.println(files[i] + ": " + dir.fileLength(files[i]));
-        }
-      }
-
-      dir.resetMaxUsedSizeInBytes();
-      dir.setTrackDiskUsage(true);
-
-      // Import to use same term index interval else a
-      // smaller one here could increase the disk usage and
-      // cause a false failure:
-      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setTermIndexInterval(termIndexInterval).setMergePolicy(newLogMergePolicy()));
-      writer.setInfoStream(VERBOSE ? System.out : null);
-      writer.optimize();
-      writer.close();
-      long maxDiskUsage = dir.getMaxUsedSizeInBytes();
-      assertTrue("optimize used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4*startDiskUsage) + " (= 4X starting usage)",
-                 maxDiskUsage <= 4*startDiskUsage);
-      dir.close();
-    }
-
     static String arrayToString(String[] l) {
       String s = "";
       for(int i=0;i<l.length;i++) {
@@ -361,277 +226,7 @@ public class TestIndexWriter extends Luc
         dir.close();
     }
 
-    /*
-     * Simple test for "commit on close": open writer then
-     * add a bunch of docs, making sure reader does not see
-     * these docs until writer is closed.
-     */
-    public void testCommitOnClose() throws IOException {
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-        for (int i = 0; i < 14; i++) {
-          addDoc(writer);
-        }
-        writer.close();
-
-        Term searchTerm = new Term("content", "aaa");
-        IndexSearcher searcher = new IndexSearcher(dir, false);
-        ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-        assertEquals("first number of hits", 14, hits.length);
-        searcher.close();
-
-        IndexReader reader = IndexReader.open(dir, true);
-
-        writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-        for(int i=0;i<3;i++) {
-          for(int j=0;j<11;j++) {
-            addDoc(writer);
-          }
-          searcher = new IndexSearcher(dir, false);
-          hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-          assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
-          searcher.close();
-          assertTrue("reader should have still been current", reader.isCurrent());
-        }
-
-        // Now, close the writer:
-        writer.close();
-        assertFalse("reader should not be current now", reader.isCurrent());
-
-        searcher = new IndexSearcher(dir, false);
-        hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-        assertEquals("reader did not see changes after writer was closed", 47, hits.length);
-        searcher.close();
-        reader.close();
-        dir.close();
-    }
-
-    /*
-     * Simple test for "commit on close": open writer, then
-     * add a bunch of docs, making sure reader does not see
-     * them until writer has closed.  Then instead of
-     * closing the writer, call abort and verify reader sees
-     * nothing was added.  Then verify we can open the index
-     * and add docs to it.
-     */
-    public void testCommitOnCloseAbort() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10));
-      for (int i = 0; i < 14; i++) {
-        addDoc(writer);
-      }
-      writer.close();
-
-      Term searchTerm = new Term("content", "aaa");
-      IndexSearcher searcher = new IndexSearcher(dir, false);
-      ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-      assertEquals("first number of hits", 14, hits.length);
-      searcher.close();
-
-      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
-        .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
-      for(int j=0;j<17;j++) {
-        addDoc(writer);
-      }
-      // Delete all docs:
-      writer.deleteDocuments(searchTerm);
-
-      searcher = new IndexSearcher(dir, false);
-      hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-      assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
-      searcher.close();
-
-      // Now, close the writer:
-      writer.rollback();
-
-      assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
-
-      searcher = new IndexSearcher(dir, false);
-      hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-      assertEquals("saw changes after writer.abort", 14, hits.length);
-      searcher.close();
-
-      // Now make sure we can re-open the index, add docs,
-      // and all is good:
-      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
-        .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
-
-      // On abort, writer in fact may write to the same
-      // segments_N file:
-      dir.setPreventDoubleWrite(false);
-
-      for(int i=0;i<12;i++) {
-        for(int j=0;j<17;j++) {
-          addDoc(writer);
-        }
-        searcher = new IndexSearcher(dir, false);
-        hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-        assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
-        searcher.close();
-      }
-
-      writer.close();
-      searcher = new IndexSearcher(dir, false);
-      hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
-      assertEquals("didn't see changes after close", 218, hits.length);
-      searcher.close();
-
-      dir.close();
-    }
-
-    /*
-     * Verify that a writer with "commit on close" indeed
-     * cleans up the temp segments created after opening
-     * that are not referenced by the starting segments
-     * file.  We check this by using MockDirectoryWrapper to
-     * measure max temp disk space used.
-     */
-    public void testCommitOnCloseDiskUsage() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      Analyzer analyzer;
-      if (random.nextBoolean()) {
-        // no payloads
-       analyzer = new Analyzer() {
-          @Override
-          public TokenStream tokenStream(String fieldName, Reader reader) {
-            return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
-          }
-        };
-      } else {
-        // fixed length payloads
-        final int length = random.nextInt(200);
-        analyzer = new Analyzer() {
-          @Override
-          public TokenStream tokenStream(String fieldName, Reader reader) {
-            return new MockFixedLengthPayloadFilter(random,
-                new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
-                length);
-          }
-        };
-      }
-      
-      IndexWriter writer  = new IndexWriter(
-          dir,
-          newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).
-              setMaxBufferedDocs(10).
-              setReaderPooling(false).
-              setMergePolicy(newLogMergePolicy(10))
-      );
-      for(int j=0;j<30;j++) {
-        addDocWithIndex(writer, j);
-      }
-      writer.close();
-      dir.resetMaxUsedSizeInBytes();
-
-      dir.setTrackDiskUsage(true);
-      long startDiskUsage = dir.getMaxUsedSizeInBytes();
-      writer = new IndexWriter(
-          dir,
-          newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
-              .setOpenMode(OpenMode.APPEND).
-              setMaxBufferedDocs(10).
-              setMergeScheduler(new SerialMergeScheduler()).
-              setReaderPooling(false).
-              setMergePolicy(newLogMergePolicy(10))
-
-      );
-      for(int j=0;j<1470;j++) {
-        addDocWithIndex(writer, j);
-      }
-      long midDiskUsage = dir.getMaxUsedSizeInBytes();
-      dir.resetMaxUsedSizeInBytes();
-      writer.optimize();
-      writer.close();
-
-      IndexReader.open(dir, true).close();
-
-      long endDiskUsage = dir.getMaxUsedSizeInBytes();
-
-      // Ending index is 50X as large as starting index; due
-      // to 3X disk usage normally we allow 150X max
-      // transient usage.  If something is wrong w/ deleter
-      // and it doesn't delete intermediate segments then it
-      // will exceed this 150X:
-      // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
-      assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
-                 midDiskUsage < 150*startDiskUsage);
-      assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
-                 endDiskUsage < 150*startDiskUsage);
-      dir.close();
-    }
-
-
-    /*
-     * Verify that calling optimize when writer is open for
-     * "commit on close" works correctly both for rollback()
-     * and close().
-     */
-    public void testCommitOnCloseOptimize() throws IOException {
-      MockDirectoryWrapper dir = newDirectory();
-      // Must disable throwing exc on double-write: this
-      // test uses IW.rollback which easily results in
-      // writing to same file more than once
-      dir.setPreventDoubleWrite(false);
-      IndexWriter writer = new IndexWriter(
-          dir,
-          newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-              setMaxBufferedDocs(10).
-              setMergePolicy(newLogMergePolicy(10))
-      );
-      for(int j=0;j<17;j++) {
-        addDocWithIndex(writer, j);
-      }
-      writer.close();
-
-      writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
-      writer.optimize();
-
-      if (VERBOSE) {
-        writer.setInfoStream(System.out);
-      }
-
-      // Open a reader before closing (commiting) the writer:
-      IndexReader reader = IndexReader.open(dir, true);
-
-      // Reader should see index as unoptimized at this
-      // point:
-      assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
-      reader.close();
-
-      // Abort the writer:
-      writer.rollback();
-      assertNoUnreferencedFiles(dir, "aborted writer after optimize");
-
-      // Open a reader after aborting writer:
-      reader = IndexReader.open(dir, true);
-
-      // Reader should still see index as unoptimized:
-      assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
-      reader.close();
-
-      if (VERBOSE) {
-        System.out.println("TEST: do real optimize");
-      }
-      writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
-      if (VERBOSE) {
-        writer.setInfoStream(System.out);
-      }
-      writer.optimize();
-      writer.close();
-
-      if (VERBOSE) {
-        System.out.println("TEST: writer closed");
-      }
-      assertNoUnreferencedFiles(dir, "aborted writer after optimize");
-
-      // Open a reader after aborting writer:
-      reader = IndexReader.open(dir, true);
 
-      // Reader should still see index as unoptimized:
-      assertTrue("Reader incorrectly sees that the index is unoptimized", reader.isOptimized());
-      reader.close();
-      dir.close();
-    }
 
     public void testIndexNoDocuments() throws IOException {
       MockDirectoryWrapper dir = newDirectory();
@@ -1027,57 +622,13 @@ public class TestIndexWriter extends Luc
       }
       writer.addDocument(new Document());
       writer.close();
-      _TestUtil.checkIndex(dir);
       IndexReader reader = IndexReader.open(dir, true);
       assertEquals(2, reader.numDocs());
       reader.close();
       dir.close();
     }
 
-    // Test calling optimize(false) whereby optimize is kicked
-    // off but we don't wait for it to finish (but
-    // writer.close()) does wait
-    public void testBackgroundOptimize() throws IOException {
-
-      Directory dir = newDirectory();
-      for(int pass=0;pass<2;pass++) {
-        IndexWriter writer = new IndexWriter(
-            dir,
-            newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-                setOpenMode(OpenMode.CREATE).
-                setMaxBufferedDocs(2).
-                setMergePolicy(newLogMergePolicy(51))
-        );
-        Document doc = new Document();
-        doc.add(newField("field", "aaa", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
-        for(int i=0;i<100;i++)
-          writer.addDocument(doc);
-        writer.optimize(false);
-
-        if (0 == pass) {
-          writer.close();
-          IndexReader reader = IndexReader.open(dir, true);
-          assertTrue(reader.isOptimized());
-          reader.close();
-        } else {
-          // Get another segment to flush so we can verify it is
-          // NOT included in the optimization
-          writer.addDocument(doc);
-          writer.addDocument(doc);
-          writer.close();
-
-          IndexReader reader = IndexReader.open(dir, true);
-          assertTrue(!reader.isOptimized());
-          reader.close();
-
-          SegmentInfos infos = new SegmentInfos();
-          infos.read(dir);
-          assertEquals(2, infos.size());
-        }
-      }
 
-      dir.close();
-    }
 
   /**
    * Test that no NullPointerException will be raised,
@@ -1121,51 +672,6 @@ public class TestIndexWriter extends Luc
     }
   }
 
-  // Just intercepts all merges & verifies that we are never
-  // merging a segment with >= 20 (maxMergeDocs) docs
-  private class MyMergeScheduler extends MergeScheduler {
-    @Override
-    synchronized public void merge(IndexWriter writer)
-      throws CorruptIndexException, IOException {
-
-      while(true) {
-        MergePolicy.OneMerge merge = writer.getNextMerge();
-        if (merge == null) {
-          break;
-        }
-        for(int i=0;i<merge.segments.size();i++) {
-          assert merge.segments.get(i).docCount < 20;
-        }
-        writer.merge(merge);
-      }
-    }
-
-    @Override
-    public void close() {}
-  }
-
-  // LUCENE-1013
-  public void testSetMaxMergeDocs() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriterConfig conf = newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random))
-      .setMergeScheduler(new MyMergeScheduler()).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
-    LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
-    lmp.setMaxMergeDocs(20);
-    lmp.setMergeFactor(2);
-    IndexWriter iw = new IndexWriter(dir, conf);
-    iw.setInfoStream(VERBOSE ? System.out : null);
-    Document document = new Document();
-    document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
-                           Field.TermVector.YES));
-    for(int i=0;i<177;i++)
-      iw.addDocument(document);
-    iw.close();
-    dir.close();
-  }
-
-
-
   public void testVariableSchema() throws Exception {
     Directory dir = newDirectory();
     int delID = 0;
@@ -1339,184 +845,7 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  // LUCENE-1044: test writer.commit() when ac=false
-  public void testForceCommit() throws IOException {
-    Directory dir = newDirectory();
-
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setMergePolicy(newLogMergePolicy(5))
-    );
-    writer.commit();
-
-    for (int i = 0; i < 23; i++)
-      addDoc(writer);
-
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-    writer.commit();
-    IndexReader reader2 = reader.reopen();
-    assertEquals(0, reader.numDocs());
-    assertEquals(23, reader2.numDocs());
-    reader.close();
-
-    for (int i = 0; i < 17; i++)
-      addDoc(writer);
-    assertEquals(23, reader2.numDocs());
-    reader2.close();
-    reader = IndexReader.open(dir, true);
-    assertEquals(23, reader.numDocs());
-    reader.close();
-    writer.commit();
 
-    reader = IndexReader.open(dir, true);
-    assertEquals(40, reader.numDocs());
-    reader.close();
-    writer.close();
-    dir.close();
-  }
-
-  // LUCENE-325: test expungeDeletes, when 2 singular merges
-  // are required
-  public void testExpungeDeletes() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new MockAnalyzer(random))
-        .setMaxBufferedDocs(2).setRAMBufferSizeMB(
-                                                  IndexWriterConfig.DISABLE_AUTO_FLUSH));
-    writer.setInfoStream(VERBOSE ? System.out : null);
-    Document document = new Document();
-
-    document = new Document();
-    Field storedField = newField("stored", "stored", Field.Store.YES,
-                                  Field.Index.NO);
-    document.add(storedField);
-    Field termVectorField = newField("termVector", "termVector",
-                                      Field.Store.NO, Field.Index.NOT_ANALYZED,
-                                      Field.TermVector.WITH_POSITIONS_OFFSETS);
-    document.add(termVectorField);
-    for(int i=0;i<10;i++)
-      writer.addDocument(document);
-    writer.close();
-
-    IndexReader ir = IndexReader.open(dir, false);
-    assertEquals(10, ir.maxDoc());
-    assertEquals(10, ir.numDocs());
-    ir.deleteDocument(0);
-    ir.deleteDocument(7);
-    assertEquals(8, ir.numDocs());
-    ir.close();
-
-    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
-    assertEquals(8, writer.numDocs());
-    assertEquals(10, writer.maxDoc());
-    writer.expungeDeletes();
-    assertEquals(8, writer.numDocs());
-    writer.close();
-    ir = IndexReader.open(dir, true);
-    assertEquals(8, ir.maxDoc());
-    assertEquals(8, ir.numDocs());
-    ir.close();
-    dir.close();
-  }
-
-  // LUCENE-325: test expungeDeletes, when many adjacent merges are required
-  public void testExpungeDeletes2() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
-            setMergePolicy(newLogMergePolicy(50))
-    );
-
-    Document document = new Document();
-
-    document = new Document();
-    Field storedField = newField("stored", "stored", Store.YES,
-                                  Index.NO);
-    document.add(storedField);
-    Field termVectorField = newField("termVector", "termVector",
-                                      Store.NO, Index.NOT_ANALYZED,
-                                      TermVector.WITH_POSITIONS_OFFSETS);
-    document.add(termVectorField);
-    for(int i=0;i<98;i++)
-      writer.addDocument(document);
-    writer.close();
-
-    IndexReader ir = IndexReader.open(dir, false);
-    assertEquals(98, ir.maxDoc());
-    assertEquals(98, ir.numDocs());
-    for(int i=0;i<98;i+=2)
-      ir.deleteDocument(i);
-    assertEquals(49, ir.numDocs());
-    ir.close();
-
-    writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMergePolicy(newLogMergePolicy(3))
-    );
-    assertEquals(49, writer.numDocs());
-    writer.expungeDeletes();
-    writer.close();
-    ir = IndexReader.open(dir, true);
-    assertEquals(49, ir.maxDoc());
-    assertEquals(49, ir.numDocs());
-    ir.close();
-    dir.close();
-  }
-
-  // LUCENE-325: test expungeDeletes without waiting, when
-  // many adjacent merges are required
-  public void testExpungeDeletes3() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
-            setMergePolicy(newLogMergePolicy(50))
-    );
-
-    Document document = new Document();
-
-    document = new Document();
-    Field storedField = newField("stored", "stored", Field.Store.YES,
-                                  Field.Index.NO);
-    document.add(storedField);
-    Field termVectorField = newField("termVector", "termVector",
-                                      Field.Store.NO, Field.Index.NOT_ANALYZED,
-                                      Field.TermVector.WITH_POSITIONS_OFFSETS);
-    document.add(termVectorField);
-    for(int i=0;i<98;i++)
-      writer.addDocument(document);
-    writer.close();
-
-    IndexReader ir = IndexReader.open(dir, false);
-    assertEquals(98, ir.maxDoc());
-    assertEquals(98, ir.numDocs());
-    for(int i=0;i<98;i+=2)
-      ir.deleteDocument(i);
-    assertEquals(49, ir.numDocs());
-    ir.close();
-
-    writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMergePolicy(newLogMergePolicy(3))
-    );
-    writer.expungeDeletes(false);
-    writer.close();
-    ir = IndexReader.open(dir, true);
-    assertEquals(49, ir.maxDoc());
-    assertEquals(49, ir.numDocs());
-    ir.close();
-    dir.close();
-  }
 
   // LUCENE-1179
   public void testEmptyFieldName() throws IOException {
@@ -1554,193 +883,28 @@ public class TestIndexWriter extends Luc
 
   // LUCENE-1222
   public void testDoBeforeAfterFlush() throws IOException {
-    Directory dir = newDirectory();
-    MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    Document doc = new Document();
-    doc.add(newField("field", "a field", Field.Store.YES,
-                      Field.Index.ANALYZED));
-    w.addDocument(doc);
-    w.commit();
-    assertTrue(w.beforeWasCalled);
-    assertTrue(w.afterWasCalled);
-    w.beforeWasCalled = false;
-    w.afterWasCalled = false;
-    w.deleteDocuments(new Term("field", "field"));
-    w.commit();
-    assertTrue(w.beforeWasCalled);
-    assertTrue(w.afterWasCalled);
-    w.close();
-
-    IndexReader ir = IndexReader.open(dir, true);
-    assertEquals(0, ir.numDocs());
-    ir.close();
-
-    dir.close();
-  }
-
-
-
-  final String[] utf8Data = new String[] {
-    // unpaired low surrogate
-    "ab\udc17cd", "ab\ufffdcd",
-    "\udc17abcd", "\ufffdabcd",
-    "\udc17", "\ufffd",
-    "ab\udc17\udc17cd", "ab\ufffd\ufffdcd",
-    "\udc17\udc17abcd", "\ufffd\ufffdabcd",
-    "\udc17\udc17", "\ufffd\ufffd",
-
-    // unpaired high surrogate
-    "ab\ud917cd", "ab\ufffdcd",
-    "\ud917abcd", "\ufffdabcd",
-    "\ud917", "\ufffd",
-    "ab\ud917\ud917cd", "ab\ufffd\ufffdcd",
-    "\ud917\ud917abcd", "\ufffd\ufffdabcd",
-    "\ud917\ud917", "\ufffd\ufffd",
-
-    // backwards surrogates
-    "ab\udc17\ud917cd", "ab\ufffd\ufffdcd",
-    "\udc17\ud917abcd", "\ufffd\ufffdabcd",
-    "\udc17\ud917", "\ufffd\ufffd",
-    "ab\udc17\ud917\udc17\ud917cd", "ab\ufffd\ud917\udc17\ufffdcd",
-    "\udc17\ud917\udc17\ud917abcd", "\ufffd\ud917\udc17\ufffdabcd",
-    "\udc17\ud917\udc17\ud917", "\ufffd\ud917\udc17\ufffd"
-  };
-
-  // LUCENE-510
-  public void testInvalidUTF16() throws Throwable {
-    Directory dir = newDirectory();
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new StringSplitAnalyzer()));
-    Document doc = new Document();
-
-    final int count = utf8Data.length/2;
-    for(int i=0;i<count;i++)
-      doc.add(newField("f" + i, utf8Data[2*i], Field.Store.YES, Field.Index.ANALYZED));
-    w.addDocument(doc);
-    w.close();
-
-    IndexReader ir = IndexReader.open(dir, true);
-    Document doc2 = ir.document(0);
-    for(int i=0;i<count;i++) {
-      assertEquals("field " + i + " was not indexed correctly", 1, ir.docFreq(new Term("f"+i, utf8Data[2*i+1])));
-      assertEquals("field " + i + " is incorrect", utf8Data[2*i+1], doc2.getField("f"+i).stringValue());
-    }
-    ir.close();
-    dir.close();
-  }
-
-  // LUCENE-510
-  public void testAllUnicodeChars() throws Throwable {
-
-    BytesRef utf8 = new BytesRef(10);
-    CharsRef utf16 = new CharsRef(10);
-    char[] chars = new char[2];
-    for(int ch=0;ch<0x0010FFFF;ch++) {
-
-      if (ch == 0xd800)
-        // Skip invalid code points
-        ch = 0xe000;
-
-      int len = 0;
-      if (ch <= 0xffff) {
-        chars[len++] = (char) ch;
-      } else {
-        chars[len++] = (char) (((ch-0x0010000) >> 10) + UnicodeUtil.UNI_SUR_HIGH_START);
-        chars[len++] = (char) (((ch-0x0010000) & 0x3FFL) + UnicodeUtil.UNI_SUR_LOW_START);
-      }
-
-      UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8);
-
-      String s1 = new String(chars, 0, len);
-      String s2 = new String(utf8.bytes, 0, utf8.length, "UTF-8");
-      assertEquals("codepoint " + ch, s1, s2);
-
-      UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
-      assertEquals("codepoint " + ch, s1, new String(utf16.chars, 0, utf16.length));
-
-      byte[] b = s1.getBytes("UTF-8");
-      assertEquals(utf8.length, b.length);
-      for(int j=0;j<utf8.length;j++)
-        assertEquals(utf8.bytes[j], b[j]);
-    }
-  }
-
-  private int nextInt(int lim) {
-    return random.nextInt(lim);
-  }
-
-  private int nextInt(int start, int end) {
-    return start + nextInt(end-start);
-  }
-
-  private boolean fillUnicode(char[] buffer, char[] expected, int offset, int count) {
-    final int len = offset + count;
-    boolean hasIllegal = false;
-
-    if (offset > 0 && buffer[offset] >= 0xdc00 && buffer[offset] < 0xe000)
-      // Don't start in the middle of a valid surrogate pair
-      offset--;
-
-    for(int i=offset;i<len;i++) {
-      int t = nextInt(6);
-      if (0 == t && i < len-1) {
-        // Make a surrogate pair
-        // High surrogate
-        expected[i] = buffer[i++] = (char) nextInt(0xd800, 0xdc00);
-        // Low surrogate
-        expected[i] = buffer[i] = (char) nextInt(0xdc00, 0xe000);
-      } else if (t <= 1)
-        expected[i] = buffer[i] = (char) nextInt(0x80);
-      else if (2 == t)
-        expected[i] = buffer[i] = (char) nextInt(0x80, 0x800);
-      else if (3 == t)
-        expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
-      else if (4 == t)
-        expected[i] = buffer[i] = (char) nextInt(0xe000, 0xffff);
-      else if (5 == t && i < len-1) {
-        // Illegal unpaired surrogate
-        if (nextInt(10) == 7) {
-          if (random.nextBoolean())
-            buffer[i] = (char) nextInt(0xd800, 0xdc00);
-          else
-            buffer[i] = (char) nextInt(0xdc00, 0xe000);
-          expected[i++] = 0xfffd;
-          expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
-          hasIllegal = true;
-        } else
-          expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
-      } else {
-        expected[i] = buffer[i] = ' ';
-      }
-    }
-
-    return hasIllegal;
-  }
-
-  // LUCENE-510
-  public void testRandomUnicodeStrings() throws Throwable {
-    char[] buffer = new char[20];
-    char[] expected = new char[20];
-
-    BytesRef utf8 = new BytesRef(20);
-    CharsRef utf16 = new CharsRef(20);
-
-    int num = 100000 * RANDOM_MULTIPLIER;
-    for (int iter = 0; iter < num; iter++) {
-      boolean hasIllegal = fillUnicode(buffer, expected, 0, 20);
+    Directory dir = newDirectory();
+    MockIndexWriter w = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+    Document doc = new Document();
+    doc.add(newField("field", "a field", Field.Store.YES,
+                      Field.Index.ANALYZED));
+    w.addDocument(doc);
+    w.commit();
+    assertTrue(w.beforeWasCalled);
+    assertTrue(w.afterWasCalled);
+    w.beforeWasCalled = false;
+    w.afterWasCalled = false;
+    w.deleteDocuments(new Term("field", "field"));
+    w.commit();
+    assertTrue(w.beforeWasCalled);
+    assertTrue(w.afterWasCalled);
+    w.close();
 
-      UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
-      if (!hasIllegal) {
-        byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
-        assertEquals(b.length, utf8.length);
-        for(int i=0;i<b.length;i++)
-          assertEquals(b[i], utf8.bytes[i]);
-      }
+    IndexReader ir = IndexReader.open(dir, true);
+    assertEquals(0, ir.numDocs());
+    ir.close();
 
-      UnicodeUtil.UTF8toUTF16(utf8.bytes, 0, utf8.length, utf16);
-      assertEquals(utf16.length, 20);
-      for(int i=0;i<20;i++)
-        assertEquals(expected[i], utf16.chars[i]);
-    }
+    dir.close();
   }
 
   // LUCENE-1255
@@ -1792,137 +956,11 @@ public class TestIndexWriter extends Luc
     assertEquals(0, tps.nextPosition());
     w.close();
 
-    _TestUtil.checkIndex(dir);
     s.close();
     dir.close();
   }
 
-  // LUCENE-1274: test writer.prepareCommit()
-  public void testPrepareCommit() throws IOException {
-    Directory dir = newDirectory();
-
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setMergePolicy(newLogMergePolicy(5))
-    );
-    writer.commit();
-
-    for (int i = 0; i < 23; i++)
-      addDoc(writer);
-
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-
-    writer.prepareCommit();
-
-    IndexReader reader2 = IndexReader.open(dir, true);
-    assertEquals(0, reader2.numDocs());
-
-    writer.commit();
-
-    IndexReader reader3 = reader.reopen();
-    assertEquals(0, reader.numDocs());
-    assertEquals(0, reader2.numDocs());
-    assertEquals(23, reader3.numDocs());
-    reader.close();
-    reader2.close();
-
-    for (int i = 0; i < 17; i++)
-      addDoc(writer);
-
-    assertEquals(23, reader3.numDocs());
-    reader3.close();
-    reader = IndexReader.open(dir, true);
-    assertEquals(23, reader.numDocs());
-    reader.close();
-
-    writer.prepareCommit();
-
-    reader = IndexReader.open(dir, true);
-    assertEquals(23, reader.numDocs());
-    reader.close();
-
-    writer.commit();
-    reader = IndexReader.open(dir, true);
-    assertEquals(40, reader.numDocs());
-    reader.close();
-    writer.close();
-    dir.close();
-  }
-
-  // LUCENE-1274: test writer.prepareCommit()
-  public void testPrepareCommitRollback() throws IOException {
-    MockDirectoryWrapper dir = newDirectory();
-    dir.setPreventDoubleWrite(false);
-
-    IndexWriter writer = new IndexWriter(
-        dir,
-        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
-            setMaxBufferedDocs(2).
-            setMergePolicy(newLogMergePolicy(5))
-    );
-    writer.commit();
-
-    for (int i = 0; i < 23; i++)
-      addDoc(writer);
-
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-
-    writer.prepareCommit();
-
-    IndexReader reader2 = IndexReader.open(dir, true);
-    assertEquals(0, reader2.numDocs());
-
-    writer.rollback();
-
-    IndexReader reader3 = reader.reopen();
-    assertEquals(0, reader.numDocs());
-    assertEquals(0, reader2.numDocs());
-    assertEquals(0, reader3.numDocs());
-    reader.close();
-    reader2.close();
-
-    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    for (int i = 0; i < 17; i++)
-      addDoc(writer);
-
-    assertEquals(0, reader3.numDocs());
-    reader3.close();
-    reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-    reader.close();
-
-    writer.prepareCommit();
-
-    reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-    reader.close();
-
-    writer.commit();
-    reader = IndexReader.open(dir, true);
-    assertEquals(17, reader.numDocs());
-    reader.close();
-    writer.close();
-    dir.close();
-  }
-
-  // LUCENE-1274
-  public void testPrepareCommitNoChanges() throws IOException {
-    Directory dir = newDirectory();
-
-    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    writer.prepareCommit();
-    writer.commit();
-    writer.close();
 
-    IndexReader reader = IndexReader.open(dir, true);
-    assertEquals(0, reader.numDocs());
-    reader.close();
-    dir.close();
-  }
 
   // LUCENE-1219
   public void testBinaryFieldOffsetLength() throws IOException {
@@ -1954,45 +992,6 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  // LUCENE-1382
-  public void testCommitUserData() throws IOException {
-    Directory dir = newDirectory();
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
-    for(int j=0;j<17;j++)
-      addDoc(w);
-    w.close();
-
-    assertEquals(0, IndexReader.getCommitUserData(dir).size());
-
-    IndexReader r = IndexReader.open(dir, true);
-    // commit(Map) never called for this index
-    assertEquals(0, r.getCommitUserData().size());
-    r.close();
-
-    w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2));
-    for(int j=0;j<17;j++)
-      addDoc(w);
-    Map<String,String> data = new HashMap<String,String>();
-    data.put("label", "test1");
-    w.commit(data);
-    w.close();
-
-    assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
-
-    r = IndexReader.open(dir, true);
-    assertEquals("test1", r.getCommitUserData().get("label"));
-    r.close();
-
-    w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    w.optimize();
-    w.close();
-
-    assertEquals("test1", IndexReader.getCommitUserData(dir).get("label"));
-
-    dir.close();
-  }
-
-
   // LUCENE-2529
   public void testPositionIncrementGapEmptyField() throws Exception {
     Directory dir = newDirectory();
@@ -2299,24 +1298,6 @@ public class TestIndexWriter extends Luc
     d.close();
   }
 
-  public void testEmbeddedFFFF() throws Throwable {
-
-    Directory d = newDirectory();
-    IndexWriter w = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    Document doc = new Document();
-    doc.add(newField("field", "a a\uffffb", Field.Store.NO, Field.Index.ANALYZED));
-    w.addDocument(doc);
-    doc = new Document();
-    doc.add(newField("field", "a", Field.Store.NO, Field.Index.ANALYZED));
-    w.addDocument(doc);
-    IndexReader r = w.getReader();
-    assertEquals(1, r.docFreq(new Term("field", "a\uffffb")));
-    r.close();
-    w.close();
-    _TestUtil.checkIndex(d);
-    d.close();
-  }
-
   public void testNoDocsIndex() throws Throwable {
     Directory dir = newDirectory();
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
@@ -2326,185 +1307,6 @@ public class TestIndexWriter extends Luc
     writer.addDocument(new Document());
     writer.close();
 
-    _TestUtil.checkIndex(dir);
-    dir.close();
-  }
-
-  // LUCENE-2095: make sure with multiple threads commit
-  // doesn't return until all changes are in fact in the
-  // index
-  public void testCommitThreadSafety() throws Throwable {
-    final int NUM_THREADS = 5;
-    final double RUN_SEC = 0.5;
-    final Directory dir = newDirectory();
-    final RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig(
-                                                                                        TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
-    _TestUtil.reduceOpenFiles(w.w);
-    w.commit();
-    final AtomicBoolean failed = new AtomicBoolean();
-    Thread[] threads = new Thread[NUM_THREADS];
-    final long endTime = System.currentTimeMillis()+((long) (RUN_SEC*1000));
-    for(int i=0;i<NUM_THREADS;i++) {
-      final int finalI = i;
-      threads[i] = new Thread() {
-          @Override
-          public void run() {
-            try {
-              final Document doc = new Document();
-              IndexReader r = IndexReader.open(dir);
-              Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
-              doc.add(f);
-              int count = 0;
-              do {
-                if (failed.get()) break;
-                for(int j=0;j<10;j++) {
-                  final String s = finalI + "_" + String.valueOf(count++);
-                  f.setValue(s);
-                  w.addDocument(doc);
-                  w.commit();
-                  IndexReader r2 = r.reopen();
-                  assertTrue(r2 != r);
-                  r.close();
-                  r = r2;
-                  assertEquals("term=f:" + s + "; r=" + r, 1, r.docFreq(new Term("f", s)));
-                }
-              } while(System.currentTimeMillis() < endTime);
-              r.close();
-            } catch (Throwable t) {
-              failed.set(true);
-              throw new RuntimeException(t);
-            }
-          }
-        };
-      threads[i].start();
-    }
-    for(int i=0;i<NUM_THREADS;i++) {
-      threads[i].join();
-    }
-    assertFalse(failed.get());
-    w.close();
-    dir.close();
-  }
-
-  // both start & end are inclusive
-  private final int getInt(Random r, int start, int end) {
-    return start + r.nextInt(1+end-start);
-  }
-
-  private void checkTermsOrder(IndexReader r, Set<String> allTerms, boolean isTop) throws IOException {
-    TermsEnum terms = MultiFields.getFields(r).terms("f").iterator();
-
-    BytesRef last = new BytesRef();
-
-    Set<String> seenTerms = new HashSet<String>();
-
-    while(true) {
-      final BytesRef term = terms.next();
-      if (term == null) {
-        break;
-      }
-
-      assertTrue(last.compareTo(term) < 0);
-      last.copy(term);
-
-      final String s = term.utf8ToString();
-      assertTrue("term " + termDesc(s) + " was not added to index (count=" + allTerms.size() + ")", allTerms.contains(s));
-      seenTerms.add(s);
-    }
-
-    if (isTop) {
-      assertTrue(allTerms.equals(seenTerms));
-    }
-
-    // Test seeking:
-    Iterator<String> it = seenTerms.iterator();
-    while(it.hasNext()) {
-      BytesRef tr = new BytesRef(it.next());
-      assertEquals("seek failed for term=" + termDesc(tr.utf8ToString()),
-                   TermsEnum.SeekStatus.FOUND,
-                   terms.seek(tr));
-    }
-  }
-
-  private final String asUnicodeChar(char c) {
-    return "U+" + Integer.toHexString(c);
-  }
-
-  private final String termDesc(String s) {
-    final String s0;
-    assertTrue(s.length() <= 2);
-    if (s.length() == 1) {
-      s0 = asUnicodeChar(s.charAt(0));
-    } else {
-      s0 = asUnicodeChar(s.charAt(0)) + "," + asUnicodeChar(s.charAt(1));
-    }
-    return s0;
-  }
-
-  // Make sure terms, including ones with surrogate pairs,
-  // sort in codepoint sort order by default
-  public void testTermUTF16SortOrder() throws Throwable {
-    Random rnd = random;
-    Directory dir = newDirectory();
-    RandomIndexWriter writer = new RandomIndexWriter(rnd, dir);
-    Document d = new Document();
-    // Single segment
-    Field f = newField("f", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
-    d.add(f);
-    char[] chars = new char[2];
-    final Set<String> allTerms = new HashSet<String>();
-
-    int num = 200 * RANDOM_MULTIPLIER;
-    for (int i = 0; i < num; i++) {
-
-      final String s;
-      if (rnd.nextBoolean()) {
-        // Single char
-        if (rnd.nextBoolean()) {
-          // Above surrogates
-          chars[0] = (char) getInt(rnd, 1+UnicodeUtil.UNI_SUR_LOW_END, 0xffff);
-        } else {
-          // Below surrogates
-          chars[0] = (char) getInt(rnd, 0, UnicodeUtil.UNI_SUR_HIGH_START-1);
-        }
-        s = new String(chars, 0, 1);
-      } else {
-        // Surrogate pair
-        chars[0] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_HIGH_START, UnicodeUtil.UNI_SUR_HIGH_END);
-        assertTrue(((int) chars[0]) >= UnicodeUtil.UNI_SUR_HIGH_START && ((int) chars[0]) <= UnicodeUtil.UNI_SUR_HIGH_END);
-        chars[1] = (char) getInt(rnd, UnicodeUtil.UNI_SUR_LOW_START, UnicodeUtil.UNI_SUR_LOW_END);
-        s = new String(chars, 0, 2);
-      }
-      allTerms.add(s);
-      f.setValue(s);
-
-      writer.addDocument(d);
-
-      if ((1+i) % 42 == 0) {
-        writer.commit();
-      }
-    }
-
-    IndexReader r = writer.getReader();
-
-    // Test each sub-segment
-    final IndexReader[] subs = r.getSequentialSubReaders();
-    for(int i=0;i<subs.length;i++) {
-      checkTermsOrder(subs[i], allTerms, false);
-    }
-    checkTermsOrder(r, allTerms, true);
-
-    // Test multi segment
-    r.close();
-
-    writer.optimize();
-
-    // Test optimized single segment
-    r = writer.getReader();
-    checkTermsOrder(r, allTerms, true);
-    r.close();
-
-    writer.close();
     dir.close();
   }
 
@@ -2573,6 +1375,11 @@ public class TestIndexWriter extends Luc
       IndexReader r2 = r.reopen();
       assertTrue(r != r2);
       files = Arrays.asList(dir.listAll());
+
+      // NOTE: here we rely on "Windows" behavior, ie, even
+      // though IW wanted to delete _0.cfs since it was
+      // optimized away, because we have a reader open
+      // against this file, it should still be here:
       assertTrue(files.contains("_0.cfs"));
       // optimize created this
       //assertTrue(files.contains("_2.cfs"));
@@ -2638,67 +1445,6 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  public void testIndexingThenDeleting() throws Exception {
-    final Random r = random;
-    Directory dir = newDirectory();
-    // note this test explicitly disables payloads
-    final Analyzer analyzer = new Analyzer() {
-      @Override
-      public TokenStream tokenStream(String fieldName, Reader reader) {
-        return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
-      }
-    };
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
-    w.setInfoStream(VERBOSE ? System.out : null);
-    Document doc = new Document();
-    doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
-    int num = TEST_NIGHTLY ? 6 * RANDOM_MULTIPLIER : 3 * RANDOM_MULTIPLIER;
-    for (int iter = 0; iter < num; iter++) {
-      int count = 0;
-
-      final boolean doIndexing = r.nextBoolean();
-      if (VERBOSE) {
-        System.out.println("TEST: iter doIndexing=" + doIndexing);
-      }
-      if (doIndexing) {
-        // Add docs until a flush is triggered
-        final int startFlushCount = w.getFlushCount();
-        while(w.getFlushCount() == startFlushCount) {
-          w.addDocument(doc);
-          count++;
-        }
-      } else {
-        // Delete docs until a flush is triggered
-        final int startFlushCount = w.getFlushCount();
-        while(w.getFlushCount() == startFlushCount) {
-          w.deleteDocuments(new Term("foo", ""+count));
-          count++;
-        }
-      }
-      assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 3000);
-    }
-    w.close();
-    dir.close();
-  }
-
-  public void testNoCommits() throws Exception {
-    // Tests that if we don't call commit(), the directory has 0 commits. This has
-    // changed since LUCENE-2386, where before IW would always commit on a fresh
-    // new index.
-    Directory dir = newDirectory();
-    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
-    try {
-      IndexReader.listCommits(dir);
-      fail("listCommits should have thrown an exception over empty index");
-    } catch (IndexNotFoundException e) {
-      // that's expected !
-    }
-    // No changes still should generate a commit, because it's a new index.
-    writer.close();
-    assertEquals("expected 1 commits!", 1, IndexReader.listCommits(dir).size());
-    dir.close();
-  }
-
   public void testEmptyFSDirWithNoLock() throws Exception {
     // Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF),
     // then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed
@@ -2775,82 +1521,6 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  public void testFutureCommit() throws Exception {
-    Directory dir = newDirectory();
-
-    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
-    Document doc = new Document();
-    w.addDocument(doc);
-
-    // commit to "first"
-    Map<String,String> commitData = new HashMap<String,String>();
-    commitData.put("tag", "first");
-    w.commit(commitData);
-
-    // commit to "second"
-    w.addDocument(doc);
-    commitData.put("tag", "second");
-    w.commit(commitData);
-    w.close();
-
-    // open "first" with IndexWriter
-    IndexCommit commit = null;
-    for(IndexCommit c : IndexReader.listCommits(dir)) {
-      if (c.getUserData().get("tag").equals("first")) {
-        commit = c;
-        break;
-      }
-    }
-
-    assertNotNull(commit);
-
-    w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE).setIndexCommit(commit));
-
-    assertEquals(1, w.numDocs());
-
-    // commit IndexWriter to "third"
-    w.addDocument(doc);
-    commitData.put("tag", "third");
-    w.commit(commitData);
-    w.close();
-
-    // make sure "second" commit is still there
-    commit = null;
-    for(IndexCommit c : IndexReader.listCommits(dir)) {
-      if (c.getUserData().get("tag").equals("second")) {
-        commit = c;
-        break;
-      }
-    }
-
-    assertNotNull(commit);
-
-    IndexReader r = IndexReader.open(commit, true);
-    assertEquals(2, r.numDocs());
-    r.close();
-
-    // open "second", w/ writeable IndexReader & commit
-    r = IndexReader.open(commit, NoDeletionPolicy.INSTANCE, false);
-    assertEquals(2, r.numDocs());
-    r.deleteDocument(0);
-    r.deleteDocument(1);
-    commitData.put("tag", "fourth");
-    r.commit(commitData);
-    r.close();
-
-    // make sure "third" commit is still there
-    commit = null;
-    for(IndexCommit c : IndexReader.listCommits(dir)) {
-      if (c.getUserData().get("tag").equals("third")) {
-        commit = c;
-        break;
-      }
-    }
-    assertNotNull(commit);
-
-    dir.close();
-  }
-
   public void testRandomStoredFields() throws IOException {
     Directory dir = newDirectory();
     Random rand = random;
@@ -2860,7 +1530,7 @@ public class TestIndexWriter extends Luc
     if (VERBOSE) {
       w.w.setInfoStream(System.out);
     }
-    final int docCount = 200*RANDOM_MULTIPLIER;
+    final int docCount = atLeast(200);
     final int fieldCount = _TestUtil.nextInt(rand, 1, 5);
 
     final List<Integer> fieldIDs = new ArrayList<Integer>();
@@ -2925,7 +1595,8 @@ public class TestIndexWriter extends Luc
           System.out.println("TEST: cycle x=" + x + " r=" + r);
         }
 
-        for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
+        int num = atLeast(1000);
+        for(int iter=0;iter<num;iter++) {
           String testID = idsList[rand.nextInt(idsList.length)];
           if (VERBOSE) {
             System.out.println("TEST: test id=" + testID);
@@ -2980,39 +1651,7 @@ public class TestIndexWriter extends Luc
     dir.close();
   }
 
-  public void testDeleteAllSlowly() throws Exception {
-    final Directory dir = newDirectory();
-    RandomIndexWriter w = new RandomIndexWriter(random, dir);
-    final int NUM_DOCS = 1000 * RANDOM_MULTIPLIER;
-    final List<Integer> ids = new ArrayList<Integer>(NUM_DOCS);
-    for(int id=0;id<NUM_DOCS;id++) {
-      ids.add(id);
-    }
-    Collections.shuffle(ids, random);
-    for(int id : ids) {
-      Document doc = new Document();
-      doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
-      w.addDocument(doc);
-    }
-    Collections.shuffle(ids, random);
-    int upto = 0;
-    while(upto < ids.size()) {
-      final int left = ids.size() - upto;
-      final int inc = Math.min(left, _TestUtil.nextInt(random, 1, 20));
-      final int limit = upto + inc;
-      while(upto < limit) {
-        w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
-      }
-      final IndexReader r = w.getReader();
-      assertEquals(NUM_DOCS - upto, r.numDocs());
-      r.close();
-    }
-
-    w.close();
-    dir.close();
-  }
-
-  private static class StringSplitAnalyzer extends Analyzer {
+  static final class StringSplitAnalyzer extends Analyzer {
     @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new StringSplitTokenizer(reader);
@@ -3130,4 +1769,18 @@ public class TestIndexWriter extends Luc
     reader.close();
     dir.close();
   }
+
+  // LUCENE-3183
+  public void testEmptyFieldNameTIIOne() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+    iwc.setTermIndexInterval(1);
+    iwc.setReaderTermsIndexDivisor(1);
+    IndexWriter writer = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(newField("", "a b c", Field.Store.NO, Field.Index.ANALYZED));
+    writer.addDocument(doc);
+    writer.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Fri Jun 24 23:07:10 2011
@@ -18,9 +18,16 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
 
+import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.search.IndexSearcher;
@@ -860,4 +867,79 @@ public class TestIndexWriterDelete exten
     modifier.close();
     dir.close();
   }
+  
+  public void testDeleteAllSlowly() throws Exception {
+    final Directory dir = newDirectory();
+    RandomIndexWriter w = new RandomIndexWriter(random, dir);
+    final int NUM_DOCS = atLeast(1000);
+    final List<Integer> ids = new ArrayList<Integer>(NUM_DOCS);
+    for(int id=0;id<NUM_DOCS;id++) {
+      ids.add(id);
+    }
+    Collections.shuffle(ids, random);
+    for(int id : ids) {
+      Document doc = new Document();
+      doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
+      w.addDocument(doc);
+    }
+    Collections.shuffle(ids, random);
+    int upto = 0;
+    while(upto < ids.size()) {
+      final int left = ids.size() - upto;
+      final int inc = Math.min(left, _TestUtil.nextInt(random, 1, 20));
+      final int limit = upto + inc;
+      while(upto < limit) {
+        w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
+      }
+      final IndexReader r = w.getReader();
+      assertEquals(NUM_DOCS - upto, r.numDocs());
+      r.close();
+    }
+
+    w.close();
+    dir.close();
+  }
+  
+  public void testIndexingThenDeleting() throws Exception {
+    final Random r = random;
+    Directory dir = newDirectory();
+    // note this test explicitly disables payloads
+    final Analyzer analyzer = new Analyzer() {
+      @Override
+      public TokenStream tokenStream(String fieldName, Reader reader) {
+        return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+      }
+    };
+    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
+    w.setInfoStream(VERBOSE ? System.out : null);
+    Document doc = new Document();
+    doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
+    int num = atLeast(3);
+    for (int iter = 0; iter < num; iter++) {
+      int count = 0;
+
+      final boolean doIndexing = r.nextBoolean();
+      if (VERBOSE) {
+        System.out.println("TEST: iter doIndexing=" + doIndexing);
+      }
+      if (doIndexing) {
+        // Add docs until a flush is triggered
+        final int startFlushCount = w.getFlushCount();
+        while(w.getFlushCount() == startFlushCount) {
+          w.addDocument(doc);
+          count++;
+        }
+      } else {
+        // Delete docs until a flush is triggered
+        final int startFlushCount = w.getFlushCount();
+        while(w.getFlushCount() == startFlushCount) {
+          w.deleteDocuments(new Term("foo", ""+count));
+          count++;
+        }
+      }
+      assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 2500);
+    }
+    w.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Jun 24 23:07:10 2011
@@ -236,7 +236,6 @@ public class TestIndexWriterExceptions e
     assertEquals(count, count2);
     r2.close();
 
-    _TestUtil.checkIndex(dir);
     dir.close();
   }
 
@@ -287,7 +286,6 @@ public class TestIndexWriterExceptions e
     assertEquals(count, count2);
     r2.close();
 
-    _TestUtil.checkIndex(dir);
     dir.close();
   }
 
@@ -349,7 +347,6 @@ public class TestIndexWriterExceptions e
       // expected
     }
     w.close();
-    _TestUtil.checkIndex(dir);
     dir.close();
   }
 
@@ -933,7 +930,8 @@ public class TestIndexWriterExceptions e
       addDoc(w);
     w.close();
 
-    for(int i=0;i<200;i++) {
+    int iter = TEST_NIGHTLY ? 200 : 20;
+    for(int i=0;i<iter;i++) {
       if (VERBOSE) {
         System.out.println("TEST: iter " + i);
       }
@@ -1225,7 +1223,8 @@ public class TestIndexWriterExceptions e
     FailOnTermVectors[] failures = new FailOnTermVectors[] {
         new FailOnTermVectors(FailOnTermVectors.AFTER_INIT_STAGE),
         new FailOnTermVectors(FailOnTermVectors.INIT_STAGE), };
-    for (int j = 0; j < 3 * RANDOM_MULTIPLIER; j++) {
+    int num = atLeast(3);
+    for (int j = 0; j < num; j++) {
       for (FailOnTermVectors failure : failures) {
         MockDirectoryWrapper dir = newDirectory();
         IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Fri Jun 24 23:07:10 2011
@@ -104,9 +104,12 @@ public class TestIndexWriterMergePolicy 
         dir,
         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
             setMaxBufferedDocs(10).
-            setMergePolicy(newLogMergePolicy())
+            setMergePolicy(newLogMergePolicy()).
+            setMergeScheduler(new SerialMergeScheduler())
     );
 
+    writer.setInfoStream(VERBOSE ? System.out : null);
+
     for (int i = 0; i < 250; i++) {
       addDoc(writer);
       checkInvariants(writer);

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterMerging.java Fri Jun 24 23:07:10 2011
@@ -19,6 +19,9 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -113,4 +116,187 @@ public class TestIndexWriterMerging exte
     }
     writer.close();
   }
+  
+  // LUCENE-325: test expungeDeletes, when 2 singular merges
+  // are required
+  public void testExpungeDeletes() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+        TEST_VERSION_CURRENT, new MockAnalyzer(random))
+        .setMaxBufferedDocs(2).setRAMBufferSizeMB(
+                                                  IndexWriterConfig.DISABLE_AUTO_FLUSH));
+    writer.setInfoStream(VERBOSE ? System.out : null);
+    Document document = new Document();
+
+    document = new Document();
+    Field storedField = newField("stored", "stored", Field.Store.YES,
+                                  Field.Index.NO);
+    document.add(storedField);
+    Field termVectorField = newField("termVector", "termVector",
+                                      Field.Store.NO, Field.Index.NOT_ANALYZED,
+                                      Field.TermVector.WITH_POSITIONS_OFFSETS);
+    document.add(termVectorField);
+    for(int i=0;i<10;i++)
+      writer.addDocument(document);
+    writer.close();
+
+    IndexReader ir = IndexReader.open(dir, false);
+    assertEquals(10, ir.maxDoc());
+    assertEquals(10, ir.numDocs());
+    ir.deleteDocument(0);
+    ir.deleteDocument(7);
+    assertEquals(8, ir.numDocs());
+    ir.close();
+
+    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
+    assertEquals(8, writer.numDocs());
+    assertEquals(10, writer.maxDoc());
+    writer.expungeDeletes();
+    assertEquals(8, writer.numDocs());
+    writer.close();
+    ir = IndexReader.open(dir, true);
+    assertEquals(8, ir.maxDoc());
+    assertEquals(8, ir.numDocs());
+    ir.close();
+    dir.close();
+  }
+
+  // LUCENE-325: test expungeDeletes, when many adjacent merges are required
+  public void testExpungeDeletes2() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(
+        dir,
+        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+            setMaxBufferedDocs(2).
+            setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
+            setMergePolicy(newLogMergePolicy(50))
+    );
+
+    Document document = new Document();
+
+    document = new Document();
+    Field storedField = newField("stored", "stored", Store.YES,
+                                  Index.NO);
+    document.add(storedField);
+    Field termVectorField = newField("termVector", "termVector",
+                                      Store.NO, Index.NOT_ANALYZED,
+                                      TermVector.WITH_POSITIONS_OFFSETS);
+    document.add(termVectorField);
+    for(int i=0;i<98;i++)
+      writer.addDocument(document);
+    writer.close();
+
+    IndexReader ir = IndexReader.open(dir, false);
+    assertEquals(98, ir.maxDoc());
+    assertEquals(98, ir.numDocs());
+    for(int i=0;i<98;i+=2)
+      ir.deleteDocument(i);
+    assertEquals(49, ir.numDocs());
+    ir.close();
+
+    writer = new IndexWriter(
+        dir,
+        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+            setMergePolicy(newLogMergePolicy(3))
+    );
+    assertEquals(49, writer.numDocs());
+    writer.expungeDeletes();
+    writer.close();
+    ir = IndexReader.open(dir, true);
+    assertEquals(49, ir.maxDoc());
+    assertEquals(49, ir.numDocs());
+    ir.close();
+    dir.close();
+  }
+
+  // LUCENE-325: test expungeDeletes without waiting, when
+  // many adjacent merges are required
+  public void testExpungeDeletes3() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriter writer = new IndexWriter(
+        dir,
+        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+            setMaxBufferedDocs(2).
+            setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).
+            setMergePolicy(newLogMergePolicy(50))
+    );
+
+    Document document = new Document();
+
+    document = new Document();
+    Field storedField = newField("stored", "stored", Field.Store.YES,
+                                  Field.Index.NO);
+    document.add(storedField);
+    Field termVectorField = newField("termVector", "termVector",
+                                      Field.Store.NO, Field.Index.NOT_ANALYZED,
+                                      Field.TermVector.WITH_POSITIONS_OFFSETS);
+    document.add(termVectorField);
+    for(int i=0;i<98;i++)
+      writer.addDocument(document);
+    writer.close();
+
+    IndexReader ir = IndexReader.open(dir, false);
+    assertEquals(98, ir.maxDoc());
+    assertEquals(98, ir.numDocs());
+    for(int i=0;i<98;i+=2)
+      ir.deleteDocument(i);
+    assertEquals(49, ir.numDocs());
+    ir.close();
+
+    writer = new IndexWriter(
+        dir,
+        newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+            setMergePolicy(newLogMergePolicy(3))
+    );
+    writer.expungeDeletes(false);
+    writer.close();
+    ir = IndexReader.open(dir, true);
+    assertEquals(49, ir.maxDoc());
+    assertEquals(49, ir.numDocs());
+    ir.close();
+    dir.close();
+  }
+  
+  // Just intercepts all merges & verifies that we are never
+  // merging a segment with >= 20 (maxMergeDocs) docs
+  private class MyMergeScheduler extends MergeScheduler {
+    @Override
+    synchronized public void merge(IndexWriter writer)
+      throws CorruptIndexException, IOException {
+
+      while(true) {
+        MergePolicy.OneMerge merge = writer.getNextMerge();
+        if (merge == null) {
+          break;
+        }
+        for(int i=0;i<merge.segments.size();i++) {
+          assert merge.segments.get(i).docCount < 20;
+        }
+        writer.merge(merge);
+      }
+    }
+
+    @Override
+    public void close() {}
+  }
+
+  // LUCENE-1013
+  public void testSetMaxMergeDocs() throws IOException {
+    Directory dir = newDirectory();
+    IndexWriterConfig conf = newIndexWriterConfig(
+        TEST_VERSION_CURRENT, new MockAnalyzer(random))
+      .setMergeScheduler(new MyMergeScheduler()).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
+    LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
+    lmp.setMaxMergeDocs(20);
+    lmp.setMergeFactor(2);
+    IndexWriter iw = new IndexWriter(dir, conf);
+    iw.setInfoStream(VERBOSE ? System.out : null);
+    Document document = new Document();
+    document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
+                           Field.TermVector.YES));
+    for(int i=0;i<177;i++)
+      iw.addDocument(document);
+    iw.close();
+    dir.close();
+  }
 }

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Fri Jun 24 23:07:10 2011
@@ -114,7 +114,6 @@ public class TestIndexWriterOnDiskFull e
             
             // Make sure reader can open the index:
             IndexReader.open(dir, true).close();
-            _TestUtil.checkIndex(dir);
           }
             
           dir.close();
@@ -491,7 +490,6 @@ public class TestIndexWriterOnDiskFull e
     w.addDocument(doc);
     w.close();
 
-    _TestUtil.checkIndex(dir);
     dir.close();
   }
   

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Fri Jun 24 23:07:10 2011
@@ -28,6 +28,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util._TestUtil;
 
@@ -63,7 +64,7 @@ public class TestIndexWriterOnJRECrash e
       }
     } else {
       // we are the fork, setup a crashing thread
-      final int crashTime = _TestUtil.nextInt(random, 500, 4000);
+      final int crashTime = _TestUtil.nextInt(random, 3000, 4000);
       Thread t = new Thread() {
         @Override
         public void run() {
@@ -92,6 +93,8 @@ public class TestIndexWriterOnJRECrash e
         + "java");
     cmd.add("-Xmx512m");
     cmd.add("-Dtests.crashmode=true");
+    // passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act...
+    cmd.add("-Dtests.nightly=true");
     cmd.add("-DtempDir=" + tempDir.getPath());
     cmd.add("-Dtests.seed=" + random.nextLong() + ":" + random.nextLong());
     cmd.add("-ea");
@@ -123,7 +126,8 @@ public class TestIndexWriterOnJRECrash e
    */
   public boolean checkIndexes(File file) throws IOException {
     if (file.isDirectory()) {
-      Directory dir = newFSDirectory(file);
+      MockDirectoryWrapper dir = newFSDirectory(file);
+      dir.setCheckIndexOnClose(false); // don't double-checkindex
       if (IndexReader.indexExists(dir)) {
         if (VERBOSE) {
           System.err.println("Checking index: " + file);

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java Fri Jun 24 23:07:10 2011
@@ -658,7 +658,7 @@ public class TestIndexWriterReader exten
     
     ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2);
 
-    int num = 100 * RANDOM_MULTIPLIER;
+    int num = atLeast(100);
     for (int i = 0; i < num; i++) {
       writer.addDocument(createDocument(i, "test", 4));
     }
@@ -821,7 +821,6 @@ public class TestIndexWriterReader exten
 
     writer.close();
 
-    _TestUtil.checkIndex(dir1);
     dir1.close();
   }
 
@@ -908,7 +907,6 @@ public class TestIndexWriterReader exten
     assertEquals(0, excs.size());
     writer.close();
 
-    _TestUtil.checkIndex(dir1);
     r.close();
     dir1.close();
   }
@@ -1015,6 +1013,7 @@ public class TestIndexWriterReader exten
     HashSet<String> illegalCodecs = new HashSet<String>();
     illegalCodecs.add("PreFlex");
     illegalCodecs.add("SimpleText");
+    illegalCodecs.add("Memory");
 
     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
         new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1);

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyBug.java Fri Jun 24 23:07:10 2011
@@ -25,6 +25,8 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.*;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
 
 
 /**
@@ -33,8 +35,8 @@ import org.apache.lucene.util.LuceneTest
  */
 public class TestLazyBug extends LuceneTestCase {
 
-  public static int NUM_DOCS = 500;
-  public static int NUM_FIELDS = 100;
+  public static int NUM_DOCS = TEST_NIGHTLY ? 500 : 50;
+  public static int NUM_FIELDS = TEST_NIGHTLY ? 100 : 10;
 
   private static String[] data = new String[] {
     "now",
@@ -49,6 +51,19 @@ public class TestLazyBug extends LuceneT
   private static Set<String> dataset = asSet(data);
 
   private static String MAGIC_FIELD = "f"+(NUM_FIELDS/3);
+  
+  private static Directory directory;
+  
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    directory = makeIndex();
+  }
+  
+  @AfterClass
+  public static void afterClass() throws Exception {
+    directory.close();
+    directory = null;
+  }
 
   private static FieldSelector SELECTOR = new FieldSelector() {
       public FieldSelectorResult accept(String f) {
@@ -59,7 +74,7 @@ public class TestLazyBug extends LuceneT
       }
     };
 
-  private Directory makeIndex() throws Exception {
+  private static Directory makeIndex() throws Exception {
     Directory dir = newDirectory();
     try {
       IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
@@ -72,7 +87,7 @@ public class TestLazyBug extends LuceneT
           doc.add(newField("f"+f,
                             data[f % data.length]
                             + '#' + data[random.nextInt(data.length)],
-                            Field.Store.YES,
+                            Field.Store.NO,
                             Field.Index.ANALYZED));
         }
         writer.addDocument(doc);
@@ -85,8 +100,7 @@ public class TestLazyBug extends LuceneT
   }
 
   public void doTest(int[] docs) throws Exception {
-    Directory dir = makeIndex();
-    IndexReader reader = IndexReader.open(dir, true);
+    IndexReader reader = IndexReader.open(directory, true);
     for (int i = 0; i < docs.length; i++) {
       Document d = reader.document(docs[i], SELECTOR);
       d.get(MAGIC_FIELD);
@@ -109,19 +123,18 @@ public class TestLazyBug extends LuceneT
       }
     }
     reader.close();
-    dir.close();
   }
 
   public void testLazyWorks() throws Exception {
-    doTest(new int[] { 399 });
+    doTest(new int[] { NUM_DOCS-1 });
   }
 
   public void testLazyAlsoWorks() throws Exception {
-    doTest(new int[] { 399, 150 });
+    doTest(new int[] { NUM_DOCS-1, NUM_DOCS/2 });
   }
 
   public void testLazyBroken() throws Exception {
-    doTest(new int[] { 150, 399 });
+    doTest(new int[] { NUM_DOCS/2, NUM_DOCS-1 });
   }
 
 }

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Fri Jun 24 23:07:10 2011
@@ -133,6 +133,7 @@ public class TestLazyProxSkipping extend
  
     public void testLazySkipping() throws IOException {
         assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText"));
+        assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory"));
         // test whether only the minimum amount of seeks()
         // are performed
         performTest(5);

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestLongPostings.java Fri Jun 24 23:07:10 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
-import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 
@@ -65,12 +64,13 @@ public class TestLongPostings extends Lu
 
   public void testLongPostings() throws Exception {
     assumeFalse("Too slow with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText"));
+    assumeFalse("Too slow with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory"));
 
     // Don't use _TestUtil.getTempDir so that we own the
     // randomness (ie same seed will point to same dir):
     Directory dir = newFSDirectory(_TestUtil.getTempDir("longpostings" + "." + random.nextLong()));
 
-    final int NUM_DOCS = (int) ((TEST_NIGHTLY ? 4e6 : (RANDOM_MULTIPLIER*2e4)) * (1+random.nextDouble()));
+    final int NUM_DOCS = atLeast(2000);
 
     if (VERBOSE) {
       System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
@@ -145,7 +145,8 @@ public class TestLongPostings extends Lu
     assertTrue(r.docFreq(new Term("field", s1)) > 0);
     assertTrue(r.docFreq(new Term("field", s2)) > 0);
 
-    for(int iter=0;iter<1000*RANDOM_MULTIPLIER;iter++) {
+    int num = atLeast(1000);
+    for(int iter=0;iter<num;iter++) {
 
       final String term;
       final boolean doS1;

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestMultiFields.java Fri Jun 24 23:07:10 2011
@@ -27,7 +27,7 @@ public class TestMultiFields extends Luc
 
   public void testRandom() throws Exception {
 
-    int num = 2 * RANDOM_MULTIPLIER;
+    int num = atLeast(2);
     for (int iter = 0; iter < num; iter++) {
       Directory dir = newDirectory();
 

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java Fri Jun 24 23:07:10 2011
@@ -43,7 +43,6 @@ import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
@@ -97,14 +96,16 @@ public class TestNRTThreads extends Luce
 
     final long t0 = System.currentTimeMillis();
 
-    if (CodecProvider.getDefault().getDefaultFieldCodec().equals("SimpleText")) {
+    final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec();
+    if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) {
       // no
       CodecProvider.getDefault().setDefaultFieldCodec("Standard");
     }
 
     final LineFileDocs docs = new LineFileDocs(random);
     final File tempDir = _TestUtil.getTempDir("nrtopenfiles");
-    final MockDirectoryWrapper dir = new MockDirectoryWrapper(random, FSDirectory.open(tempDir));
+    final MockDirectoryWrapper dir = newFSDirectory(tempDir);
+    dir.setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves.
     final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
 
     if (LuceneTestCase.TEST_NIGHTLY) {
@@ -157,7 +158,7 @@ public class TestNRTThreads extends Luce
     final int NUM_INDEX_THREADS = 2;
     final int NUM_SEARCH_THREADS = 3;
 
-    final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : 5;
+    final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 300 : RANDOM_MULTIPLIER;
 
     final AtomicBoolean failed = new AtomicBoolean();
     final AtomicInteger addCount = new AtomicInteger();
@@ -328,11 +329,11 @@ public class TestNRTThreads extends Luce
                 if (addedField != null) {
                   doc.removeField(addedField);
                 }
-              } catch (Exception exc) {
+              } catch (Throwable t) {
                 System.out.println(Thread.currentThread().getName() + ": hit exc");
-                exc.printStackTrace();
+                t.printStackTrace();
                 failed.set(true);
-                throw new RuntimeException(exc);
+                throw new RuntimeException(t);
               }
             }
             if (VERBOSE) {
@@ -405,30 +406,30 @@ public class TestNRTThreads extends Luce
         for(int thread=0;thread<NUM_SEARCH_THREADS;thread++) {
           searchThreads[thread] = new Thread() {
               @Override
-                public void run() {
+              public void run() {
                 try {
                   TermsEnum termsEnum = MultiFields.getTerms(s.getIndexReader(), "body").iterator();
                   int seenTermCount = 0;
                   int shift;
                   int trigger;
-                  if (totTermCount.get() == 0) {
+                  if (totTermCount.get() < 10) {
                     shift = 0;
                     trigger = 1;
                   } else {
-                    shift = random.nextInt(totTermCount.get()/10);
                     trigger = totTermCount.get()/10;
+                    shift = random.nextInt(trigger);
                   }
                   while(System.currentTimeMillis() < searchStopTime) {
                     BytesRef term = termsEnum.next();
                     if (term == null) {
-                      if (seenTermCount == 0) {
+                      if (seenTermCount < 10) {
                         break;
                       }
                       totTermCount.set(seenTermCount);
                       seenTermCount = 0;
                       trigger = totTermCount.get()/10;
                       //System.out.println("trigger " + trigger);
-                      shift = random.nextInt(totTermCount.get()/10);
+                      shift = random.nextInt(trigger);
                       termsEnum.seek(new BytesRef(""));
                       continue;
                     }
@@ -448,6 +449,7 @@ public class TestNRTThreads extends Luce
                     System.out.println(Thread.currentThread().getName() + ": search done");
                   }
                 } catch (Throwable t) {
+                  System.out.println(Thread.currentThread().getName() + ": hit exc");
                   failed.set(true);
                   t.printStackTrace(System.out);
                   throw new RuntimeException(t);
@@ -577,7 +579,7 @@ public class TestNRTThreads extends Luce
 
   private int runQuery(IndexSearcher s, Query q) throws Exception {
     s.search(q, 10);
-    return s.search(q, null, 10, new Sort(new SortField("title", SortField.STRING))).totalHits;
+    return s.search(q, null, 10, new Sort(new SortField("title", SortField.Type.STRING))).totalHits;
   }
 
   private void smokeTestReader(IndexReader r) throws Exception {

Modified: lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1139458&r1=1139457&r2=1139458&view=diff
==============================================================================
--- lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/solr2193/lucene/src/test/org/apache/lucene/index/TestNorms.java Fri Jun 24 23:07:10 2011
@@ -143,7 +143,8 @@ public class TestNorms extends LuceneTes
   }
 
   private void doTestNorms(Random random, Directory dir) throws IOException {
-    for (int i=0; i<5; i++) {
+    int num = atLeast(1);
+    for (int i=0; i<num; i++) {
       addDocs(random, dir,12,true);
       verifyIndex(dir);
       modifyNormsForF1(dir);