You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/05/07 13:21:14 UTC

svn commit: r1479862 [16/38] - in /lucene/dev/branches/lucene4258: ./ dev-tools/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/ dev-tools/maven/ dev-tools/maven/solr/ dev-tools/maven/solr/core/src/java/ dev-tools/maven/solr/solrj/src/java/ dev-t...

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Tue May  7 11:20:55 2013
@@ -18,32 +18,18 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
 import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.automaton.AutomatonTestUtil;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
 
 /**
  * Compares one codec against another
@@ -157,646 +143,8 @@ public class TestDuelingCodecs extends L
   /**
    * checks the two indexes are equivalent
    */
-  public void testEquals() throws Exception {
-    assertReaderStatistics(leftReader, rightReader);
-    assertFields(MultiFields.getFields(leftReader), MultiFields.getFields(rightReader), true);
-    assertNorms(leftReader, rightReader);
-    assertStoredFields(leftReader, rightReader);
-    assertTermVectors(leftReader, rightReader);
-    assertDocValues(leftReader, rightReader);
-    assertDeletedDocs(leftReader, rightReader);
-    assertFieldInfos(leftReader, rightReader);
+  public void testEquals() throws IOException {
+    assertReaderEquals(info, leftReader, rightReader);
   }
-  
-  /** 
-   * checks that reader-level statistics are the same 
-   */
-  public void assertReaderStatistics(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    // Somewhat redundant: we never delete docs
-    assertEquals(info, leftReader.maxDoc(), rightReader.maxDoc());
-    assertEquals(info, leftReader.numDocs(), rightReader.numDocs());
-    assertEquals(info, leftReader.numDeletedDocs(), rightReader.numDeletedDocs());
-    assertEquals(info, leftReader.hasDeletions(), rightReader.hasDeletions());
-  }
-  
-  /** 
-   * Fields api equivalency 
-   */
-  public void assertFields(Fields leftFields, Fields rightFields, boolean deep) throws Exception {
-    // Fields could be null if there are no postings,
-    // but then it must be null for both
-    if (leftFields == null || rightFields == null) {
-      assertNull(info, leftFields);
-      assertNull(info, rightFields);
-      return;
-    }
-    assertFieldStatistics(leftFields, rightFields);
-    
-    Iterator<String> leftEnum = leftFields.iterator();
-    Iterator<String> rightEnum = rightFields.iterator();
-    
-    while (leftEnum.hasNext()) {
-      String field = leftEnum.next();
-      assertEquals(info, field, rightEnum.next());
-      assertTerms(leftFields.terms(field), rightFields.terms(field), deep);
-    }
-    assertFalse(rightEnum.hasNext());
-  }
-  
-  /** 
-   * checks that top-level statistics on Fields are the same 
-   */
-  public void assertFieldStatistics(Fields leftFields, Fields rightFields) throws Exception {
-    if (leftFields.size() != -1 && rightFields.size() != -1) {
-      assertEquals(info, leftFields.size(), rightFields.size());
-    }
-  }
-  
-  /** 
-   * Terms api equivalency 
-   */
-  public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception {
-    if (leftTerms == null || rightTerms == null) {
-      assertNull(info, leftTerms);
-      assertNull(info, rightTerms);
-      return;
-    }
-    assertTermsStatistics(leftTerms, rightTerms);
-    assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets());
-    assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions());
-    assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads());
-
-    TermsEnum leftTermsEnum = leftTerms.iterator(null);
-    TermsEnum rightTermsEnum = rightTerms.iterator(null);
-    assertTermsEnum(leftTermsEnum, rightTermsEnum, true);
-    
-    assertTermsSeeking(leftTerms, rightTerms);
-    
-    if (deep) {
-      int numIntersections = atLeast(3);
-      for (int i = 0; i < numIntersections; i++) {
-        String re = AutomatonTestUtil.randomRegexp(random());
-        CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton());
-        if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
-          // TODO: test start term too
-          TermsEnum leftIntersection = leftTerms.intersect(automaton, null);
-          TermsEnum rightIntersection = rightTerms.intersect(automaton, null);
-          assertTermsEnum(leftIntersection, rightIntersection, rarely());
-        }
-      }
-    }
-  }
-  
-  private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {
-    TermsEnum leftEnum = null;
-    TermsEnum rightEnum = null;
-
-    // just an upper bound
-    int numTests = atLeast(20);
-    Random random = random();
-
-    // collect this number of terms from the left side
-    HashSet<BytesRef> tests = new HashSet<BytesRef>();
-    int numPasses = 0;
-    while (numPasses < 10 && tests.size() < numTests) {
-      leftEnum = leftTerms.iterator(leftEnum);
-      BytesRef term = null;
-      while ((term = leftEnum.next()) != null) {
-        int code = random.nextInt(10);
-        if (code == 0) {
-          // the term
-          tests.add(BytesRef.deepCopyOf(term));
-        } else if (code == 1) {
-          // truncated subsequence of term
-          term = BytesRef.deepCopyOf(term);
-          if (term.length > 0) {
-            // truncate it
-            term.length = random.nextInt(term.length);
-          }
-        } else if (code == 2) {
-          // term, but ensure a non-zero offset
-          byte newbytes[] = new byte[term.length+5];
-          System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
-          tests.add(new BytesRef(newbytes, 5, term.length));
-        } else if (code == 3) {
-          switch (random().nextInt(3)) {
-            case 0:
-              tests.add(new BytesRef()); // before the first term
-              break;
-            case 1:
-              tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term
-              break;
-            case 2:
-              tests.add(new BytesRef(_TestUtil.randomSimpleString(random()))); // random term
-              break;
-            default:
-              throw new AssertionError();
-          }
-        }
-      }
-      numPasses++;
-    }
-
-    rightEnum = rightTerms.iterator(rightEnum);
-
-    ArrayList<BytesRef> shuffledTests = new ArrayList<BytesRef>(tests);
-    Collections.shuffle(shuffledTests, random);
-
-    for (BytesRef b : shuffledTests) {
-      if (rarely()) {
-        // reuse the enums
-        leftEnum = leftTerms.iterator(leftEnum);
-        rightEnum = rightTerms.iterator(rightEnum);
-      }
-
-      final boolean useCache = random().nextBoolean();
-      final boolean seekExact = random().nextBoolean();
 
-      if (seekExact) {
-        assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
-      } else {
-        SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
-        SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
-        assertEquals(info, leftStatus, rightStatus);
-        if (leftStatus != SeekStatus.END) {
-          assertEquals(info, leftEnum.term(), rightEnum.term());
-          assertTermStats(leftEnum, rightEnum);
-        }
-      }
-    }
-  }
-  
-  /** 
-   * checks collection-level statistics on Terms 
-   */
-  public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {
-    assert leftTerms.getComparator() == rightTerms.getComparator();
-    if (leftTerms.getDocCount() != -1 && rightTerms.getDocCount() != -1) {
-      assertEquals(info, leftTerms.getDocCount(), rightTerms.getDocCount());
-    }
-    if (leftTerms.getSumDocFreq() != -1 && rightTerms.getSumDocFreq() != -1) {
-      assertEquals(info, leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq());
-    }
-    if (leftTerms.getSumTotalTermFreq() != -1 && rightTerms.getSumTotalTermFreq() != -1) {
-      assertEquals(info, leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq());
-    }
-    if (leftTerms.size() != -1 && rightTerms.size() != -1) {
-      assertEquals(info, leftTerms.size(), rightTerms.size());
-    }
-  }
-
-  /** 
-   * checks the terms enum sequentially
-   * if deep is false, it does a 'shallow' test that doesnt go down to the docsenums
-   */
-  public void assertTermsEnum(TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean deep) throws Exception {
-    BytesRef term;
-    Bits randomBits = new RandomBits(leftReader.maxDoc(), random().nextDouble(), random());
-    DocsAndPositionsEnum leftPositions = null;
-    DocsAndPositionsEnum rightPositions = null;
-    DocsEnum leftDocs = null;
-    DocsEnum rightDocs = null;
-    
-    while ((term = leftTermsEnum.next()) != null) {
-      assertEquals(info, term, rightTermsEnum.next());
-      assertTermStats(leftTermsEnum, rightTermsEnum);
-      if (deep) {
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-                                   rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
-        assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-                                   rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
-
-        assertPositionsSkipping(leftTermsEnum.docFreq(), 
-                                leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
-                                rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
-        assertPositionsSkipping(leftTermsEnum.docFreq(), 
-                                leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
-                                rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
-
-        // with freqs:
-        assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs),
-            rightDocs = rightTermsEnum.docs(null, rightDocs),
-            true);
-        assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
-            rightDocs = rightTermsEnum.docs(randomBits, rightDocs),
-            true);
-
-        // w/o freqs:
-        assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, DocsEnum.FLAG_NONE),
-            rightDocs = rightTermsEnum.docs(null, rightDocs, DocsEnum.FLAG_NONE),
-            false);
-        assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs, DocsEnum.FLAG_NONE),
-            rightDocs = rightTermsEnum.docs(randomBits, rightDocs, DocsEnum.FLAG_NONE),
-            false);
-        
-        // with freqs:
-        assertDocsSkipping(leftTermsEnum.docFreq(), 
-            leftDocs = leftTermsEnum.docs(null, leftDocs),
-            rightDocs = rightTermsEnum.docs(null, rightDocs),
-            true);
-        assertDocsSkipping(leftTermsEnum.docFreq(), 
-            leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
-            rightDocs = rightTermsEnum.docs(randomBits, rightDocs),
-            true);
-
-        // w/o freqs:
-        assertDocsSkipping(leftTermsEnum.docFreq(), 
-            leftDocs = leftTermsEnum.docs(null, leftDocs, DocsEnum.FLAG_NONE),
-            rightDocs = rightTermsEnum.docs(null, rightDocs, DocsEnum.FLAG_NONE),
-            false);
-        assertDocsSkipping(leftTermsEnum.docFreq(), 
-            leftDocs = leftTermsEnum.docs(randomBits, leftDocs, DocsEnum.FLAG_NONE),
-            rightDocs = rightTermsEnum.docs(randomBits, rightDocs, DocsEnum.FLAG_NONE),
-            false);
-      }
-    }
-    assertNull(info, rightTermsEnum.next());
-  }
-  
-  /**
-   * checks term-level statistics
-   */
-  public void assertTermStats(TermsEnum leftTermsEnum, TermsEnum rightTermsEnum) throws Exception {
-    assertEquals(info, leftTermsEnum.docFreq(), rightTermsEnum.docFreq());
-    if (leftTermsEnum.totalTermFreq() != -1 && rightTermsEnum.totalTermFreq() != -1) {
-      assertEquals(info, leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq());
-    }
-  }
-  
-  /**
-   * checks docs + freqs + positions + payloads, sequentially
-   */
-  public void assertDocsAndPositionsEnum(DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception {
-    if (leftDocs == null || rightDocs == null) {
-      assertNull(leftDocs);
-      assertNull(rightDocs);
-      return;
-    }
-    assertTrue(info, leftDocs.docID() == -1 || leftDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
-    assertTrue(info, rightDocs.docID() == -1 || rightDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
-    int docid;
-    while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-      assertEquals(info, docid, rightDocs.nextDoc());
-      int freq = leftDocs.freq();
-      assertEquals(info, freq, rightDocs.freq());
-      for (int i = 0; i < freq; i++) {
-        assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
-        assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
-        assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
-        assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
-      }
-    }
-    assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
-  }
-  
-  /**
-   * checks docs + freqs, sequentially
-   */
-  public void assertDocsEnum(DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
-    if (leftDocs == null) {
-      assertNull(rightDocs);
-      return;
-    }
-    assertTrue(info, leftDocs.docID() == -1 || leftDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
-    assertTrue(info, rightDocs.docID() == -1 || rightDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
-    int docid;
-    while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
-      assertEquals(info, docid, rightDocs.nextDoc());
-      if (hasFreqs) {
-        assertEquals(info, leftDocs.freq(), rightDocs.freq());
-      }
-    }
-    assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
-  }
-  
-  /**
-   * checks advancing docs
-   */
-  public void assertDocsSkipping(int docFreq, DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
-    if (leftDocs == null) {
-      assertNull(rightDocs);
-      return;
-    }
-    int docid = -1;
-    int averageGap = leftReader.maxDoc() / (1+docFreq);
-    int skipInterval = 16;
-
-    while (true) {
-      if (random().nextBoolean()) {
-        // nextDoc()
-        docid = leftDocs.nextDoc();
-        assertEquals(info, docid, rightDocs.nextDoc());
-      } else {
-        // advance()
-        int skip = docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
-        docid = leftDocs.advance(skip);
-        assertEquals(info, docid, rightDocs.advance(skip));
-      }
-      
-      if (docid == DocIdSetIterator.NO_MORE_DOCS) {
-        return;
-      }
-      if (hasFreqs) {
-        assertEquals(info, leftDocs.freq(), rightDocs.freq());
-      }
-    }
-  }
-  
-  /**
-   * checks advancing docs + positions
-   */
-  public void assertPositionsSkipping(int docFreq, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception {
-    if (leftDocs == null || rightDocs == null) {
-      assertNull(leftDocs);
-      assertNull(rightDocs);
-      return;
-    }
-    
-    int docid = -1;
-    int averageGap = leftReader.maxDoc() / (1+docFreq);
-    int skipInterval = 16;
-
-    while (true) {
-      if (random().nextBoolean()) {
-        // nextDoc()
-        docid = leftDocs.nextDoc();
-        assertEquals(info, docid, rightDocs.nextDoc());
-      } else {
-        // advance()
-        int skip = docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
-        docid = leftDocs.advance(skip);
-        assertEquals(info, docid, rightDocs.advance(skip));
-      }
-      
-      if (docid == DocIdSetIterator.NO_MORE_DOCS) {
-        return;
-      }
-      int freq = leftDocs.freq();
-      assertEquals(info, freq, rightDocs.freq());
-      for (int i = 0; i < freq; i++) {
-        assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
-        assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
-      }
-    }
-  }
-  
-  /** 
-   * checks that norms are the same across all fields 
-   */
-  public void assertNorms(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    Fields leftFields = MultiFields.getFields(leftReader);
-    Fields rightFields = MultiFields.getFields(rightReader);
-    // Fields could be null if there are no postings,
-    // but then it must be null for both
-    if (leftFields == null || rightFields == null) {
-      assertNull(info, leftFields);
-      assertNull(info, rightFields);
-      return;
-    }
-    
-    for (String field : leftFields) {
-      NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field);
-      NumericDocValues rightNorms = MultiDocValues.getNormValues(rightReader, field);
-      if (leftNorms != null && rightNorms != null) {
-        assertDocValues(leftReader.maxDoc(), leftNorms, rightNorms);
-      } else {
-        assertNull(info, leftNorms);
-        assertNull(info, rightNorms);
-      }
-    }
-  }
-  
-  /** 
-   * checks that stored fields of all documents are the same 
-   */
-  public void assertStoredFields(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    assert leftReader.maxDoc() == rightReader.maxDoc();
-    for (int i = 0; i < leftReader.maxDoc(); i++) {
-      StoredDocument leftDoc = leftReader.document(i);
-      StoredDocument rightDoc = rightReader.document(i);
-      
-      // TODO: I think this is bogus because we don't document what the order should be
-      // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff
-      // in whatever way it wants (e.g. maybe it packs related fields together or something)
-      // To fix this, we sort the fields in both documents by name, but
-      // we still assume that all instances with same name are in order:
-      Comparator<StorableField> comp = new Comparator<StorableField>() {
-        @Override
-        public int compare(StorableField arg0, StorableField arg1) {
-          return arg0.name().compareTo(arg1.name());
-        }        
-      };
-      Collections.sort(leftDoc.getFields(), comp);
-      Collections.sort(rightDoc.getFields(), comp);
-
-      Iterator<StorableField> leftIterator = leftDoc.iterator();
-      Iterator<StorableField> rightIterator = rightDoc.iterator();
-      while (leftIterator.hasNext()) {
-        assertTrue(info, rightIterator.hasNext());
-        assertStoredField(leftIterator.next(), rightIterator.next());
-      }
-      assertFalse(info, rightIterator.hasNext());
-    }
-  }
-  
-  /** 
-   * checks that two stored fields are equivalent 
-   */
-  public void assertStoredField(StorableField leftField, StorableField rightField) {
-    assertEquals(info, leftField.name(), rightField.name());
-    assertEquals(info, leftField.binaryValue(), rightField.binaryValue());
-    assertEquals(info, leftField.stringValue(), rightField.stringValue());
-    assertEquals(info, leftField.numericValue(), rightField.numericValue());
-    // TODO: should we check the FT at all?
-  }
-  
-  /** 
-   * checks that term vectors across all fields are equivalent 
-   */
-  public void assertTermVectors(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    assert leftReader.maxDoc() == rightReader.maxDoc();
-    for (int i = 0; i < leftReader.maxDoc(); i++) {
-      Fields leftFields = leftReader.getTermVectors(i);
-      Fields rightFields = rightReader.getTermVectors(i);
-      assertFields(leftFields, rightFields, rarely());
-    }
-  }
-
-  private static Set<String> getDVFields(IndexReader reader) {
-    Set<String> fields = new HashSet<String>();
-    for(FieldInfo fi : MultiFields.getMergedFieldInfos(reader)) {
-      if (fi.hasDocValues()) {
-        fields.add(fi.name);
-      }
-    }
-
-    return fields;
-  }
-  
-  /**
-   * checks that docvalues across all fields are equivalent
-   */
-  public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    Set<String> leftFields = getDVFields(leftReader);
-    Set<String> rightFields = getDVFields(rightReader);
-    assertEquals(info, leftFields, rightFields);
-
-    for (String field : leftFields) {
-      // TODO: clean this up... very messy
-      {
-        NumericDocValues leftValues = MultiDocValues.getNumericValues(leftReader, field);
-        NumericDocValues rightValues = MultiDocValues.getNumericValues(rightReader, field);
-        if (leftValues != null && rightValues != null) {
-          assertDocValues(leftReader.maxDoc(), leftValues, rightValues);
-        } else {
-          assertNull(info, leftValues);
-          assertNull(info, rightValues);
-        }
-      }
-
-      {
-        BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field);
-        BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field);
-        if (leftValues != null && rightValues != null) {
-          BytesRef scratchLeft = new BytesRef();
-          BytesRef scratchRight = new BytesRef();
-          for(int docID=0;docID<leftReader.maxDoc();docID++) {
-            leftValues.get(docID, scratchLeft);
-            rightValues.get(docID, scratchRight);
-            assertEquals(info, scratchLeft, scratchRight);
-          }
-        } else {
-          assertNull(info, leftValues);
-          assertNull(info, rightValues);
-        }
-      }
-      
-      {
-        SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field);
-        SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field);
-        if (leftValues != null && rightValues != null) {
-          // numOrds
-          assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
-          // ords
-          BytesRef scratchLeft = new BytesRef();
-          BytesRef scratchRight = new BytesRef();
-          for (int i = 0; i < leftValues.getValueCount(); i++) {
-            leftValues.lookupOrd(i, scratchLeft);
-            rightValues.lookupOrd(i, scratchRight);
-            assertEquals(info, scratchLeft, scratchRight);
-          }
-          // bytes
-          for(int docID=0;docID<leftReader.maxDoc();docID++) {
-            leftValues.get(docID, scratchLeft);
-            rightValues.get(docID, scratchRight);
-            assertEquals(info, scratchLeft, scratchRight);
-          }
-        } else {
-          assertNull(info, leftValues);
-          assertNull(info, rightValues);
-        }
-      }
-      
-      {
-        SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field);
-        SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field);
-        if (leftValues != null && rightValues != null) {
-          // numOrds
-          assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
-          // ords
-          BytesRef scratchLeft = new BytesRef();
-          BytesRef scratchRight = new BytesRef();
-          for (int i = 0; i < leftValues.getValueCount(); i++) {
-            leftValues.lookupOrd(i, scratchLeft);
-            rightValues.lookupOrd(i, scratchRight);
-            assertEquals(info, scratchLeft, scratchRight);
-          }
-          // ord lists
-          for(int docID=0;docID<leftReader.maxDoc();docID++) {
-            leftValues.setDocument(docID);
-            rightValues.setDocument(docID);
-            long ord;
-            while ((ord = leftValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
-              assertEquals(info, ord, rightValues.nextOrd());
-            }
-            assertEquals(info, SortedSetDocValues.NO_MORE_ORDS, rightValues.nextOrd());
-          }
-        } else {
-          assertNull(info, leftValues);
-          assertNull(info, rightValues);
-        }
-      }
-    }
-  }
-  
-  public void assertDocValues(int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) throws Exception {
-    assertNotNull(info, leftDocValues);
-    assertNotNull(info, rightDocValues);
-    for(int docID=0;docID<num;docID++) {
-      assertEquals(leftDocValues.get(docID),
-                   rightDocValues.get(docID));
-    }
-  }
-  
-  // TODO: this is kinda stupid, we don't delete documents in the test.
-  public void assertDeletedDocs(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    assert leftReader.numDeletedDocs() == rightReader.numDeletedDocs();
-    Bits leftBits = MultiFields.getLiveDocs(leftReader);
-    Bits rightBits = MultiFields.getLiveDocs(rightReader);
-    
-    if (leftBits == null || rightBits == null) {
-      assertNull(info, leftBits);
-      assertNull(info, rightBits);
-      return;
-    }
-    
-    assert leftReader.maxDoc() == rightReader.maxDoc();
-    assertEquals(info, leftBits.length(), rightBits.length());
-    for (int i = 0; i < leftReader.maxDoc(); i++) {
-      assertEquals(info, leftBits.get(i), rightBits.get(i));
-    }
-  }
-  
-  public void assertFieldInfos(IndexReader leftReader, IndexReader rightReader) throws Exception {
-    FieldInfos leftInfos = MultiFields.getMergedFieldInfos(leftReader);
-    FieldInfos rightInfos = MultiFields.getMergedFieldInfos(rightReader);
-    
-    // TODO: would be great to verify more than just the names of the fields!
-    TreeSet<String> left = new TreeSet<String>();
-    TreeSet<String> right = new TreeSet<String>();
-    
-    for (FieldInfo fi : leftInfos) {
-      left.add(fi.name);
-    }
-    
-    for (FieldInfo fi : rightInfos) {
-      right.add(fi.name);
-    }
-    
-    assertEquals(info, left, right);
-  }
-  
-  
-  private static class RandomBits implements Bits {
-    FixedBitSet bits;
-    
-    RandomBits(int maxDoc, double pctLive, Random random) {
-      bits = new FixedBitSet(maxDoc);
-      for (int i = 0; i < maxDoc; i++) {
-        if (random.nextDouble() <= pctLive) {        
-          bits.set(i);
-        }
-      }
-    }
-    
-    @Override
-    public boolean get(int index) {
-      return bits.get(index);
-    }
-
-    @Override
-    public int length() {
-      return bits.length();
-    }
-  }
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFieldReplacements.java Tue May  7 11:20:55 2013
@@ -27,7 +27,6 @@ import java.util.Locale;
 import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
 import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field.Store;
@@ -139,11 +138,9 @@ public class TestFieldReplacements exten
     
     final IndexWriterConfig config;
     if (randomConfig) {
-      config = newIndexWriterConfig(TEST_VERSION_CURRENT,
-          new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+      config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
     } else {
-      config = new IndexWriterConfig(TEST_VERSION_CURRENT,
-          new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
+      config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
     }
     System.out.println(config.getMergePolicy());
     config.setCodec(new SimpleTextCodec());
@@ -614,7 +611,7 @@ public class TestFieldReplacements exten
   
   public void testReplaceAndAddAgain() throws IOException {
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+        TEST_VERSION_CURRENT, new MockAnalyzer(random())));
     
     FieldType fieldType = new FieldType();
     fieldType.setIndexed(true);
@@ -703,7 +700,7 @@ public class TestFieldReplacements exten
   
   public void testReplaceAndAddSameField() throws IOException {
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
-        TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+        TEST_VERSION_CURRENT, new MockAnalyzer(random())));
     
     FieldType fieldType = new FieldType();
     fieldType.setIndexed(true);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java Tue May  7 11:20:55 2013
@@ -21,12 +21,13 @@ package org.apache.lucene.index;
 import java.io.IOException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
+
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
@@ -159,21 +160,27 @@ public class TestFilterAtomicReader exte
     directory.close();
     target.close();
   }
-  
-  private void checkOverrideMethods(Class<?> clazz) {
-    boolean fail = false;
-    for (Method m : clazz.getMethods()) {
-      int mods = m.getModifiers();
-      if (Modifier.isStatic(mods) || Modifier.isFinal(mods) || m.isSynthetic()) {
+
+  private static void checkOverrideMethods(Class<?> clazz) throws NoSuchMethodException, SecurityException {
+    final Class<?> superClazz = clazz.getSuperclass();
+    for (Method m : superClazz.getMethods()) {
+      final int mods = m.getModifiers();
+      if (Modifier.isStatic(mods) || Modifier.isAbstract(mods) || Modifier.isFinal(mods) || m.isSynthetic()
+          || m.getName().equals("attributes")) {
         continue;
       }
-      Class<?> declaringClass = m.getDeclaringClass();
-      if (declaringClass != clazz && declaringClass != Object.class) {
-        System.err.println("method is not overridden by "+clazz.getName()+": " + m.toGenericString());
-        fail = true;
+      // The point of these checks is to ensure that methods that have a default
+      // impl through other methods are not overridden. This makes the number of
+      // methods to override to have a working impl minimal and prevents from some
+      // traps: for example, think about having getCoreCacheKey delegate to the
+      // filtered impl by default
+      final Method subM = clazz.getMethod(m.getName(), m.getParameterTypes());
+      if (subM.getDeclaringClass() == clazz
+          && m.getDeclaringClass() != Object.class
+          && m.getDeclaringClass() != subM.getDeclaringClass()) {
+        fail(clazz + " overrides " + m + " although it has a default impl");
       }
     }
-    assertFalse(clazz.getName()+" does not override some methods; see log above", fail);
   }
 
   public void testOverrideMethods() throws Exception {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java Tue May  7 11:20:55 2013
@@ -60,7 +60,7 @@ public class TestForTooMuchCloning exten
     //System.out.println("merge clone count=" + cloneCount);
     assertTrue("too many calls to IndexInput.clone during merging: " + dir.getInputCloneCount(), cloneCount < 500);
 
-    final IndexSearcher s = new IndexSearcher(r);
+    final IndexSearcher s = newSearcher(r);
 
     // MTQ that matches all terms so the AUTO_REWRITE should
     // cutover to filter rewrite and reuse a single DocsEnum

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Tue May  7 11:20:55 2013
@@ -52,7 +52,9 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.BaseDirectoryWrapper;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockFactory;
@@ -456,7 +458,7 @@ public class TestIndexWriter extends Luc
       writer.close();
 
       IndexReader reader = DirectoryReader.open(dir);
-      IndexSearcher searcher = new IndexSearcher(reader);
+      IndexSearcher searcher = newSearcher(reader);
       int totalHits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1).totalHits;
       assertEquals(n*100, totalHits);
       reader.close();
@@ -487,7 +489,7 @@ public class TestIndexWriter extends Luc
       Term searchTerm = new Term("field", "aaa");
 
       IndexReader reader = DirectoryReader.open(dir);
-      IndexSearcher searcher = new IndexSearcher(reader);
+      IndexSearcher searcher = newSearcher(reader);
       ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals(10, hits.length);
       reader.close();
@@ -509,7 +511,7 @@ public class TestIndexWriter extends Luc
       }
       writer.close();
       reader = DirectoryReader.open(dir);
-      searcher = new IndexSearcher(reader);
+      searcher = newSearcher(reader);
       hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals(27, hits.length);
       reader.close();
@@ -587,7 +589,7 @@ public class TestIndexWriter extends Luc
       writer.close();
       Term searchTerm = new Term("content", "aaa");
       IndexReader reader = DirectoryReader.open(dir);
-      IndexSearcher searcher = new IndexSearcher(reader);
+      IndexSearcher searcher = newSearcher(reader);
       ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals("did not get right number of hits", 100, hits.length);
       reader.close();
@@ -1371,10 +1373,10 @@ public class TestIndexWriter extends Luc
     // Validates that iw.deleteUnusedFiles() also deletes unused index commits
     // in case a deletion policy which holds onto commits is used.
     Directory dir = newDirectory();
-    SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
     IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random()))
-        .setIndexDeletionPolicy(sdp));
+        .setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
+    SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
 
     // First commit
     Document doc = new Document();
@@ -1390,7 +1392,7 @@ public class TestIndexWriter extends Luc
     assertEquals(1, DirectoryReader.listCommits(dir).size());
 
     // Keep that commit
-    sdp.snapshot("id");
+    IndexCommit id = sdp.snapshot();
 
     // Second commit - now KeepOnlyLastCommit cannot delete the prev commit.
     doc = new Document();
@@ -1400,7 +1402,7 @@ public class TestIndexWriter extends Luc
     assertEquals(2, DirectoryReader.listCommits(dir).size());
 
     // Should delete the unreferenced commit
-    sdp.release("id");
+    sdp.release(id);
     writer.deleteUnusedFiles();
     assertEquals(1, DirectoryReader.listCommits(dir).size());
 
@@ -1477,7 +1479,7 @@ public class TestIndexWriter extends Luc
   }
 
   public void testNoSegmentFile() throws IOException {
-    Directory dir = newDirectory();
+    BaseDirectoryWrapper dir = newDirectory();
     dir.setLockFactory(NoLockFactory.getNoLockFactory());
     IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2));
@@ -1497,6 +1499,10 @@ public class TestIndexWriter extends Luc
     w2.close();
     // If we don't do that, the test fails on Windows
     w.rollback();
+
+    // This test leaves only segments.gen, which causes
+    // DirectoryReader.indexExists to return true:
+    dir.setCheckIndexOnClose(false);
     dir.close();
   }
 
@@ -2115,5 +2121,52 @@ public class TestIndexWriter extends Luc
     }
     
   }
-  
+
+  // LUCENE-2727/LUCENE-2812/LUCENE-4738:
+  public void testCorruptFirstCommit() throws Exception {
+    for(int i=0;i<6;i++) {
+      BaseDirectoryWrapper dir = newDirectory();
+      dir.createOutput("segments_0", IOContext.DEFAULT).close();
+      IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+      int mode = i/2;
+      if (mode == 0) {
+        iwc.setOpenMode(OpenMode.CREATE);
+      } else if (mode == 1) {
+        iwc.setOpenMode(OpenMode.APPEND);
+      } else if (mode == 2) {
+        iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
+      }
+
+      if (VERBOSE) {
+        System.out.println("\nTEST: i=" + i);
+      }
+
+      try {
+        if ((i & 1) == 0) {
+          new IndexWriter(dir, iwc).close();
+        } else {
+          new IndexWriter(dir, iwc).rollback();
+        }
+        if (mode != 0) {
+          fail("expected exception");
+        }
+      } catch (IOException ioe) {
+        // OpenMode.APPEND should throw an exception since no
+        // index exists:
+        if (mode == 0) {
+          // Unexpected
+          throw ioe;
+        }
+      }
+
+      if (VERBOSE) {
+        System.out.println("  at close: " + Arrays.toString(dir.listAll()));
+      }
+
+      if (mode != 0) {
+        dir.setCheckIndexOnClose(false);
+      }
+      dir.close();
+    }
+  }
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Tue May  7 11:20:55 2013
@@ -51,7 +51,7 @@ public class TestIndexWriterCommit exten
 
       Term searchTerm = new Term("content", "aaa");
       DirectoryReader reader = DirectoryReader.open(dir);
-      IndexSearcher searcher = new IndexSearcher(reader);
+      IndexSearcher searcher = newSearcher(reader);
       ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals("first number of hits", 14, hits.length);
       reader.close();
@@ -64,7 +64,7 @@ public class TestIndexWriterCommit exten
           TestIndexWriter.addDoc(writer);
         }
         IndexReader r = DirectoryReader.open(dir);
-        searcher = new IndexSearcher(r);
+        searcher = newSearcher(r);
         hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
         assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
         r.close();
@@ -76,7 +76,7 @@ public class TestIndexWriterCommit exten
       assertFalse("reader should not be current now", reader.isCurrent());
 
       IndexReader r = DirectoryReader.open(dir);
-      searcher = new IndexSearcher(r);
+      searcher = newSearcher(r);
       hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals("reader did not see changes after writer was closed", 47, hits.length);
       r.close();
@@ -102,7 +102,7 @@ public class TestIndexWriterCommit exten
 
     Term searchTerm = new Term("content", "aaa");
     IndexReader reader = DirectoryReader.open(dir);
-    IndexSearcher searcher = new IndexSearcher(reader);
+    IndexSearcher searcher = newSearcher(reader);
     ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
     assertEquals("first number of hits", 14, hits.length);
     reader.close();
@@ -116,7 +116,7 @@ public class TestIndexWriterCommit exten
     writer.deleteDocuments(searchTerm);
 
     reader = DirectoryReader.open(dir);
-    searcher = new IndexSearcher(reader);
+    searcher = newSearcher(reader);
     hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
     assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
     reader.close();
@@ -127,7 +127,7 @@ public class TestIndexWriterCommit exten
     TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
 
     reader = DirectoryReader.open(dir);
-    searcher = new IndexSearcher(reader);
+    searcher = newSearcher(reader);
     hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
     assertEquals("saw changes after writer.abort", 14, hits.length);
     reader.close();
@@ -148,7 +148,7 @@ public class TestIndexWriterCommit exten
         TestIndexWriter.addDoc(writer);
       }
       IndexReader r = DirectoryReader.open(dir);
-      searcher = new IndexSearcher(r);
+      searcher = newSearcher(r);
       hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
       assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
       r.close();
@@ -156,7 +156,7 @@ public class TestIndexWriterCommit exten
 
     writer.close();
     IndexReader r = DirectoryReader.open(dir);
-    searcher = new IndexSearcher(r);
+    searcher = newSearcher(r);
     hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
     assertEquals("didn't see changes after close", 218, hits.length);
     r.close();

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Tue May  7 11:20:55 2013
@@ -216,8 +216,26 @@ public class TestIndexWriterConfig exten
     IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
     IndexWriterConfig clone = conf.clone();
 
-    // Clone is shallow since not all parameters are cloneable.
-    assertTrue(conf.getIndexDeletionPolicy() == clone.getIndexDeletionPolicy());
+    // Make sure parameters that can't be reused are cloned
+    IndexDeletionPolicy delPolicy = conf.delPolicy;
+    IndexDeletionPolicy delPolicyClone = clone.delPolicy;
+    assertTrue(delPolicy.getClass() == delPolicyClone.getClass() && (delPolicy != delPolicyClone || delPolicy.clone() == delPolicyClone.clone()));
+
+    FlushPolicy flushPolicy = conf.flushPolicy;
+    FlushPolicy flushPolicyClone = clone.flushPolicy;
+    assertTrue(flushPolicy.getClass() == flushPolicyClone.getClass() && (flushPolicy != flushPolicyClone || flushPolicy.clone() == flushPolicyClone.clone()));
+
+    DocumentsWriterPerThreadPool pool = conf.indexerThreadPool;
+    DocumentsWriterPerThreadPool poolClone = clone.indexerThreadPool;
+    assertTrue(pool.getClass() == poolClone.getClass() && (pool != poolClone || pool.clone() == poolClone.clone()));
+
+    MergePolicy mergePolicy = conf.mergePolicy;
+    MergePolicy mergePolicyClone = clone.mergePolicy;
+    assertTrue(mergePolicy.getClass() == mergePolicyClone.getClass() && (mergePolicy != mergePolicyClone || mergePolicy.clone() == mergePolicyClone.clone()));
+
+    MergeScheduler mergeSched = conf.mergeScheduler;
+    MergeScheduler mergeSchedClone = clone.mergeScheduler;
+    assertTrue(mergeSched.getClass() == mergeSchedClone.getClass() && (mergeSched != mergeSchedClone || mergeSched.clone() == mergeSchedClone.clone()));
 
     conf.setMergeScheduler(new SerialMergeScheduler());
     assertEquals(ConcurrentMergeScheduler.class, clone.getMergeScheduler().getClass());

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Tue May  7 11:20:55 2013
@@ -40,7 +40,6 @@ import org.apache.lucene.search.TermQuer
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
@@ -407,7 +406,7 @@ public class TestIndexWriterDelete exten
 
   private int getHitCount(Directory dir, Term term) throws IOException {
     IndexReader reader = DirectoryReader.open(dir);
-    IndexSearcher searcher = new IndexSearcher(reader);
+    IndexSearcher searcher = newSearcher(reader);
     int hitCount = searcher.search(new TermQuery(term), null, 1000).totalHits;
     reader.close();
     return hitCount;
@@ -499,6 +498,7 @@ public class TestIndexWriterDelete exten
           }
           testName = "disk full during reader.close() @ " + thisDiskFree
             + " bytes";
+          dir.setRandomIOExceptionRateOnOpen(random().nextDouble()*0.01);
         } else {
           thisDiskFree = 0;
           rate = 0.0;
@@ -506,6 +506,7 @@ public class TestIndexWriterDelete exten
             System.out.println("\ncycle: same writer: unlimited disk space");
           }
           testName = "reader re-use after disk full";
+          dir.setRandomIOExceptionRateOnOpen(0.0);
         }
 
         dir.setMaxSizeInBytes(thisDiskFree);
@@ -549,6 +550,7 @@ public class TestIndexWriterDelete exten
         final double randomIOExceptionRate = dir.getRandomIOExceptionRate();
         final long maxSizeInBytes = dir.getMaxSizeInBytes();
         dir.setRandomIOExceptionRate(0.0);
+        dir.setRandomIOExceptionRateOnOpen(0.0);
         dir.setMaxSizeInBytes(0);
         if (!success) {
           // Must force the close else the writer can have

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Tue May  7 11:20:55 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
  * limitations under the License.
  */
 
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
@@ -1393,7 +1394,7 @@ public class TestIndexWriterExceptions e
     final IndexReader r = w.getReader();
     w.close();
 
-    final IndexSearcher s = new IndexSearcher(r);
+    final IndexSearcher s = newSearcher(r);
     PhraseQuery pq = new PhraseQuery();
     pq.add(new Term("content", "silly"));
     pq.add(new Term("content", "content"));
@@ -1473,7 +1474,7 @@ public class TestIndexWriterExceptions e
     final IndexReader r = w.getReader();
     w.close();
 
-    final IndexSearcher s = new IndexSearcher(r);
+    final IndexSearcher s = newSearcher(r);
     PhraseQuery pq = new PhraseQuery();
     pq.add(new Term("content", "silly"));
     pq.add(new Term("content", "content"));
@@ -1495,7 +1496,7 @@ public class TestIndexWriterExceptions e
       if (doFail && name.startsWith("segments_")) {
         StackTraceElement[] trace = new Exception().getStackTrace();
         for (int i = 0; i < trace.length; i++) {
-          if ("indexExists".equals(trace[i].getMethodName())) {
+          if ("read".equals(trace[i].getMethodName())) {
             throw new UnsupportedOperationException("expected UOE");
           }
         }
@@ -1515,8 +1516,8 @@ public class TestIndexWriterExceptions e
       new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
       fail("should have gotten a UOE");
     } catch (UnsupportedOperationException expected) {
-      
     }
+
     uoe.doFail = false;
     d.close();
   }
@@ -1627,4 +1628,75 @@ public class TestIndexWriterExceptions e
     iw.close();
     dir.close();
   }
+  
+  // See LUCENE-4870 TooManyOpenFiles errors are thrown as
+  // FNFExceptions which can trigger data loss.
+  public void testTooManyFileException() throws Exception {
+
+    // Create failure that throws Too many open files exception randomly
+    MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
+
+      @Override
+      public MockDirectoryWrapper.Failure reset() {
+        doFail = false;
+        return this;
+      }
+
+      @Override
+      public void eval(MockDirectoryWrapper dir) throws IOException {
+        if (doFail) {
+          if (random().nextBoolean()) {
+            throw new FileNotFoundException("some/file/name.ext (Too many open files)");
+          }
+        }
+      }
+    };
+
+    MockDirectoryWrapper dir = newMockDirectory();
+    // The exception is only thrown on open input
+    dir.setFailOnOpenInput(true);
+    dir.failOn(failure);
+
+    // Create an index with one document
+    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+    IndexWriter iw = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new StringField("foo", "bar", Field.Store.NO));
+    iw.addDocument(doc); // add a document
+    iw.commit();
+    DirectoryReader ir = DirectoryReader.open(dir);
+    assertEquals(1, ir.numDocs());
+    ir.close();
+    iw.close();
+
+    // Open and close the index a few times
+    for (int i = 0; i < 10; i++) {
+      failure.setDoFail();
+      iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+      try {
+        iw = new IndexWriter(dir, iwc);
+      } catch (CorruptIndexException ex) {
+        // Exceptions are fine - we are running out of file handlers here
+        continue;
+      } catch (FileNotFoundException ex) {
+        continue;
+      }
+      failure.clearDoFail();
+      iw.close();
+      ir = DirectoryReader.open(dir);
+      assertEquals("lost document after iteration: " + i, 1, ir.numDocs());
+      ir.close();
+    }
+
+    // Check if document is still there
+    failure.clearDoFail();
+    ir = DirectoryReader.open(dir);
+    assertEquals(1, ir.numDocs());
+    ir.close();
+
+    dir.close();
+  }
+  
+  
+
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Tue May  7 11:20:55 2013
@@ -275,6 +275,7 @@ public class TestIndexWriterOnDiskFull e
           String testName = null;
           
           if (0 == x) {
+            dir.setRandomIOExceptionRateOnOpen(random().nextDouble()*0.01);
             thisDiskFree = diskFree;
             if (diskRatio >= 2.0) {
               rate /= 2;
@@ -288,6 +289,7 @@ public class TestIndexWriterOnDiskFull e
             if (VERBOSE)
               testName = "disk full test " + methodName + " with disk full at " + diskFree + " bytes";
           } else {
+            dir.setRandomIOExceptionRateOnOpen(0.0);
             thisDiskFree = 0;
             rate = 0.0;
             if (VERBOSE)
@@ -363,6 +365,7 @@ public class TestIndexWriterOnDiskFull e
           // we succeeded, we see all docs added, and if we
           // failed, we see either all docs or no docs added
           // (transactional semantics):
+          dir.setRandomIOExceptionRateOnOpen(0.0);
           try {
             reader = DirectoryReader.open(dir);
           } catch (IOException e) {
@@ -432,6 +435,7 @@ public class TestIndexWriterOnDiskFull e
         // Make sure we don't hit disk full during close below:
         dir.setMaxSizeInBytes(0);
         dir.setRandomIOExceptionRate(0.0);
+        dir.setRandomIOExceptionRateOnOpen(0.0);
         
         writer.close();
         

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Tue May  7 11:20:55 2013
@@ -138,7 +138,13 @@ public class TestIndexWriterOnJRECrash e
         if (VERBOSE) {
           System.err.println("Checking index: " + file);
         }
-        _TestUtil.checkIndex(dir);
+        // LUCENE-4738: if we crashed while writing first
+        // commit it's possible index will be corrupt (by
+        // design we don't try to be smart about this case
+        // since that too risky):
+        if (SegmentInfos.getLastCommitGeneration(dir) > 1) {
+          _TestUtil.checkIndex(dir);
+        }
         dir.close();
         return true;
       }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Tue May  7 11:20:55 2013
@@ -39,6 +39,7 @@ import org.apache.lucene.store.Directory
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.ThreadInterruptedException;
 import org.apache.lucene.util._TestUtil;
@@ -965,6 +966,46 @@ public class TestIndexWriterReader exten
     assertTrue(didWarm.get());
   }
   
+  public void testSimpleMergedSegmentWramer() throws Exception {
+    Directory dir = newDirectory();
+    final AtomicBoolean didWarm = new AtomicBoolean();
+    InfoStream infoStream = new InfoStream() {
+      @Override
+      public void close() throws IOException {}
+
+      @Override
+      public void message(String component, String message) {
+        if ("SMSW".equals(component)) {
+          didWarm.set(true);
+        }
+      }
+
+      @Override
+      public boolean isEnabled(String component) {
+        return true;
+      }
+    };
+    IndexWriter w = new IndexWriter(
+        dir,
+        newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+            setMaxBufferedDocs(2).
+            setReaderPooling(true).
+            setInfoStream(infoStream).
+            setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream)).
+            setMergePolicy(newLogMergePolicy(10))
+    );
+
+    Document doc = new Document();
+    doc.add(newStringField("foo", "bar", Field.Store.NO));
+    for(int i=0;i<20;i++) {
+      w.addDocument(doc);
+    }
+    w.waitForMerges();
+    w.close();
+    dir.close();
+    assertTrue(didWarm.get());
+  }
+  
   public void testNoTermsIndex() throws Exception {
     // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if
     // they're picked.

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Tue May  7 11:20:55 2013
@@ -282,7 +282,7 @@ public class TestIndexableField extends 
     final IndexReader r = w.getReader();
     w.close();
 
-    final IndexSearcher s = new IndexSearcher(r);
+    final IndexSearcher s = newSearcher(r);
     int counter = 0;
     for(int id=0;id<NUM_DOCS;id++) {
       if (VERBOSE) {

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Tue May  7 11:20:55 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.concurrent.ExecutionException;
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
@@ -281,7 +282,7 @@ public class TestOmitTf extends LuceneTe
      * Verify the index
      */         
     IndexReader reader = DirectoryReader.open(dir);
-    IndexSearcher searcher = new IndexSearcher(reader);
+    IndexSearcher searcher = newSearcher(reader);
     searcher.setSimilarity(new SimpleSimilarity());
         
     Term a = new Term("noTf", term);
@@ -299,8 +300,13 @@ public class TestOmitTf extends LuceneTe
     try {
       searcher.search(pq, 10);
       fail("did not hit expected exception");
-    } catch (IllegalStateException ise) {
-      // expected
+    } catch (Exception e) {
+      Throwable cause = e;
+      // If the searcher uses an executor service, the IAE is wrapped into other exceptions
+      while (cause.getCause() != null) {
+        cause = cause.getCause();
+      }
+      assertTrue("Expected an IAE, got " + cause, cause instanceof IllegalStateException);
     }
         
     searcher.search(q1,

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java Tue May  7 11:20:55 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
 import org.apache.lucene.document.Field;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.*;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
@@ -114,6 +115,29 @@ public class TestParallelAtomicReader ex
     dir2.close();    
   }
   
+  public void testCloseInnerReader() throws Exception {
+    Directory dir1 = getDir1(random());
+    AtomicReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1));
+    
+    // with overlapping
+    ParallelAtomicReader pr = new ParallelAtomicReader(true,
+     new AtomicReader[] {ir1},
+     new AtomicReader[] {ir1});
+
+    ir1.close();
+    
+    try {
+      pr.document(0);
+      fail("ParallelAtomicReader should be already closed because inner reader was closed!");
+    } catch (AlreadyClosedException e) {
+      // pass
+    }
+    
+    // noop:
+    pr.close();
+    dir1.close();
+  }
+
   public void testIncompatibleIndexes() throws IOException {
     // two documents:
     Directory dir1 = getDir1(random());

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java Tue May  7 11:20:55 2013
@@ -23,8 +23,10 @@ import java.util.Random;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader.ReaderClosedListener;
 import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.*;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -82,12 +84,15 @@ public class TestParallelCompositeReader
     // close subreaders, ParallelReader will not change refCounts, but close on its own close
     ParallelCompositeReader pr = new ParallelCompositeReader(ir1 = DirectoryReader.open(dir1),
                                                              ir2 = DirectoryReader.open(dir2));
+    IndexReader psub1 = pr.getSequentialSubReaders().get(0);
     // check RefCounts
     assertEquals(1, ir1.getRefCount());
     assertEquals(1, ir2.getRefCount());
+    assertEquals(1, psub1.getRefCount());
     pr.close();
     assertEquals(0, ir1.getRefCount());
     assertEquals(0, ir2.getRefCount());
+    assertEquals(0, psub1.getRefCount());
     dir1.close();
     dir2.close();    
   }
@@ -100,20 +105,115 @@ public class TestParallelCompositeReader
 
     // don't close subreaders, so ParallelReader will increment refcounts
     ParallelCompositeReader pr = new ParallelCompositeReader(false, ir1, ir2);
+    IndexReader psub1 = pr.getSequentialSubReaders().get(0);
     // check RefCounts
     assertEquals(2, ir1.getRefCount());
     assertEquals(2, ir2.getRefCount());
+    assertEquals("refCount must be 1, as the synthetic reader was created by ParallelCompositeReader", 1, psub1.getRefCount());
     pr.close();
     assertEquals(1, ir1.getRefCount());
     assertEquals(1, ir2.getRefCount());
+    assertEquals("refcount must be 0 because parent was closed", 0, psub1.getRefCount());
     ir1.close();
     ir2.close();
     assertEquals(0, ir1.getRefCount());
     assertEquals(0, ir2.getRefCount());
+    assertEquals("refcount should not change anymore", 0, psub1.getRefCount());
     dir1.close();
     dir2.close();    
   }
   
+  // closeSubreaders=false
+  public void testReaderClosedListener1() throws Exception {
+    Directory dir1 = getDir1(random());
+    CompositeReader ir1 = DirectoryReader.open(dir1);
+    
+    // with overlapping
+    ParallelCompositeReader pr = new ParallelCompositeReader(false,
+     new CompositeReader[] {ir1},
+     new CompositeReader[] {ir1});
+
+    final int[] listenerClosedCount = new int[1];
+
+    assertEquals(3, pr.leaves().size());
+
+    for(AtomicReaderContext cxt : pr.leaves()) {
+      cxt.reader().addReaderClosedListener(new ReaderClosedListener() {
+          @Override
+          public void onClose(IndexReader reader) {
+            listenerClosedCount[0]++;
+          }
+        });
+    }
+    pr.close();
+    ir1.close();
+    assertEquals(3, listenerClosedCount[0]);
+    dir1.close();
+  }
+
+  // closeSubreaders=true
+  public void testReaderClosedListener2() throws Exception {
+    Directory dir1 = getDir1(random());
+    CompositeReader ir1 = DirectoryReader.open(dir1);
+    
+    // with overlapping
+    ParallelCompositeReader pr = new ParallelCompositeReader(true,
+     new CompositeReader[] {ir1},
+     new CompositeReader[] {ir1});
+
+    final int[] listenerClosedCount = new int[1];
+
+    assertEquals(3, pr.leaves().size());
+
+    for(AtomicReaderContext cxt : pr.leaves()) {
+      cxt.reader().addReaderClosedListener(new ReaderClosedListener() {
+          @Override
+          public void onClose(IndexReader reader) {
+            listenerClosedCount[0]++;
+          }
+        });
+    }
+    pr.close();
+    assertEquals(3, listenerClosedCount[0]);
+    dir1.close();
+  }
+
+  public void testCloseInnerReader() throws Exception {
+    Directory dir1 = getDir1(random());
+    CompositeReader ir1 = DirectoryReader.open(dir1);
+    assertEquals(1, ir1.getSequentialSubReaders().get(0).getRefCount());
+    
+    // with overlapping
+    ParallelCompositeReader pr = new ParallelCompositeReader(true,
+     new CompositeReader[] {ir1},
+     new CompositeReader[] {ir1});
+
+    IndexReader psub = pr.getSequentialSubReaders().get(0);
+    assertEquals(1, psub.getRefCount());
+
+    ir1.close();
+
+    assertEquals("refCount of synthetic subreader should be unchanged", 1, psub.getRefCount());
+    try {
+      psub.document(0);
+      fail("Subreader should be already closed because inner reader was closed!");
+    } catch (AlreadyClosedException e) {
+      // pass
+    }
+    
+    try {
+      pr.document(0);
+      fail("ParallelCompositeReader should be already closed because inner reader was closed!");
+    } catch (AlreadyClosedException e) {
+      // pass
+    }
+    
+    // noop:
+    pr.close();
+    assertEquals(0, psub.getRefCount());
+    dir1.close();
+  }
+
   public void testIncompatibleIndexes1() throws IOException {
     // two documents:
     Directory dir1 = getDir1(random());
@@ -277,6 +377,30 @@ public class TestParallelCompositeReader
     dir2.close();
   }
   
+  public void testToString() throws IOException {
+    Directory dir1 = getDir1(random());
+    CompositeReader ir1 = DirectoryReader.open(dir1);
+    ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] {ir1});
+    
+    final String s = pr.toString();
+    assertTrue("toString incorrect: " + s, s.startsWith("ParallelCompositeReader(ParallelAtomicReader("));
+
+    pr.close();
+    dir1.close();
+  }
+  
+  public void testToStringCompositeComposite() throws IOException {
+    Directory dir1 = getDir1(random());
+    CompositeReader ir1 = DirectoryReader.open(dir1);
+    ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] {new MultiReader(ir1)});
+    
+    final String s = pr.toString();
+    assertTrue("toString incorrect: " + s, s.startsWith("ParallelCompositeReader(ParallelCompositeReader(ParallelAtomicReader("));
+
+    pr.close();
+    dir1.close();
+  }
+  
   private void queryTest(Query query) throws IOException {
     ScoreDoc[] parallelHits = parallel.search(query, null, 1000).scoreDocs;
     ScoreDoc[] singleHits = single.search(query, null, 1000).scoreDocs;

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java Tue May  7 11:20:55 2013
@@ -18,13 +18,10 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
-import java.util.Map;
-import java.util.Map.Entry;
 
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
 import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.junit.After;
 import org.junit.Before;
@@ -32,161 +29,153 @@ import org.junit.Test;
 
 public class TestPersistentSnapshotDeletionPolicy extends TestSnapshotDeletionPolicy {
 
-  // Keep it a class member so that getDeletionPolicy can use it
-  private Directory snapshotDir;
-  
-  // so we can close it if called by SDP tests
-  private PersistentSnapshotDeletionPolicy psdp;
-  
   @Before
   @Override
   public void setUp() throws Exception {
     super.setUp();
-    snapshotDir = newDirectory();
   }
   
   @After
   @Override
   public void tearDown() throws Exception {
-    if (psdp != null) psdp.close();
-    snapshotDir.close();
     super.tearDown();
   }
   
-  @Override
-  protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
-    if (psdp != null) psdp.close();
-    snapshotDir.close();
-    snapshotDir = newDirectory();
-    return psdp = new PersistentSnapshotDeletionPolicy(
-        new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.CREATE,
-        TEST_VERSION_CURRENT);
+  private SnapshotDeletionPolicy getDeletionPolicy(Directory dir) throws IOException {
+    return new PersistentSnapshotDeletionPolicy(
+        new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE);
   }
 
-  @Override
-  protected SnapshotDeletionPolicy getDeletionPolicy(Map<String, String> snapshots) throws IOException {
-    SnapshotDeletionPolicy sdp = getDeletionPolicy();
-    if (snapshots != null) {
-      for (Entry<String, String> e: snapshots.entrySet()) {
-        sdp.registerSnapshotInfo(e.getKey(), e.getValue(), null);
-      }
-    }
-    return sdp;
-  }
-
-  @Override
   @Test
   public void testExistingSnapshots() throws Exception {
     int numSnapshots = 3;
-    Directory dir = newDirectory();
-    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) getDeletionPolicy();
-    IndexWriter writer = new IndexWriter(dir, getConfig(random(), psdp));
-    prepareIndexAndSnapshots(psdp, writer, numSnapshots, "snapshot");
+    MockDirectoryWrapper dir = newMockDirectory();
+    IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
+    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+    assertNull(psdp.getLastSaveFile());
+    prepareIndexAndSnapshots(psdp, writer, numSnapshots);
+    assertNotNull(psdp.getLastSaveFile());
     writer.close();
-    psdp.close();
+
+    // Make sure only 1 save file exists:
+    int count = 0;
+    for(String file : dir.listAll()) {
+      if (file.startsWith(PersistentSnapshotDeletionPolicy.SNAPSHOTS_PREFIX)) {
+        count++;
+      }
+    }
+    assertEquals(1, count);
+
+    // Make sure we fsync:
+    dir.crash();
+    dir.clearCrash();
 
     // Re-initialize and verify snapshots were persisted
     psdp = new PersistentSnapshotDeletionPolicy(
-        new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-        TEST_VERSION_CURRENT);
-    new IndexWriter(dir, getConfig(random(), psdp)).close();
+        new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+
+    writer = new IndexWriter(dir, getConfig(random(), psdp));
+    psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
 
-    assertSnapshotExists(dir, psdp, numSnapshots);
     assertEquals(numSnapshots, psdp.getSnapshots().size());
-    psdp.close();
-    dir.close();
-  }
+    assertEquals(numSnapshots, psdp.getSnapshotCount());
+    assertSnapshotExists(dir, psdp, numSnapshots, false);
 
-  @Test(expected=IllegalArgumentException.class)
-  public void testIllegalSnapshotId() throws Exception {
-    getDeletionPolicy().snapshot("$SNAPSHOTS_DOC$");
-  }
-  
-  @Test
-  public void testInvalidSnapshotInfos() throws Exception {
-    // Add the correct number of documents (1), but without snapshot information
-    IndexWriter writer = new IndexWriter(snapshotDir, getConfig(random(), null));
     writer.addDocument(new Document());
+    writer.commit();
+    snapshots.add(psdp.snapshot());
+    assertEquals(numSnapshots+1, psdp.getSnapshots().size());
+    assertEquals(numSnapshots+1, psdp.getSnapshotCount());
+    assertSnapshotExists(dir, psdp, numSnapshots+1, false);
+
     writer.close();
-    try {
-      new PersistentSnapshotDeletionPolicy(
-          new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-          TEST_VERSION_CURRENT);
-      fail("should not have succeeded to read from an invalid Directory");
-    } catch (IllegalStateException e) {
-    }
+    dir.close();
   }
 
   @Test
   public void testNoSnapshotInfos() throws Exception {
-    // Initialize an empty index in snapshotDir - PSDP should initialize successfully.
-    new IndexWriter(snapshotDir, getConfig(random(), null)).close();
+    Directory dir = newDirectory();
     new PersistentSnapshotDeletionPolicy(
-        new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-        TEST_VERSION_CURRENT).close();
+        new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE);
+    dir.close();
   }
 
-  @Test(expected=IllegalStateException.class)
-  public void testTooManySnapshotInfos() throws Exception {
-    // Write two documents to the snapshots directory - illegal.
-    IndexWriter writer = new IndexWriter(snapshotDir, getConfig(random(), null));
-    writer.addDocument(new Document());
+  @Test
+  public void testMissingSnapshots() throws Exception {
+    Directory dir = newDirectory();
+    try {
+      new PersistentSnapshotDeletionPolicy(
+                                           new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+      fail("did not hit expected exception");
+    } catch (IllegalStateException ise) {
+      // expected
+    }
+    dir.close();
+  }
+
+  public void testExceptionDuringSave() throws Exception {
+    MockDirectoryWrapper dir = newMockDirectory();
+    dir.failOn(new MockDirectoryWrapper.Failure() {
+      @Override
+      public void eval(MockDirectoryWrapper dir) throws IOException {
+        StackTraceElement[] trace = Thread.currentThread().getStackTrace();
+        for (int i = 0; i < trace.length; i++) {
+          if (PersistentSnapshotDeletionPolicy.class.getName().equals(trace[i].getClassName()) && "persist".equals(trace[i].getMethodName())) {
+            throw new IOException("now fail on purpose");
+          }
+        }
+      }
+      });
+    IndexWriter writer = new IndexWriter(dir, getConfig(random(), new PersistentSnapshotDeletionPolicy(
+                                         new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.CREATE_OR_APPEND)));
     writer.addDocument(new Document());
+    writer.commit();
+
+    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+    try {
+      psdp.snapshot();
+    } catch (IOException ioe) {
+      if (ioe.getMessage().equals("now fail on purpose")) {
+        // ok
+      } else {
+        throw ioe;
+      }
+    }
+    assertEquals(0, psdp.getSnapshotCount());
     writer.close();
-    
-    new PersistentSnapshotDeletionPolicy(
-        new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-        TEST_VERSION_CURRENT).close();
-    fail("should not have succeeded to open an invalid directory");
+    assertEquals(1, DirectoryReader.listCommits(dir).size());
+    dir.close();
   }
 
   @Test
   public void testSnapshotRelease() throws Exception {
     Directory dir = newDirectory();
-    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) getDeletionPolicy();
-    IndexWriter writer = new IndexWriter(dir, getConfig(random(), psdp));
-    prepareIndexAndSnapshots(psdp, writer, 1, "snapshot");
+    IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
+    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+    prepareIndexAndSnapshots(psdp, writer, 1);
     writer.close();
 
-    psdp.release("snapshot0");
-    psdp.close();
+    psdp.release(snapshots.get(0));
 
     psdp = new PersistentSnapshotDeletionPolicy(
-        new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-        TEST_VERSION_CURRENT);
-    assertEquals("Should have no snapshots !", 0, psdp.getSnapshots().size());
-    psdp.close();
+        new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+    assertEquals("Should have no snapshots !", 0, psdp.getSnapshotCount());
     dir.close();
   }
 
   @Test
-  public void testStaticRead() throws Exception {
-    // While PSDP is open, it keeps a lock on the snapshots directory and thus
-    // prevents reading the snapshots information. This test checks that the 
-    // static read method works.
-    int numSnapshots = 1;
+  public void testSnapshotReleaseByGeneration() throws Exception {
     Directory dir = newDirectory();
-    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) getDeletionPolicy();
-    IndexWriter writer = new IndexWriter(dir, getConfig(random(), psdp));
-    prepareIndexAndSnapshots(psdp, writer, numSnapshots, "snapshot");
+    IndexWriter writer = new IndexWriter(dir, getConfig(random(), getDeletionPolicy(dir)));
+    PersistentSnapshotDeletionPolicy psdp = (PersistentSnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
+    prepareIndexAndSnapshots(psdp, writer, 1);
     writer.close();
+
+    psdp.release(snapshots.get(0).getGeneration());
+
+    psdp = new PersistentSnapshotDeletionPolicy(
+        new KeepOnlyLastCommitDeletionPolicy(), dir, OpenMode.APPEND);
+    assertEquals("Should have no snapshots !", 0, psdp.getSnapshotCount());
     dir.close();
-    
-    try {
-      // This should fail, since the snapshots directory is locked - we didn't close it !
-      new PersistentSnapshotDeletionPolicy(
-          new KeepOnlyLastCommitDeletionPolicy(), snapshotDir, OpenMode.APPEND,
-          TEST_VERSION_CURRENT);
-     fail("should not have reached here - the snapshots directory should be locked!");
-    } catch (LockObtainFailedException e) {
-      // expected
-    } finally {
-      psdp.close();
-    }
-    
-    // Reading the snapshots info should succeed though
-    Map<String, String> snapshots = PersistentSnapshotDeletionPolicy.readSnapshotsInfo(snapshotDir);
-    assertEquals("expected " + numSnapshots + " snapshots, got " + snapshots.size(), numSnapshots, snapshots.size());
   }
-  
 }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java Tue May  7 11:20:55 2013
@@ -112,7 +112,7 @@ public class TestRollingUpdates extends 
 
         r = w.getReader(applyDeletions);
         if (applyDeletions) {
-          s = new IndexSearcher(r);
+          s = newSearcher(r);
         } else {
           s = null;
         }

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java Tue May  7 11:20:55 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.Arrays;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Document;
@@ -29,7 +30,6 @@ import org.apache.lucene.util.FixedBitSe
 import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.packed.PackedInts;
 
 public class TestSegmentMerger extends LuceneTestCase {
   //The variables for the new merged segment
@@ -80,10 +80,9 @@ public class TestSegmentMerger extends L
     final Codec codec = Codec.getDefault();
     final SegmentInfo si = new SegmentInfo(mergedDir, Constants.LUCENE_MAIN_VERSION, mergedSegment, -1, false, codec, null, null);
 
-    SegmentMerger merger = new SegmentMerger(si, InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
-                                             MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
-    merger.add(reader1);
-    merger.add(reader2);
+    SegmentMerger merger = new SegmentMerger(Arrays.<AtomicReader>asList(reader1, reader2),
+        si, InfoStream.getDefault(), mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
+        MergeState.CheckAbort.NONE, new FieldInfos.FieldNumbers(), newIOContext(random()));
     MergeState mergeState = merger.merge();
     int docsMerged = mergeState.segmentInfo.getDocCount();
     assertTrue(docsMerged == 2);

Modified: lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java?rev=1479862&r1=1479861&r2=1479862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/core/src/test/org/apache/lucene/index/TestSegmentTermDocs.java Tue May  7 11:20:55 2013
@@ -149,7 +149,7 @@ public class TestSegmentTermDocs extends
     assertTrue(tdocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(1, tdocs.docID());
     assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.advance(0) != DocIdSetIterator.NO_MORE_DOCS);
+    assertTrue(tdocs.advance(2) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(2, tdocs.docID());
     assertTrue(tdocs.advance(4) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(4, tdocs.docID());
@@ -189,7 +189,7 @@ public class TestSegmentTermDocs extends
     assertTrue(tdocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(11, tdocs.docID());
     assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.advance(5) != DocIdSetIterator.NO_MORE_DOCS);
+    assertTrue(tdocs.advance(12) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(12, tdocs.docID());
     assertTrue(tdocs.advance(15) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(15, tdocs.docID());
@@ -233,7 +233,7 @@ public class TestSegmentTermDocs extends
     assertTrue(tdocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(27, tdocs.docID());
     assertEquals(4, tdocs.freq());
-    assertTrue(tdocs.advance(5) != DocIdSetIterator.NO_MORE_DOCS);
+    assertTrue(tdocs.advance(28) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(28, tdocs.docID());
     assertTrue(tdocs.advance(40) != DocIdSetIterator.NO_MORE_DOCS);
     assertEquals(40, tdocs.docID());