You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ja...@apache.org on 2013/05/30 09:53:46 UTC
svn commit: r1487777 [21/50] - in /lucene/dev/branches/security: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/replicator/
dev-tools/maven/ dev-tools/maven/lucene/ dev-tools/ma...
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Thu May 30 07:53:18 2013
@@ -18,32 +18,18 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.Random;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.SortedSetDocValuesField;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.automaton.AutomatonTestUtil;
-import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.automaton.RegExp;
/**
* Compares one codec against another
@@ -157,646 +143,8 @@ public class TestDuelingCodecs extends L
/**
* checks the two indexes are equivalent
*/
- public void testEquals() throws Exception {
- assertReaderStatistics(leftReader, rightReader);
- assertFields(MultiFields.getFields(leftReader), MultiFields.getFields(rightReader), true);
- assertNorms(leftReader, rightReader);
- assertStoredFields(leftReader, rightReader);
- assertTermVectors(leftReader, rightReader);
- assertDocValues(leftReader, rightReader);
- assertDeletedDocs(leftReader, rightReader);
- assertFieldInfos(leftReader, rightReader);
+ public void testEquals() throws IOException {
+ assertReaderEquals(info, leftReader, rightReader);
}
-
- /**
- * checks that reader-level statistics are the same
- */
- public void assertReaderStatistics(IndexReader leftReader, IndexReader rightReader) throws Exception {
- // Somewhat redundant: we never delete docs
- assertEquals(info, leftReader.maxDoc(), rightReader.maxDoc());
- assertEquals(info, leftReader.numDocs(), rightReader.numDocs());
- assertEquals(info, leftReader.numDeletedDocs(), rightReader.numDeletedDocs());
- assertEquals(info, leftReader.hasDeletions(), rightReader.hasDeletions());
- }
-
- /**
- * Fields api equivalency
- */
- public void assertFields(Fields leftFields, Fields rightFields, boolean deep) throws Exception {
- // Fields could be null if there are no postings,
- // but then it must be null for both
- if (leftFields == null || rightFields == null) {
- assertNull(info, leftFields);
- assertNull(info, rightFields);
- return;
- }
- assertFieldStatistics(leftFields, rightFields);
-
- Iterator<String> leftEnum = leftFields.iterator();
- Iterator<String> rightEnum = rightFields.iterator();
-
- while (leftEnum.hasNext()) {
- String field = leftEnum.next();
- assertEquals(info, field, rightEnum.next());
- assertTerms(leftFields.terms(field), rightFields.terms(field), deep);
- }
- assertFalse(rightEnum.hasNext());
- }
-
- /**
- * checks that top-level statistics on Fields are the same
- */
- public void assertFieldStatistics(Fields leftFields, Fields rightFields) throws Exception {
- if (leftFields.size() != -1 && rightFields.size() != -1) {
- assertEquals(info, leftFields.size(), rightFields.size());
- }
- }
-
- /**
- * Terms api equivalency
- */
- public void assertTerms(Terms leftTerms, Terms rightTerms, boolean deep) throws Exception {
- if (leftTerms == null || rightTerms == null) {
- assertNull(info, leftTerms);
- assertNull(info, rightTerms);
- return;
- }
- assertTermsStatistics(leftTerms, rightTerms);
- assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets());
- assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions());
- assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads());
-
- TermsEnum leftTermsEnum = leftTerms.iterator(null);
- TermsEnum rightTermsEnum = rightTerms.iterator(null);
- assertTermsEnum(leftTermsEnum, rightTermsEnum, true);
-
- assertTermsSeeking(leftTerms, rightTerms);
-
- if (deep) {
- int numIntersections = atLeast(3);
- for (int i = 0; i < numIntersections; i++) {
- String re = AutomatonTestUtil.randomRegexp(random());
- CompiledAutomaton automaton = new CompiledAutomaton(new RegExp(re, RegExp.NONE).toAutomaton());
- if (automaton.type == CompiledAutomaton.AUTOMATON_TYPE.NORMAL) {
- // TODO: test start term too
- TermsEnum leftIntersection = leftTerms.intersect(automaton, null);
- TermsEnum rightIntersection = rightTerms.intersect(automaton, null);
- assertTermsEnum(leftIntersection, rightIntersection, rarely());
- }
- }
- }
- }
-
- private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {
- TermsEnum leftEnum = null;
- TermsEnum rightEnum = null;
-
- // just an upper bound
- int numTests = atLeast(20);
- Random random = random();
-
- // collect this number of terms from the left side
- HashSet<BytesRef> tests = new HashSet<BytesRef>();
- int numPasses = 0;
- while (numPasses < 10 && tests.size() < numTests) {
- leftEnum = leftTerms.iterator(leftEnum);
- BytesRef term = null;
- while ((term = leftEnum.next()) != null) {
- int code = random.nextInt(10);
- if (code == 0) {
- // the term
- tests.add(BytesRef.deepCopyOf(term));
- } else if (code == 1) {
- // truncated subsequence of term
- term = BytesRef.deepCopyOf(term);
- if (term.length > 0) {
- // truncate it
- term.length = random.nextInt(term.length);
- }
- } else if (code == 2) {
- // term, but ensure a non-zero offset
- byte newbytes[] = new byte[term.length+5];
- System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
- tests.add(new BytesRef(newbytes, 5, term.length));
- } else if (code == 3) {
- switch (random().nextInt(3)) {
- case 0:
- tests.add(new BytesRef()); // before the first term
- break;
- case 1:
- tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term
- break;
- case 2:
- tests.add(new BytesRef(_TestUtil.randomSimpleString(random()))); // random term
- break;
- default:
- throw new AssertionError();
- }
- }
- }
- numPasses++;
- }
-
- rightEnum = rightTerms.iterator(rightEnum);
-
- ArrayList<BytesRef> shuffledTests = new ArrayList<BytesRef>(tests);
- Collections.shuffle(shuffledTests, random);
-
- for (BytesRef b : shuffledTests) {
- if (rarely()) {
- // reuse the enums
- leftEnum = leftTerms.iterator(leftEnum);
- rightEnum = rightTerms.iterator(rightEnum);
- }
-
- final boolean useCache = random().nextBoolean();
- final boolean seekExact = random().nextBoolean();
- if (seekExact) {
- assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
- } else {
- SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
- SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
- assertEquals(info, leftStatus, rightStatus);
- if (leftStatus != SeekStatus.END) {
- assertEquals(info, leftEnum.term(), rightEnum.term());
- assertTermStats(leftEnum, rightEnum);
- }
- }
- }
- }
-
- /**
- * checks collection-level statistics on Terms
- */
- public void assertTermsStatistics(Terms leftTerms, Terms rightTerms) throws Exception {
- assert leftTerms.getComparator() == rightTerms.getComparator();
- if (leftTerms.getDocCount() != -1 && rightTerms.getDocCount() != -1) {
- assertEquals(info, leftTerms.getDocCount(), rightTerms.getDocCount());
- }
- if (leftTerms.getSumDocFreq() != -1 && rightTerms.getSumDocFreq() != -1) {
- assertEquals(info, leftTerms.getSumDocFreq(), rightTerms.getSumDocFreq());
- }
- if (leftTerms.getSumTotalTermFreq() != -1 && rightTerms.getSumTotalTermFreq() != -1) {
- assertEquals(info, leftTerms.getSumTotalTermFreq(), rightTerms.getSumTotalTermFreq());
- }
- if (leftTerms.size() != -1 && rightTerms.size() != -1) {
- assertEquals(info, leftTerms.size(), rightTerms.size());
- }
- }
-
- /**
- * checks the terms enum sequentially
- * if deep is false, it does a 'shallow' test that doesnt go down to the docsenums
- */
- public void assertTermsEnum(TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean deep) throws Exception {
- BytesRef term;
- Bits randomBits = new RandomBits(leftReader.maxDoc(), random().nextDouble(), random());
- DocsAndPositionsEnum leftPositions = null;
- DocsAndPositionsEnum rightPositions = null;
- DocsEnum leftDocs = null;
- DocsEnum rightDocs = null;
-
- while ((term = leftTermsEnum.next()) != null) {
- assertEquals(info, term, rightTermsEnum.next());
- assertTermStats(leftTermsEnum, rightTermsEnum);
- if (deep) {
- assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
- assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
-
- assertPositionsSkipping(leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions));
- assertPositionsSkipping(leftTermsEnum.docFreq(),
- leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
- rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
-
- // with freqs:
- assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs),
- rightDocs = rightTermsEnum.docs(null, rightDocs),
- true);
- assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs),
- true);
-
- // w/o freqs:
- assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, DocsEnum.FLAG_NONE),
- rightDocs = rightTermsEnum.docs(null, rightDocs, DocsEnum.FLAG_NONE),
- false);
- assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs, DocsEnum.FLAG_NONE),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs, DocsEnum.FLAG_NONE),
- false);
-
- // with freqs:
- assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(null, leftDocs),
- rightDocs = rightTermsEnum.docs(null, rightDocs),
- true);
- assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs),
- true);
-
- // w/o freqs:
- assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(null, leftDocs, DocsEnum.FLAG_NONE),
- rightDocs = rightTermsEnum.docs(null, rightDocs, DocsEnum.FLAG_NONE),
- false);
- assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(randomBits, leftDocs, DocsEnum.FLAG_NONE),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs, DocsEnum.FLAG_NONE),
- false);
- }
- }
- assertNull(info, rightTermsEnum.next());
- }
-
- /**
- * checks term-level statistics
- */
- public void assertTermStats(TermsEnum leftTermsEnum, TermsEnum rightTermsEnum) throws Exception {
- assertEquals(info, leftTermsEnum.docFreq(), rightTermsEnum.docFreq());
- if (leftTermsEnum.totalTermFreq() != -1 && rightTermsEnum.totalTermFreq() != -1) {
- assertEquals(info, leftTermsEnum.totalTermFreq(), rightTermsEnum.totalTermFreq());
- }
- }
-
- /**
- * checks docs + freqs + positions + payloads, sequentially
- */
- public void assertDocsAndPositionsEnum(DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception {
- if (leftDocs == null || rightDocs == null) {
- assertNull(leftDocs);
- assertNull(rightDocs);
- return;
- }
- assertTrue(info, leftDocs.docID() == -1 || leftDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(info, rightDocs.docID() == -1 || rightDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
- int docid;
- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- assertEquals(info, docid, rightDocs.nextDoc());
- int freq = leftDocs.freq();
- assertEquals(info, freq, rightDocs.freq());
- for (int i = 0; i < freq; i++) {
- assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- assertEquals(info, leftDocs.startOffset(), rightDocs.startOffset());
- assertEquals(info, leftDocs.endOffset(), rightDocs.endOffset());
- }
- }
- assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
- }
-
- /**
- * checks docs + freqs, sequentially
- */
- public void assertDocsEnum(DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
- if (leftDocs == null) {
- assertNull(rightDocs);
- return;
- }
- assertTrue(info, leftDocs.docID() == -1 || leftDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(info, rightDocs.docID() == -1 || rightDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
- int docid;
- while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- assertEquals(info, docid, rightDocs.nextDoc());
- if (hasFreqs) {
- assertEquals(info, leftDocs.freq(), rightDocs.freq());
- }
- }
- assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
- }
-
- /**
- * checks advancing docs
- */
- public void assertDocsSkipping(int docFreq, DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
- if (leftDocs == null) {
- assertNull(rightDocs);
- return;
- }
- int docid = -1;
- int averageGap = leftReader.maxDoc() / (1+docFreq);
- int skipInterval = 16;
-
- while (true) {
- if (random().nextBoolean()) {
- // nextDoc()
- docid = leftDocs.nextDoc();
- assertEquals(info, docid, rightDocs.nextDoc());
- } else {
- // advance()
- int skip = docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
- docid = leftDocs.advance(skip);
- assertEquals(info, docid, rightDocs.advance(skip));
- }
-
- if (docid == DocIdSetIterator.NO_MORE_DOCS) {
- return;
- }
- if (hasFreqs) {
- assertEquals(info, leftDocs.freq(), rightDocs.freq());
- }
- }
- }
-
- /**
- * checks advancing docs + positions
- */
- public void assertPositionsSkipping(int docFreq, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception {
- if (leftDocs == null || rightDocs == null) {
- assertNull(leftDocs);
- assertNull(rightDocs);
- return;
- }
-
- int docid = -1;
- int averageGap = leftReader.maxDoc() / (1+docFreq);
- int skipInterval = 16;
-
- while (true) {
- if (random().nextBoolean()) {
- // nextDoc()
- docid = leftDocs.nextDoc();
- assertEquals(info, docid, rightDocs.nextDoc());
- } else {
- // advance()
- int skip = docid + (int) Math.ceil(Math.abs(skipInterval + random().nextGaussian() * averageGap));
- docid = leftDocs.advance(skip);
- assertEquals(info, docid, rightDocs.advance(skip));
- }
-
- if (docid == DocIdSetIterator.NO_MORE_DOCS) {
- return;
- }
- int freq = leftDocs.freq();
- assertEquals(info, freq, rightDocs.freq());
- for (int i = 0; i < freq; i++) {
- assertEquals(info, leftDocs.nextPosition(), rightDocs.nextPosition());
- assertEquals(info, leftDocs.getPayload(), rightDocs.getPayload());
- }
- }
- }
-
- /**
- * checks that norms are the same across all fields
- */
- public void assertNorms(IndexReader leftReader, IndexReader rightReader) throws Exception {
- Fields leftFields = MultiFields.getFields(leftReader);
- Fields rightFields = MultiFields.getFields(rightReader);
- // Fields could be null if there are no postings,
- // but then it must be null for both
- if (leftFields == null || rightFields == null) {
- assertNull(info, leftFields);
- assertNull(info, rightFields);
- return;
- }
-
- for (String field : leftFields) {
- NumericDocValues leftNorms = MultiDocValues.getNormValues(leftReader, field);
- NumericDocValues rightNorms = MultiDocValues.getNormValues(rightReader, field);
- if (leftNorms != null && rightNorms != null) {
- assertDocValues(leftReader.maxDoc(), leftNorms, rightNorms);
- } else {
- assertNull(info, leftNorms);
- assertNull(info, rightNorms);
- }
- }
- }
-
- /**
- * checks that stored fields of all documents are the same
- */
- public void assertStoredFields(IndexReader leftReader, IndexReader rightReader) throws Exception {
- assert leftReader.maxDoc() == rightReader.maxDoc();
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- StoredDocument leftDoc = leftReader.document(i);
- StoredDocument rightDoc = rightReader.document(i);
-
- // TODO: I think this is bogus because we don't document what the order should be
- // from these iterators, etc. I think the codec/IndexReader should be free to order this stuff
- // in whatever way it wants (e.g. maybe it packs related fields together or something)
- // To fix this, we sort the fields in both documents by name, but
- // we still assume that all instances with same name are in order:
- Comparator<StorableField> comp = new Comparator<StorableField>() {
- @Override
- public int compare(StorableField arg0, StorableField arg1) {
- return arg0.name().compareTo(arg1.name());
- }
- };
- Collections.sort(leftDoc.getFields(), comp);
- Collections.sort(rightDoc.getFields(), comp);
-
- Iterator<StorableField> leftIterator = leftDoc.iterator();
- Iterator<StorableField> rightIterator = rightDoc.iterator();
- while (leftIterator.hasNext()) {
- assertTrue(info, rightIterator.hasNext());
- assertStoredField(leftIterator.next(), rightIterator.next());
- }
- assertFalse(info, rightIterator.hasNext());
- }
- }
-
- /**
- * checks that two stored fields are equivalent
- */
- public void assertStoredField(StorableField leftField, StorableField rightField) {
- assertEquals(info, leftField.name(), rightField.name());
- assertEquals(info, leftField.binaryValue(), rightField.binaryValue());
- assertEquals(info, leftField.stringValue(), rightField.stringValue());
- assertEquals(info, leftField.numericValue(), rightField.numericValue());
- // TODO: should we check the FT at all?
- }
-
- /**
- * checks that term vectors across all fields are equivalent
- */
- public void assertTermVectors(IndexReader leftReader, IndexReader rightReader) throws Exception {
- assert leftReader.maxDoc() == rightReader.maxDoc();
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- Fields leftFields = leftReader.getTermVectors(i);
- Fields rightFields = rightReader.getTermVectors(i);
- assertFields(leftFields, rightFields, rarely());
- }
- }
-
- private static Set<String> getDVFields(IndexReader reader) {
- Set<String> fields = new HashSet<String>();
- for(FieldInfo fi : MultiFields.getMergedFieldInfos(reader)) {
- if (fi.hasDocValues()) {
- fields.add(fi.name);
- }
- }
-
- return fields;
- }
-
- /**
- * checks that docvalues across all fields are equivalent
- */
- public void assertDocValues(IndexReader leftReader, IndexReader rightReader) throws Exception {
- Set<String> leftFields = getDVFields(leftReader);
- Set<String> rightFields = getDVFields(rightReader);
- assertEquals(info, leftFields, rightFields);
-
- for (String field : leftFields) {
- // TODO: clean this up... very messy
- {
- NumericDocValues leftValues = MultiDocValues.getNumericValues(leftReader, field);
- NumericDocValues rightValues = MultiDocValues.getNumericValues(rightReader, field);
- if (leftValues != null && rightValues != null) {
- assertDocValues(leftReader.maxDoc(), leftValues, rightValues);
- } else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
- }
- }
-
- {
- BinaryDocValues leftValues = MultiDocValues.getBinaryValues(leftReader, field);
- BinaryDocValues rightValues = MultiDocValues.getBinaryValues(rightReader, field);
- if (leftValues != null && rightValues != null) {
- BytesRef scratchLeft = new BytesRef();
- BytesRef scratchRight = new BytesRef();
- for(int docID=0;docID<leftReader.maxDoc();docID++) {
- leftValues.get(docID, scratchLeft);
- rightValues.get(docID, scratchRight);
- assertEquals(info, scratchLeft, scratchRight);
- }
- } else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
- }
- }
-
- {
- SortedDocValues leftValues = MultiDocValues.getSortedValues(leftReader, field);
- SortedDocValues rightValues = MultiDocValues.getSortedValues(rightReader, field);
- if (leftValues != null && rightValues != null) {
- // numOrds
- assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
- // ords
- BytesRef scratchLeft = new BytesRef();
- BytesRef scratchRight = new BytesRef();
- for (int i = 0; i < leftValues.getValueCount(); i++) {
- leftValues.lookupOrd(i, scratchLeft);
- rightValues.lookupOrd(i, scratchRight);
- assertEquals(info, scratchLeft, scratchRight);
- }
- // bytes
- for(int docID=0;docID<leftReader.maxDoc();docID++) {
- leftValues.get(docID, scratchLeft);
- rightValues.get(docID, scratchRight);
- assertEquals(info, scratchLeft, scratchRight);
- }
- } else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
- }
- }
-
- {
- SortedSetDocValues leftValues = MultiDocValues.getSortedSetValues(leftReader, field);
- SortedSetDocValues rightValues = MultiDocValues.getSortedSetValues(rightReader, field);
- if (leftValues != null && rightValues != null) {
- // numOrds
- assertEquals(info, leftValues.getValueCount(), rightValues.getValueCount());
- // ords
- BytesRef scratchLeft = new BytesRef();
- BytesRef scratchRight = new BytesRef();
- for (int i = 0; i < leftValues.getValueCount(); i++) {
- leftValues.lookupOrd(i, scratchLeft);
- rightValues.lookupOrd(i, scratchRight);
- assertEquals(info, scratchLeft, scratchRight);
- }
- // ord lists
- for(int docID=0;docID<leftReader.maxDoc();docID++) {
- leftValues.setDocument(docID);
- rightValues.setDocument(docID);
- long ord;
- while ((ord = leftValues.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- assertEquals(info, ord, rightValues.nextOrd());
- }
- assertEquals(info, SortedSetDocValues.NO_MORE_ORDS, rightValues.nextOrd());
- }
- } else {
- assertNull(info, leftValues);
- assertNull(info, rightValues);
- }
- }
- }
- }
-
- public void assertDocValues(int num, NumericDocValues leftDocValues, NumericDocValues rightDocValues) throws Exception {
- assertNotNull(info, leftDocValues);
- assertNotNull(info, rightDocValues);
- for(int docID=0;docID<num;docID++) {
- assertEquals(leftDocValues.get(docID),
- rightDocValues.get(docID));
- }
- }
-
- // TODO: this is kinda stupid, we don't delete documents in the test.
- public void assertDeletedDocs(IndexReader leftReader, IndexReader rightReader) throws Exception {
- assert leftReader.numDeletedDocs() == rightReader.numDeletedDocs();
- Bits leftBits = MultiFields.getLiveDocs(leftReader);
- Bits rightBits = MultiFields.getLiveDocs(rightReader);
-
- if (leftBits == null || rightBits == null) {
- assertNull(info, leftBits);
- assertNull(info, rightBits);
- return;
- }
-
- assert leftReader.maxDoc() == rightReader.maxDoc();
- assertEquals(info, leftBits.length(), rightBits.length());
- for (int i = 0; i < leftReader.maxDoc(); i++) {
- assertEquals(info, leftBits.get(i), rightBits.get(i));
- }
- }
-
- public void assertFieldInfos(IndexReader leftReader, IndexReader rightReader) throws Exception {
- FieldInfos leftInfos = MultiFields.getMergedFieldInfos(leftReader);
- FieldInfos rightInfos = MultiFields.getMergedFieldInfos(rightReader);
-
- // TODO: would be great to verify more than just the names of the fields!
- TreeSet<String> left = new TreeSet<String>();
- TreeSet<String> right = new TreeSet<String>();
-
- for (FieldInfo fi : leftInfos) {
- left.add(fi.name);
- }
-
- for (FieldInfo fi : rightInfos) {
- right.add(fi.name);
- }
-
- assertEquals(info, left, right);
- }
-
-
- private static class RandomBits implements Bits {
- FixedBitSet bits;
-
- RandomBits(int maxDoc, double pctLive, Random random) {
- bits = new FixedBitSet(maxDoc);
- for (int i = 0; i < maxDoc; i++) {
- if (random.nextDouble() <= pctLive) {
- bits.set(i);
- }
- }
- }
-
- @Override
- public boolean get(int index) {
- return bits.get(index);
- }
-
- @Override
- public int length() {
- return bits.length();
- }
- }
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Thu May 30 07:53:18 2013
@@ -232,92 +232,4 @@ public class TestFieldsReader extends Lu
}
}
-
- public void testNumericField() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- final int numDocs = atLeast(500);
- final Number[] answers = new Number[numDocs];
- final NumericType[] typeAnswers = new NumericType[numDocs];
- for(int id=0;id<numDocs;id++) {
- Document doc = new Document();
- final Field nf;
- final Field sf;
- final Number answer;
- final NumericType typeAnswer;
- if (random().nextBoolean()) {
- // float/double
- if (random().nextBoolean()) {
- final float f = random().nextFloat();
- answer = Float.valueOf(f);
- nf = new FloatField("nf", f, Field.Store.NO);
- sf = new StoredField("nf", f);
- typeAnswer = NumericType.FLOAT;
- } else {
- final double d = random().nextDouble();
- answer = Double.valueOf(d);
- nf = new DoubleField("nf", d, Field.Store.NO);
- sf = new StoredField("nf", d);
- typeAnswer = NumericType.DOUBLE;
- }
- } else {
- // int/long
- if (random().nextBoolean()) {
- final int i = random().nextInt();
- answer = Integer.valueOf(i);
- nf = new IntField("nf", i, Field.Store.NO);
- sf = new StoredField("nf", i);
- typeAnswer = NumericType.INT;
- } else {
- final long l = random().nextLong();
- answer = Long.valueOf(l);
- nf = new LongField("nf", l, Field.Store.NO);
- sf = new StoredField("nf", l);
- typeAnswer = NumericType.LONG;
- }
- }
- doc.add(nf);
- doc.add(sf);
- answers[id] = answer;
- typeAnswers[id] = typeAnswer;
- FieldType ft = new FieldType(IntField.TYPE_STORED);
- ft.setNumericPrecisionStep(Integer.MAX_VALUE);
- doc.add(new IntField("id", id, ft));
- w.addDocument(doc);
- }
- final DirectoryReader r = w.getReader();
- w.close();
-
- assertEquals(numDocs, r.numDocs());
-
- for(AtomicReaderContext ctx : r.leaves()) {
- final AtomicReader sub = ctx.reader();
- final FieldCache.Ints ids = FieldCache.DEFAULT.getInts(sub, "id", false);
- for(int docID=0;docID<sub.numDocs();docID++) {
- final StoredDocument doc = sub.document(docID);
- final Field f = (Field) doc.getField("nf");
- assertTrue("got f=" + f, f instanceof StoredField);
- assertEquals(answers[ids.get(docID)], f.numericValue());
- }
- }
- r.close();
- dir.close();
- }
-
- public void testIndexedBit() throws Exception {
- Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), dir);
- Document doc = new Document();
- FieldType onlyStored = new FieldType();
- onlyStored.setStored(true);
- doc.add(new Field("field", "value", onlyStored));
- doc.add(new StringField("field2", "value", Field.Store.YES));
- w.addDocument(doc);
- IndexReader r = w.getReader();
- w.close();
- assertFalse(r.document(0).getField("field").fieldType().indexed());
- assertTrue(r.document(0).getField("field2").fieldType().indexed());
- r.close();
- dir.close();
- }
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java Thu May 30 07:53:18 2013
@@ -21,12 +21,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.store.Directory;
import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -159,21 +160,27 @@ public class TestFilterAtomicReader exte
directory.close();
target.close();
}
-
- private void checkOverrideMethods(Class<?> clazz) {
- boolean fail = false;
- for (Method m : clazz.getMethods()) {
- int mods = m.getModifiers();
- if (Modifier.isStatic(mods) || Modifier.isFinal(mods) || m.isSynthetic()) {
+
+ private static void checkOverrideMethods(Class<?> clazz) throws NoSuchMethodException, SecurityException {
+ final Class<?> superClazz = clazz.getSuperclass();
+ for (Method m : superClazz.getMethods()) {
+ final int mods = m.getModifiers();
+ if (Modifier.isStatic(mods) || Modifier.isAbstract(mods) || Modifier.isFinal(mods) || m.isSynthetic()
+ || m.getName().equals("attributes")) {
continue;
}
- Class<?> declaringClass = m.getDeclaringClass();
- if (declaringClass != clazz && declaringClass != Object.class) {
- System.err.println("method is not overridden by "+clazz.getName()+": " + m.toGenericString());
- fail = true;
+ // The point of these checks is to ensure that methods that have a default
+ // impl through other methods are not overridden. This makes the number of
+ // methods to override to have a working impl minimal and prevents from some
+ // traps: for example, think about having getCoreCacheKey delegate to the
+ // filtered impl by default
+ final Method subM = clazz.getMethod(m.getName(), m.getParameterTypes());
+ if (subM.getDeclaringClass() == clazz
+ && m.getDeclaringClass() != Object.class
+ && m.getDeclaringClass() != subM.getDeclaringClass()) {
+ fail(clazz + " overrides " + m + " although it has a default impl");
}
}
- assertFalse(clazz.getName()+" does not override some methods; see log above", fail);
}
public void testOverrideMethods() throws Exception {
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java Thu May 30 07:53:18 2013
@@ -60,7 +60,7 @@ public class TestForTooMuchCloning exten
//System.out.println("merge clone count=" + cloneCount);
assertTrue("too many calls to IndexInput.clone during merging: " + dir.getInputCloneCount(), cloneCount < 500);
- final IndexSearcher s = new IndexSearcher(r);
+ final IndexSearcher s = newSearcher(r);
// MTQ that matches all terms so the AUTO_REWRITE should
// cutover to filter rewrite and reuse a single DocsEnum
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java Thu May 30 07:53:18 2013
@@ -52,7 +52,9 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockFactory;
@@ -456,7 +458,7 @@ public class TestIndexWriter extends Luc
writer.close();
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
int totalHits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1).totalHits;
assertEquals(n*100, totalHits);
reader.close();
@@ -487,7 +489,7 @@ public class TestIndexWriter extends Luc
Term searchTerm = new Term("field", "aaa");
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals(10, hits.length);
reader.close();
@@ -509,7 +511,7 @@ public class TestIndexWriter extends Luc
}
writer.close();
reader = DirectoryReader.open(dir);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals(27, hits.length);
reader.close();
@@ -587,7 +589,7 @@ public class TestIndexWriter extends Luc
writer.close();
Term searchTerm = new Term("content", "aaa");
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("did not get right number of hits", 100, hits.length);
reader.close();
@@ -896,36 +898,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- // LUCENE-1219
- public void testBinaryFieldOffsetLength() throws IOException {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- byte[] b = new byte[50];
- for(int i=0;i<50;i++)
- b[i] = (byte) (i+77);
-
- Document doc = new Document();
- Field f = new StoredField("binary", b, 10, 17);
- byte[] bx = f.binaryValue().bytes;
- assertTrue(bx != null);
- assertEquals(50, bx.length);
- assertEquals(10, f.binaryValue().offset);
- assertEquals(17, f.binaryValue().length);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader ir = DirectoryReader.open(dir);
- StoredDocument doc2 = ir.document(0);
- StorableField f2 = doc2.getField("binary");
- b = f2.binaryValue().bytes;
- assertTrue(b != null);
- assertEquals(17, b.length, 17);
- assertEquals(87, b[0]);
- ir.close();
- dir.close();
- }
-
// LUCENE-2529
public void testPositionIncrementGapEmptyField() throws Exception {
Directory dir = newDirectory();
@@ -1211,6 +1183,46 @@ public class TestIndexWriter extends Luc
t.join();
assertFalse(t.failed);
}
+
+ /** testThreadInterruptDeadlock but with 2 indexer threads */
+ public void testTwoThreadsInterruptDeadlock() throws Exception {
+ IndexerThreadInterrupt t1 = new IndexerThreadInterrupt();
+ t1.setDaemon(true);
+ t1.start();
+
+ IndexerThreadInterrupt t2 = new IndexerThreadInterrupt();
+ t2.setDaemon(true);
+ t2.start();
+
+ // Force class loader to load ThreadInterruptedException
+ // up front... else we can see a false failure if 2nd
+ // interrupt arrives while class loader is trying to
+ // init this class (in servicing a first interrupt):
+ assertTrue(new ThreadInterruptedException(new InterruptedException()).getCause() instanceof InterruptedException);
+
+ // issue 300 interrupts to child thread
+ final int numInterrupts = atLeast(300);
+ int i = 0;
+ while(i < numInterrupts) {
+ // TODO: would be nice to also sometimes interrupt the
+ // CMS merge threads too ...
+ Thread.sleep(10);
+ IndexerThreadInterrupt t = random().nextBoolean() ? t1 : t2;
+ if (t.allowInterrupt) {
+ i++;
+ t.interrupt();
+ }
+ if (!t1.isAlive() && !t2.isAlive()) {
+ break;
+ }
+ }
+ t1.finish = true;
+ t2.finish = true;
+ t1.join();
+ t2.join();
+ assertFalse(t1.failed);
+ assertFalse(t2.failed);
+ }
public void testIndexStoreCombos() throws Exception {
@@ -1283,41 +1295,6 @@ public class TestIndexWriter extends Luc
}
- // LUCENE-1727: make sure doc fields are stored in order
- public void testStoredFieldsOrder() throws Throwable {
- Directory d = newDirectory();
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- Document doc = new Document();
-
- FieldType customType = new FieldType();
- customType.setStored(true);
- doc.add(newField("zzz", "a b c", customType));
- doc.add(newField("aaa", "a b c", customType));
- doc.add(newField("zzz", "1 2 3", customType));
- w.addDocument(doc);
- IndexReader r = w.getReader();
- StoredDocument doc2 = r.document(0);
- Iterator<StorableField> it = doc2.getFields().iterator();
- assertTrue(it.hasNext());
- Field f = (Field) it.next();
- assertEquals(f.name(), "zzz");
- assertEquals(f.stringValue(), "a b c");
-
- assertTrue(it.hasNext());
- f = (Field) it.next();
- assertEquals(f.name(), "aaa");
- assertEquals(f.stringValue(), "a b c");
-
- assertTrue(it.hasNext());
- f = (Field) it.next();
- assertEquals(f.name(), "zzz");
- assertEquals(f.stringValue(), "1 2 3");
- assertFalse(it.hasNext());
- r.close();
- w.close();
- d.close();
- }
-
public void testNoDocsIndex() throws Throwable {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
@@ -1436,10 +1413,10 @@ public class TestIndexWriter extends Luc
// Validates that iw.deleteUnusedFiles() also deletes unused index commits
// in case a deletion policy which holds onto commits is used.
Directory dir = newDirectory();
- SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random()))
- .setIndexDeletionPolicy(sdp));
+ .setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
+ SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
// First commit
Document doc = new Document();
@@ -1455,7 +1432,7 @@ public class TestIndexWriter extends Luc
assertEquals(1, DirectoryReader.listCommits(dir).size());
// Keep that commit
- sdp.snapshot("id");
+ IndexCommit id = sdp.snapshot();
// Second commit - now KeepOnlyLastCommit cannot delete the prev commit.
doc = new Document();
@@ -1465,7 +1442,7 @@ public class TestIndexWriter extends Luc
assertEquals(2, DirectoryReader.listCommits(dir).size());
// Should delete the unreferenced commit
- sdp.release("id");
+ sdp.release(id);
writer.deleteUnusedFiles();
assertEquals(1, DirectoryReader.listCommits(dir).size());
@@ -1542,7 +1519,7 @@ public class TestIndexWriter extends Luc
}
public void testNoSegmentFile() throws IOException {
- Directory dir = newDirectory();
+ BaseDirectoryWrapper dir = newDirectory();
dir.setLockFactory(NoLockFactory.getNoLockFactory());
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMaxBufferedDocs(2));
@@ -1562,6 +1539,10 @@ public class TestIndexWriter extends Luc
w2.close();
// If we don't do that, the test fails on Windows
w.rollback();
+
+ // This test leaves only segments.gen, which causes
+ // DirectoryReader.indexExists to return true:
+ dir.setCheckIndexOnClose(false);
dir.close();
}
@@ -2180,5 +2161,52 @@ public class TestIndexWriter extends Luc
}
}
-
+
+ // LUCENE-2727/LUCENE-2812/LUCENE-4738:
+ public void testCorruptFirstCommit() throws Exception {
+ for(int i=0;i<6;i++) {
+ BaseDirectoryWrapper dir = newDirectory();
+ dir.createOutput("segments_0", IOContext.DEFAULT).close();
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ int mode = i/2;
+ if (mode == 0) {
+ iwc.setOpenMode(OpenMode.CREATE);
+ } else if (mode == 1) {
+ iwc.setOpenMode(OpenMode.APPEND);
+ } else if (mode == 2) {
+ iwc.setOpenMode(OpenMode.CREATE_OR_APPEND);
+ }
+
+ if (VERBOSE) {
+ System.out.println("\nTEST: i=" + i);
+ }
+
+ try {
+ if ((i & 1) == 0) {
+ new IndexWriter(dir, iwc).close();
+ } else {
+ new IndexWriter(dir, iwc).rollback();
+ }
+ if (mode != 0) {
+ fail("expected exception");
+ }
+ } catch (IOException ioe) {
+ // OpenMode.APPEND should throw an exception since no
+ // index exists:
+ if (mode == 0) {
+ // Unexpected
+ throw ioe;
+ }
+ }
+
+ if (VERBOSE) {
+ System.out.println(" at close: " + Arrays.toString(dir.listAll()));
+ }
+
+ if (mode != 0) {
+ dir.setCheckIndexOnClose(false);
+ }
+ dir.close();
+ }
+ }
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java Thu May 30 07:53:18 2013
@@ -51,7 +51,7 @@ public class TestIndexWriterCommit exten
Term searchTerm = new Term("content", "aaa");
DirectoryReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("first number of hits", 14, hits.length);
reader.close();
@@ -64,7 +64,7 @@ public class TestIndexWriterCommit exten
TestIndexWriter.addDoc(writer);
}
IndexReader r = DirectoryReader.open(dir);
- searcher = new IndexSearcher(r);
+ searcher = newSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
r.close();
@@ -76,7 +76,7 @@ public class TestIndexWriterCommit exten
assertFalse("reader should not be current now", reader.isCurrent());
IndexReader r = DirectoryReader.open(dir);
- searcher = new IndexSearcher(r);
+ searcher = newSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader did not see changes after writer was closed", 47, hits.length);
r.close();
@@ -102,7 +102,7 @@ public class TestIndexWriterCommit exten
Term searchTerm = new Term("content", "aaa");
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("first number of hits", 14, hits.length);
reader.close();
@@ -116,7 +116,7 @@ public class TestIndexWriterCommit exten
writer.deleteDocuments(searchTerm);
reader = DirectoryReader.open(dir);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
reader.close();
@@ -127,7 +127,7 @@ public class TestIndexWriterCommit exten
TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");
reader = DirectoryReader.open(dir);
- searcher = new IndexSearcher(reader);
+ searcher = newSearcher(reader);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("saw changes after writer.abort", 14, hits.length);
reader.close();
@@ -148,7 +148,7 @@ public class TestIndexWriterCommit exten
TestIndexWriter.addDoc(writer);
}
IndexReader r = DirectoryReader.open(dir);
- searcher = new IndexSearcher(r);
+ searcher = newSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
r.close();
@@ -156,7 +156,7 @@ public class TestIndexWriterCommit exten
writer.close();
IndexReader r = DirectoryReader.open(dir);
- searcher = new IndexSearcher(r);
+ searcher = newSearcher(r);
hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
assertEquals("didn't see changes after close", 218, hits.length);
r.close();
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Thu May 30 07:53:18 2013
@@ -216,8 +216,26 @@ public class TestIndexWriterConfig exten
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
IndexWriterConfig clone = conf.clone();
- // Clone is shallow since not all parameters are cloneable.
- assertTrue(conf.getIndexDeletionPolicy() == clone.getIndexDeletionPolicy());
+ // Make sure parameters that can't be reused are cloned
+ IndexDeletionPolicy delPolicy = conf.delPolicy;
+ IndexDeletionPolicy delPolicyClone = clone.delPolicy;
+ assertTrue(delPolicy.getClass() == delPolicyClone.getClass() && (delPolicy != delPolicyClone || delPolicy.clone() == delPolicyClone.clone()));
+
+ FlushPolicy flushPolicy = conf.flushPolicy;
+ FlushPolicy flushPolicyClone = clone.flushPolicy;
+ assertTrue(flushPolicy.getClass() == flushPolicyClone.getClass() && (flushPolicy != flushPolicyClone || flushPolicy.clone() == flushPolicyClone.clone()));
+
+ DocumentsWriterPerThreadPool pool = conf.indexerThreadPool;
+ DocumentsWriterPerThreadPool poolClone = clone.indexerThreadPool;
+ assertTrue(pool.getClass() == poolClone.getClass() && (pool != poolClone || pool.clone() == poolClone.clone()));
+
+ MergePolicy mergePolicy = conf.mergePolicy;
+ MergePolicy mergePolicyClone = clone.mergePolicy;
+ assertTrue(mergePolicy.getClass() == mergePolicyClone.getClass() && (mergePolicy != mergePolicyClone || mergePolicy.clone() == mergePolicyClone.clone()));
+
+ MergeScheduler mergeSched = conf.mergeScheduler;
+ MergeScheduler mergeSchedClone = clone.mergeScheduler;
+ assertTrue(mergeSched.getClass() == mergeSchedClone.getClass() && (mergeSched != mergeSchedClone || mergeSched.clone() == mergeSchedClone.clone()));
conf.setMergeScheduler(new SerialMergeScheduler());
assertEquals(ConcurrentMergeScheduler.class, clone.getMergeScheduler().getClass());
@@ -231,30 +249,46 @@ public class TestIndexWriterConfig exten
assertEquals(KeepOnlyLastCommitDeletionPolicy.class, conf.getIndexDeletionPolicy().getClass());
conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(null));
assertEquals(SnapshotDeletionPolicy.class, conf.getIndexDeletionPolicy().getClass());
- conf.setIndexDeletionPolicy(null);
- assertEquals(KeepOnlyLastCommitDeletionPolicy.class, conf.getIndexDeletionPolicy().getClass());
+ try {
+ conf.setIndexDeletionPolicy(null);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
// Test MergeScheduler
assertEquals(ConcurrentMergeScheduler.class, conf.getMergeScheduler().getClass());
conf.setMergeScheduler(new SerialMergeScheduler());
assertEquals(SerialMergeScheduler.class, conf.getMergeScheduler().getClass());
- conf.setMergeScheduler(null);
- assertEquals(ConcurrentMergeScheduler.class, conf.getMergeScheduler().getClass());
+ try {
+ conf.setMergeScheduler(null);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
// Test Similarity:
// we shouldnt assert what the default is, just that its not null.
assertTrue(IndexSearcher.getDefaultSimilarity() == conf.getSimilarity());
conf.setSimilarity(new MySimilarity());
assertEquals(MySimilarity.class, conf.getSimilarity().getClass());
- conf.setSimilarity(null);
- assertTrue(IndexSearcher.getDefaultSimilarity() == conf.getSimilarity());
+ try {
+ conf.setSimilarity(null);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
// Test IndexingChain
assertTrue(DocumentsWriterPerThread.defaultIndexingChain == conf.getIndexingChain());
conf.setIndexingChain(new MyIndexingChain());
assertEquals(MyIndexingChain.class, conf.getIndexingChain().getClass());
- conf.setIndexingChain(null);
- assertTrue(DocumentsWriterPerThread.defaultIndexingChain == conf.getIndexingChain());
+ try {
+ conf.setIndexingChain(null);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
try {
conf.setMaxBufferedDeleteTerms(0);
@@ -324,8 +358,12 @@ public class TestIndexWriterConfig exten
assertEquals(TieredMergePolicy.class, conf.getMergePolicy().getClass());
conf.setMergePolicy(new LogDocMergePolicy());
assertEquals(LogDocMergePolicy.class, conf.getMergePolicy().getClass());
- conf.setMergePolicy(null);
- assertEquals(LogByteSizeMergePolicy.class, conf.getMergePolicy().getClass());
+ try {
+ conf.setMergePolicy(null);
+ fail();
+ } catch (IllegalArgumentException e) {
+ // ok
+ }
}
public void testLiveChangeToCFS() throws Exception {
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Thu May 30 07:53:18 2013
@@ -25,6 +25,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@@ -40,7 +42,6 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -303,6 +304,69 @@ public class TestIndexWriterDelete exten
modifier.close();
dir.close();
}
+
+
+ public void testDeleteAllNoDeadLock() throws IOException, InterruptedException {
+ Directory dir = newDirectory();
+ final RandomIndexWriter modifier = new RandomIndexWriter(random(), dir);
+ int numThreads = atLeast(2);
+ Thread[] threads = new Thread[numThreads];
+ final CountDownLatch latch = new CountDownLatch(1);
+ final CountDownLatch doneLatch = new CountDownLatch(numThreads);
+ for (int i = 0; i < numThreads; i++) {
+ final int offset = i;
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ int id = offset * 1000;
+ int value = 100;
+ try {
+ latch.await();
+ for (int i = 0; i < 1000; i++) {
+ Document doc = new Document();
+ doc.add(newTextField("content", "aaa", Field.Store.NO));
+ doc.add(newStringField("id", String.valueOf(id++), Field.Store.YES));
+ doc.add(newStringField("value", String.valueOf(value), Field.Store.NO));
+ doc.add(new NumericDocValuesField("dv", value));
+ modifier.addDocument(doc);
+ if (VERBOSE) {
+ System.out.println("\tThread["+offset+"]: add doc: " + id);
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ } finally {
+ doneLatch.countDown();
+ if (VERBOSE) {
+ System.out.println("\tThread["+offset+"]: done indexing" );
+ }
+ }
+ }
+ };
+ threads[i].start();
+ }
+ latch.countDown();
+ while(!doneLatch.await(1, TimeUnit.MILLISECONDS)) {
+ modifier.deleteAll();
+ if (VERBOSE) {
+ System.out.println("del all");
+ }
+ }
+
+ modifier.deleteAll();
+ for (Thread thread : threads) {
+ thread.join();
+ }
+
+ modifier.close();
+ DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(reader.maxDoc(), 0);
+ assertEquals(reader.numDocs(), 0);
+ assertEquals(reader.numDeletedDocs(), 0);
+ reader.close();
+
+ dir.close();
+ }
// test rollback of deleteAll()
public void testDeleteAllRollback() throws IOException {
@@ -407,7 +471,7 @@ public class TestIndexWriterDelete exten
private int getHitCount(Directory dir, Term term) throws IOException {
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
int hitCount = searcher.search(new TermQuery(term), null, 1000).totalHits;
reader.close();
return hitCount;
@@ -499,6 +563,7 @@ public class TestIndexWriterDelete exten
}
testName = "disk full during reader.close() @ " + thisDiskFree
+ " bytes";
+ dir.setRandomIOExceptionRateOnOpen(random().nextDouble()*0.01);
} else {
thisDiskFree = 0;
rate = 0.0;
@@ -506,6 +571,7 @@ public class TestIndexWriterDelete exten
System.out.println("\ncycle: same writer: unlimited disk space");
}
testName = "reader re-use after disk full";
+ dir.setRandomIOExceptionRateOnOpen(0.0);
}
dir.setMaxSizeInBytes(thisDiskFree);
@@ -549,6 +615,7 @@ public class TestIndexWriterDelete exten
final double randomIOExceptionRate = dir.getRandomIOExceptionRate();
final long maxSizeInBytes = dir.getMaxSizeInBytes();
dir.setRandomIOExceptionRate(0.0);
+ dir.setRandomIOExceptionRateOnOpen(0.0);
dir.setMaxSizeInBytes(0);
if (!success) {
// Must force the close else the writer can have
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu May 30 07:53:18 2013
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
@@ -1393,7 +1394,7 @@ public class TestIndexWriterExceptions e
final IndexReader r = w.getReader();
w.close();
- final IndexSearcher s = new IndexSearcher(r);
+ final IndexSearcher s = newSearcher(r);
PhraseQuery pq = new PhraseQuery();
pq.add(new Term("content", "silly"));
pq.add(new Term("content", "content"));
@@ -1473,7 +1474,7 @@ public class TestIndexWriterExceptions e
final IndexReader r = w.getReader();
w.close();
- final IndexSearcher s = new IndexSearcher(r);
+ final IndexSearcher s = newSearcher(r);
PhraseQuery pq = new PhraseQuery();
pq.add(new Term("content", "silly"));
pq.add(new Term("content", "content"));
@@ -1495,7 +1496,7 @@ public class TestIndexWriterExceptions e
if (doFail && name.startsWith("segments_")) {
StackTraceElement[] trace = new Exception().getStackTrace();
for (int i = 0; i < trace.length; i++) {
- if ("indexExists".equals(trace[i].getMethodName())) {
+ if ("read".equals(trace[i].getMethodName())) {
throw new UnsupportedOperationException("expected UOE");
}
}
@@ -1515,8 +1516,8 @@ public class TestIndexWriterExceptions e
new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
fail("should have gotten a UOE");
} catch (UnsupportedOperationException expected) {
-
}
+
uoe.doFail = false;
d.close();
}
@@ -1627,4 +1628,75 @@ public class TestIndexWriterExceptions e
iw.close();
dir.close();
}
+
+ // See LUCENE-4870 TooManyOpenFiles errors are thrown as
+ // FNFExceptions which can trigger data loss.
+ public void testTooManyFileException() throws Exception {
+
+ // Create failure that throws Too many open files exception randomly
+ MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
+
+ @Override
+ public MockDirectoryWrapper.Failure reset() {
+ doFail = false;
+ return this;
+ }
+
+ @Override
+ public void eval(MockDirectoryWrapper dir) throws IOException {
+ if (doFail) {
+ if (random().nextBoolean()) {
+ throw new FileNotFoundException("some/file/name.ext (Too many open files)");
+ }
+ }
+ }
+ };
+
+ MockDirectoryWrapper dir = newMockDirectory();
+ // The exception is only thrown on open input
+ dir.setFailOnOpenInput(true);
+ dir.failOn(failure);
+
+ // Create an index with one document
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ IndexWriter iw = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new StringField("foo", "bar", Field.Store.NO));
+ iw.addDocument(doc); // add a document
+ iw.commit();
+ DirectoryReader ir = DirectoryReader.open(dir);
+ assertEquals(1, ir.numDocs());
+ ir.close();
+ iw.close();
+
+ // Open and close the index a few times
+ for (int i = 0; i < 10; i++) {
+ failure.setDoFail();
+ iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
+ try {
+ iw = new IndexWriter(dir, iwc);
+ } catch (CorruptIndexException ex) {
+ // Exceptions are fine - we are running out of file handlers here
+ continue;
+ } catch (FileNotFoundException ex) {
+ continue;
+ }
+ failure.clearDoFail();
+ iw.close();
+ ir = DirectoryReader.open(dir);
+ assertEquals("lost document after iteration: " + i, 1, ir.numDocs());
+ ir.close();
+ }
+
+ // Check if document is still there
+ failure.clearDoFail();
+ ir = DirectoryReader.open(dir);
+ assertEquals(1, ir.numDocs());
+ ir.close();
+
+ dir.close();
+ }
+
+
+
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Thu May 30 07:53:18 2013
@@ -275,6 +275,7 @@ public class TestIndexWriterOnDiskFull e
String testName = null;
if (0 == x) {
+ dir.setRandomIOExceptionRateOnOpen(random().nextDouble()*0.01);
thisDiskFree = diskFree;
if (diskRatio >= 2.0) {
rate /= 2;
@@ -288,6 +289,7 @@ public class TestIndexWriterOnDiskFull e
if (VERBOSE)
testName = "disk full test " + methodName + " with disk full at " + diskFree + " bytes";
} else {
+ dir.setRandomIOExceptionRateOnOpen(0.0);
thisDiskFree = 0;
rate = 0.0;
if (VERBOSE)
@@ -363,6 +365,7 @@ public class TestIndexWriterOnDiskFull e
// we succeeded, we see all docs added, and if we
// failed, we see either all docs or no docs added
// (transactional semantics):
+ dir.setRandomIOExceptionRateOnOpen(0.0);
try {
reader = DirectoryReader.open(dir);
} catch (IOException e) {
@@ -432,6 +435,7 @@ public class TestIndexWriterOnDiskFull e
// Make sure we don't hit disk full during close below:
dir.setMaxSizeInBytes(0);
dir.setRandomIOExceptionRate(0.0);
+ dir.setRandomIOExceptionRateOnOpen(0.0);
writer.close();
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Thu May 30 07:53:18 2013
@@ -18,10 +18,11 @@ package org.apache.lucene.index;
*
*/
-import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.PrintStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
@@ -49,10 +50,6 @@ public class TestIndexWriterOnJRECrash e
@Override @Nightly
public void testNRTThreads() throws Exception {
- String vendor = Constants.JAVA_VENDOR;
- assumeTrue(vendor + " JRE not supported.",
- vendor.startsWith("Oracle") || vendor.startsWith("Sun") || vendor.startsWith("Apple"));
-
// if we are not the fork
if (System.getProperty("tests.crashmode") == null) {
// try up to 10 times to create an index
@@ -112,18 +109,40 @@ public class TestIndexWriterOnJRECrash e
pb.directory(tempDir);
pb.redirectErrorStream(true);
Process p = pb.start();
- InputStream is = p.getInputStream();
- BufferedInputStream isl = new BufferedInputStream(is);
- byte buffer[] = new byte[1024];
- int len = 0;
- if (VERBOSE) System.err.println(">>> Begin subprocess output");
- while ((len = isl.read(buffer)) != -1) {
- if (VERBOSE) {
- System.err.write(buffer, 0, len);
- }
- }
- if (VERBOSE) System.err.println("<<< End subprocess output");
+
+ // We pump everything to stderr.
+ PrintStream childOut = System.err;
+ Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
+ Thread stderrPumper = ThreadPumper.start(p.getErrorStream(), childOut);
+ if (VERBOSE) childOut.println(">>> Begin subprocess output");
p.waitFor();
+ stdoutPumper.join();
+ stderrPumper.join();
+ if (VERBOSE) childOut.println("<<< End subprocess output");
+ }
+
+ /** A pipe thread. It'd be nice to reuse guava's implementation for this... */
+ static class ThreadPumper {
+ public static Thread start(final InputStream from, final OutputStream to) {
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ byte [] buffer = new byte [1024];
+ int len;
+ while ((len = from.read(buffer)) != -1) {
+ if (VERBOSE) {
+ to.write(buffer, 0, len);
+ }
+ }
+ } catch (IOException e) {
+ System.err.println("Couldn't pipe from the forked process: " + e.toString());
+ }
+ }
+ };
+ t.start();
+ return t;
+ }
}
/**
@@ -138,7 +157,13 @@ public class TestIndexWriterOnJRECrash e
if (VERBOSE) {
System.err.println("Checking index: " + file);
}
- _TestUtil.checkIndex(dir);
+ // LUCENE-4738: if we crashed while writing first
+ // commit it's possible index will be corrupt (by
+ // design we don't try to be smart about this case
+ // since that too risky):
+ if (SegmentInfos.getLastCommitGeneration(dir) > 1) {
+ _TestUtil.checkIndex(dir);
+ }
dir.close();
return true;
}
@@ -149,20 +174,40 @@ public class TestIndexWriterOnJRECrash e
}
return false;
}
-
+
/**
* currently, this only works/tested on Sun and IBM.
*/
public void crashJRE() {
- try {
- Class<?> clazz = Class.forName("sun.misc.Unsafe");
- // we should use getUnsafe instead, harmony implements it, etc.
- Field field = clazz.getDeclaredField("theUnsafe");
- field.setAccessible(true);
- Object o = field.get(null);
- Method m = clazz.getMethod("putAddress", long.class, long.class);
- m.invoke(o, 0L, 0L);
- } catch (Exception e) { e.printStackTrace(); }
- fail();
+ final String vendor = Constants.JAVA_VENDOR;
+ final boolean supportsUnsafeNpeDereference =
+ vendor.startsWith("Oracle") ||
+ vendor.startsWith("Sun") ||
+ vendor.startsWith("Apple");
+
+ try {
+ if (supportsUnsafeNpeDereference) {
+ try {
+ Class<?> clazz = Class.forName("sun.misc.Unsafe");
+ Field field = clazz.getDeclaredField("theUnsafe");
+ field.setAccessible(true);
+ Object o = field.get(null);
+ Method m = clazz.getMethod("putAddress", long.class, long.class);
+ m.invoke(o, 0L, 0L);
+ } catch (Throwable e) {
+ System.out.println("Couldn't kill the JVM via Unsafe.");
+ e.printStackTrace(System.out);
+ }
+ }
+
+ // Fallback attempt to Runtime.halt();
+ Runtime.getRuntime().halt(-1);
+ } catch (Exception e) {
+ System.out.println("Couldn't kill the JVM.");
+ e.printStackTrace(System.out);
+ }
+
+ // We couldn't get the JVM to crash for some reason.
+ fail();
}
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java Thu May 30 07:53:18 2013
@@ -39,6 +39,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util._TestUtil;
@@ -965,6 +966,46 @@ public class TestIndexWriterReader exten
assertTrue(didWarm.get());
}
+ public void testSimpleMergedSegmentWramer() throws Exception {
+ Directory dir = newDirectory();
+ final AtomicBoolean didWarm = new AtomicBoolean();
+ InfoStream infoStream = new InfoStream() {
+ @Override
+ public void close() throws IOException {}
+
+ @Override
+ public void message(String component, String message) {
+ if ("SMSW".equals(component)) {
+ didWarm.set(true);
+ }
+ }
+
+ @Override
+ public boolean isEnabled(String component) {
+ return true;
+ }
+ };
+ IndexWriter w = new IndexWriter(
+ dir,
+ newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+ setMaxBufferedDocs(2).
+ setReaderPooling(true).
+ setInfoStream(infoStream).
+ setMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream)).
+ setMergePolicy(newLogMergePolicy(10))
+ );
+
+ Document doc = new Document();
+ doc.add(newStringField("foo", "bar", Field.Store.NO));
+ for(int i=0;i<20;i++) {
+ w.addDocument(doc);
+ }
+ w.waitForMerges();
+ w.close();
+ dir.close();
+ assertTrue(didWarm.get());
+ }
+
public void testNoTermsIndex() throws Exception {
// Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if
// they're picked.
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Thu May 30 07:53:18 2013
@@ -282,7 +282,7 @@ public class TestIndexableField extends
final IndexReader r = w.getReader();
w.close();
- final IndexSearcher s = new IndexSearcher(r);
+ final IndexSearcher s = newSearcher(r);
int counter = 0;
for(int id=0;id<NUM_DOCS;id++) {
if (VERBOSE) {
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestOmitTf.java Thu May 30 07:53:18 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.concurrent.ExecutionException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -281,7 +282,7 @@ public class TestOmitTf extends LuceneTe
* Verify the index
*/
IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
searcher.setSimilarity(new SimpleSimilarity());
Term a = new Term("noTf", term);
@@ -299,8 +300,13 @@ public class TestOmitTf extends LuceneTe
try {
searcher.search(pq, 10);
fail("did not hit expected exception");
- } catch (IllegalStateException ise) {
- // expected
+ } catch (Exception e) {
+ Throwable cause = e;
+ // If the searcher uses an executor service, the IAE is wrapped into other exceptions
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+ assertTrue("Expected an IAE, got " + cause, cause instanceof IllegalStateException);
}
searcher.search(q1,
@@ -439,9 +445,8 @@ public class TestOmitTf extends LuceneTe
iw.addDocument(doc);
IndexReader ir = iw.getReader();
iw.close();
- Terms terms = MultiFields.getTerms(ir, "foo");
- assertEquals(-1, MultiFields.totalTermFreq(ir, "foo", new BytesRef("bar")));
- assertEquals(-1, terms.getSumTotalTermFreq());
+ assertEquals(-1, ir.totalTermFreq(new Term("foo", new BytesRef("bar"))));
+ assertEquals(-1, ir.getSumTotalTermFreq("foo"));
ir.close();
dir.close();
}
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelAtomicReader.java Thu May 30 07:53:18 2013
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.*;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -114,6 +115,29 @@ public class TestParallelAtomicReader ex
dir2.close();
}
+ public void testCloseInnerReader() throws Exception {
+ Directory dir1 = getDir1(random());
+ AtomicReader ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir1));
+
+ // with overlapping
+ ParallelAtomicReader pr = new ParallelAtomicReader(true,
+ new AtomicReader[] {ir1},
+ new AtomicReader[] {ir1});
+
+ ir1.close();
+
+ try {
+ pr.document(0);
+ fail("ParallelAtomicReader should be already closed because inner reader was closed!");
+ } catch (AlreadyClosedException e) {
+ // pass
+ }
+
+ // noop:
+ pr.close();
+ dir1.close();
+ }
+
public void testIncompatibleIndexes() throws IOException {
// two documents:
Directory dir1 = getDir1(random());
Modified: lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java?rev=1487777&r1=1487776&r2=1487777&view=diff
==============================================================================
--- lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java (original)
+++ lucene/dev/branches/security/lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java Thu May 30 07:53:18 2013
@@ -23,8 +23,10 @@ import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader.ReaderClosedListener;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.*;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -82,12 +84,15 @@ public class TestParallelCompositeReader
// close subreaders, ParallelReader will not change refCounts, but close on its own close
ParallelCompositeReader pr = new ParallelCompositeReader(ir1 = DirectoryReader.open(dir1),
ir2 = DirectoryReader.open(dir2));
+ IndexReader psub1 = pr.getSequentialSubReaders().get(0);
// check RefCounts
assertEquals(1, ir1.getRefCount());
assertEquals(1, ir2.getRefCount());
+ assertEquals(1, psub1.getRefCount());
pr.close();
assertEquals(0, ir1.getRefCount());
assertEquals(0, ir2.getRefCount());
+ assertEquals(0, psub1.getRefCount());
dir1.close();
dir2.close();
}
@@ -100,20 +105,115 @@ public class TestParallelCompositeReader
// don't close subreaders, so ParallelReader will increment refcounts
ParallelCompositeReader pr = new ParallelCompositeReader(false, ir1, ir2);
+ IndexReader psub1 = pr.getSequentialSubReaders().get(0);
// check RefCounts
assertEquals(2, ir1.getRefCount());
assertEquals(2, ir2.getRefCount());
+ assertEquals("refCount must be 1, as the synthetic reader was created by ParallelCompositeReader", 1, psub1.getRefCount());
pr.close();
assertEquals(1, ir1.getRefCount());
assertEquals(1, ir2.getRefCount());
+ assertEquals("refcount must be 0 because parent was closed", 0, psub1.getRefCount());
ir1.close();
ir2.close();
assertEquals(0, ir1.getRefCount());
assertEquals(0, ir2.getRefCount());
+ assertEquals("refcount should not change anymore", 0, psub1.getRefCount());
dir1.close();
dir2.close();
}
+ // closeSubreaders=false
+ public void testReaderClosedListener1() throws Exception {
+ Directory dir1 = getDir1(random());
+ CompositeReader ir1 = DirectoryReader.open(dir1);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(false,
+ new CompositeReader[] {ir1},
+ new CompositeReader[] {ir1});
+
+ final int[] listenerClosedCount = new int[1];
+
+ assertEquals(3, pr.leaves().size());
+
+ for(AtomicReaderContext cxt : pr.leaves()) {
+ cxt.reader().addReaderClosedListener(new ReaderClosedListener() {
+ @Override
+ public void onClose(IndexReader reader) {
+ listenerClosedCount[0]++;
+ }
+ });
+ }
+ pr.close();
+ ir1.close();
+ assertEquals(3, listenerClosedCount[0]);
+ dir1.close();
+ }
+
+ // closeSubreaders=true
+ public void testReaderClosedListener2() throws Exception {
+ Directory dir1 = getDir1(random());
+ CompositeReader ir1 = DirectoryReader.open(dir1);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(true,
+ new CompositeReader[] {ir1},
+ new CompositeReader[] {ir1});
+
+ final int[] listenerClosedCount = new int[1];
+
+ assertEquals(3, pr.leaves().size());
+
+ for(AtomicReaderContext cxt : pr.leaves()) {
+ cxt.reader().addReaderClosedListener(new ReaderClosedListener() {
+ @Override
+ public void onClose(IndexReader reader) {
+ listenerClosedCount[0]++;
+ }
+ });
+ }
+ pr.close();
+ assertEquals(3, listenerClosedCount[0]);
+ dir1.close();
+ }
+
+ public void testCloseInnerReader() throws Exception {
+ Directory dir1 = getDir1(random());
+ CompositeReader ir1 = DirectoryReader.open(dir1);
+ assertEquals(1, ir1.getSequentialSubReaders().get(0).getRefCount());
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(true,
+ new CompositeReader[] {ir1},
+ new CompositeReader[] {ir1});
+
+ IndexReader psub = pr.getSequentialSubReaders().get(0);
+ assertEquals(1, psub.getRefCount());
+
+ ir1.close();
+
+ assertEquals("refCount of synthetic subreader should be unchanged", 1, psub.getRefCount());
+ try {
+ psub.document(0);
+ fail("Subreader should be already closed because inner reader was closed!");
+ } catch (AlreadyClosedException e) {
+ // pass
+ }
+
+ try {
+ pr.document(0);
+ fail("ParallelCompositeReader should be already closed because inner reader was closed!");
+ } catch (AlreadyClosedException e) {
+ // pass
+ }
+
+ // noop:
+ pr.close();
+ assertEquals(0, psub.getRefCount());
+ dir1.close();
+ }
+
public void testIncompatibleIndexes1() throws IOException {
// two documents:
Directory dir1 = getDir1(random());
@@ -277,6 +377,30 @@ public class TestParallelCompositeReader
dir2.close();
}
+ public void testToString() throws IOException {
+ Directory dir1 = getDir1(random());
+ CompositeReader ir1 = DirectoryReader.open(dir1);
+ ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] {ir1});
+
+ final String s = pr.toString();
+ assertTrue("toString incorrect: " + s, s.startsWith("ParallelCompositeReader(ParallelAtomicReader("));
+
+ pr.close();
+ dir1.close();
+ }
+
+ public void testToStringCompositeComposite() throws IOException {
+ Directory dir1 = getDir1(random());
+ CompositeReader ir1 = DirectoryReader.open(dir1);
+ ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] {new MultiReader(ir1)});
+
+ final String s = pr.toString();
+ assertTrue("toString incorrect: " + s, s.startsWith("ParallelCompositeReader(ParallelCompositeReader(ParallelAtomicReader("));
+
+ pr.close();
+ dir1.close();
+ }
+
private void queryTest(Query query) throws IOException {
ScoreDoc[] parallelHits = parallel.search(query, null, 1000).scoreDocs;
ScoreDoc[] singleHits = single.search(query, null, 1000).scoreDocs;