You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/28 10:19:29 UTC
svn commit: r1634823 [7/9] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/
lucene/analysis/common/src/java/org/apache/lucene/analy...
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Tue Oct 28 09:19:21 2014
@@ -167,7 +167,8 @@ public abstract class SorterTestBase ext
}
protected static Directory dir;
- protected static LeafReader reader;
+ protected static LeafReader unsortedReader;
+ protected static LeafReader sortedReader;
protected static Integer[] sortedValues;
private static Document doc(final int id, PositionsTokenStream positions) {
@@ -190,8 +191,8 @@ public abstract class SorterTestBase ext
return doc;
}
- /** Creates an index for sorting. */
- public static void createIndex(Directory dir, int numDocs, Random random) throws IOException {
+ /** Creates an unsorted index; subclasses then sort this index and open sortedReader. */
+ private static void createIndex(Directory dir, int numDocs, Random random) throws IOException {
List<Integer> ids = new ArrayList<>();
for (int i = 0; i < numDocs; i++) {
ids.add(Integer.valueOf(i * 10));
@@ -230,19 +231,20 @@ public abstract class SorterTestBase ext
int numDocs = atLeast(20);
createIndex(dir, numDocs, random());
- reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
+ unsortedReader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
}
@AfterClass
public static void afterClassSorterTestBase() throws Exception {
- reader.close();
+ unsortedReader.close();
+ sortedReader.close();
dir.close();
}
@Test
public void testBinaryDocValuesField() throws Exception {
- BinaryDocValues dv = reader.getBinaryDocValues(BINARY_DV_FIELD);
- for (int i = 0; i < reader.maxDoc(); i++) {
+ BinaryDocValues dv = sortedReader.getBinaryDocValues(BINARY_DV_FIELD);
+ for (int i = 0; i < sortedReader.maxDoc(); i++) {
final BytesRef bytes = dv.get(i);
assertEquals("incorrect binary DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
}
@@ -250,7 +252,7 @@ public abstract class SorterTestBase ext
@Test
public void testDocsAndPositionsEnum() throws Exception {
- TermsEnum termsEnum = reader.terms(DOC_POSITIONS_FIELD).iterator(null);
+ TermsEnum termsEnum = sortedReader.terms(DOC_POSITIONS_FIELD).iterator(null);
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(DOC_POSITIONS_TERM)));
DocsAndPositionsEnum sortedPositions = termsEnum.docsAndPositions(null, null);
int doc;
@@ -310,8 +312,8 @@ public abstract class SorterTestBase ext
@Test
public void testDocsEnum() throws Exception {
- Bits mappedLiveDocs = randomLiveDocs(reader.maxDoc());
- TermsEnum termsEnum = reader.terms(DOCS_ENUM_FIELD).iterator(null);
+ Bits mappedLiveDocs = randomLiveDocs(sortedReader.maxDoc());
+ TermsEnum termsEnum = sortedReader.terms(DOCS_ENUM_FIELD).iterator(null);
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(DOCS_ENUM_TERM)));
DocsEnum docs = termsEnum.docs(mappedLiveDocs, null);
@@ -319,12 +321,12 @@ public abstract class SorterTestBase ext
int prev = -1;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
assertTrue("document " + doc + " marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(doc));
- assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(sortedReader.document(doc).get(ID_FIELD)));
while (++prev < doc) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
- while (++prev < reader.maxDoc()) {
+ while (++prev < sortedReader.maxDoc()) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
@@ -337,20 +339,20 @@ public abstract class SorterTestBase ext
prev = -1;
while ((doc = docs.advance(doc + 1)) != DocIdSetIterator.NO_MORE_DOCS) {
assertTrue("document " + doc + " marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(doc));
- assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(sortedReader.document(doc).get(ID_FIELD)));
while (++prev < doc) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
- while (++prev < reader.maxDoc()) {
+ while (++prev < sortedReader.maxDoc()) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
@Test
public void testNormValues() throws Exception {
- NumericDocValues dv = reader.getNormValues(NORMS_FIELD);
- int maxDoc = reader.maxDoc();
+ NumericDocValues dv = sortedReader.getNormValues(NORMS_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
assertEquals("incorrect norm value for doc " + i, sortedValues[i].intValue(), dv.get(i));
}
@@ -358,8 +360,8 @@ public abstract class SorterTestBase ext
@Test
public void testNumericDocValuesField() throws Exception {
- NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ NumericDocValues dv = sortedReader.getNumericDocValues(NUMERIC_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
assertEquals("incorrect numeric DocValues for doc " + i, sortedValues[i].intValue(), dv.get(i));
}
@@ -367,8 +369,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedDocValuesField() throws Exception {
- SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedDocValues dv = sortedReader.getSortedDocValues(SORTED_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
final BytesRef bytes = dv.get(i);
assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
@@ -377,8 +379,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedSetDocValuesField() throws Exception {
- SortedSetDocValues dv = reader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedSetDocValues dv = sortedReader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
dv.setDocument(i);
BytesRef bytes = dv.lookupOrd(dv.nextOrd());
@@ -392,8 +394,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedNumericDocValuesField() throws Exception {
- SortedNumericDocValues dv = reader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedNumericDocValues dv = sortedReader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
dv.setDocument(i);
assertEquals(2, dv.count());
@@ -405,9 +407,9 @@ public abstract class SorterTestBase ext
@Test
public void testTermVectors() throws Exception {
- int maxDoc = reader.maxDoc();
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
- Terms terms = reader.getTermVector(i, TERM_VECTORS_FIELD);
+ Terms terms = sortedReader.getTermVector(i, TERM_VECTORS_FIELD);
assertNotNull("term vectors not found for doc " + i + " field [" + TERM_VECTORS_FIELD + "]", terms);
assertEquals("incorrect term vector for doc " + i, sortedValues[i].toString(), terms.iterator(null).next().utf8ToString());
}
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java Tue Oct 28 09:19:21 2014
@@ -30,15 +30,16 @@ public class SortingLeafReaderTest exten
@BeforeClass
public static void beforeClassSortingLeafReaderTest() throws Exception {
+ // NOTE: index was created by by super's @BeforeClass
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortField.Type.INT));
- final Sorter.DocMap docMap = new Sorter(sort).sort(reader);
+ final Sorter.DocMap docMap = new Sorter(sort).sort(unsortedReader);
// Sorter.compute also sorts the values
- NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD);
- sortedValues = new Integer[reader.maxDoc()];
- for (int i = 0; i < reader.maxDoc(); ++i) {
+ NumericDocValues dv = unsortedReader.getNumericDocValues(NUMERIC_DV_FIELD);
+ sortedValues = new Integer[unsortedReader.maxDoc()];
+ for (int i = 0; i < unsortedReader.maxDoc(); ++i) {
sortedValues[docMap.oldToNew(i)] = (int)dv.get(i);
}
if (VERBOSE) {
@@ -47,11 +48,11 @@ public class SortingLeafReaderTest exten
}
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
- reader = SortingLeafReader.wrap(reader, sort);
+ sortedReader = SortingLeafReader.wrap(unsortedReader, sort);
if (VERBOSE) {
System.out.print("mapped-deleted-docs: ");
- Bits mappedLiveDocs = reader.getLiveDocs();
+ Bits mappedLiveDocs = sortedReader.getLiveDocs();
for (int i = 0; i < mappedLiveDocs.length(); i++) {
if (!mappedLiveDocs.get(i)) {
System.out.print(i + " ");
@@ -60,12 +61,12 @@ public class SortingLeafReaderTest exten
System.out.println();
}
- TestUtil.checkReader(reader);
+ TestUtil.checkReader(sortedReader);
}
public void testBadSort() throws Exception {
try {
- SortingLeafReader.wrap(reader, Sort.RELEVANCE);
+ SortingLeafReader.wrap(sortedReader, Sort.RELEVANCE);
fail("Didn't get expected exception");
} catch (IllegalArgumentException e) {
assertEquals("Cannot sort an index with a Sort that refers to the relevance score", e.getMessage());
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/index/sorter/TestBlockJoinSorter.java Tue Oct 28 09:19:21 2014
@@ -41,6 +41,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -57,7 +58,7 @@ public class TestBlockJoinSorter extends
throws IOException {
final FixedBitSet cached = new FixedBitSet(reader.maxDoc());
cached.or(iterator);
- return cached;
+ return new BitDocIdSet(cached);
}
}
@@ -90,7 +91,7 @@ public class TestBlockJoinSorter extends
final LeafReader reader = getOnlySegmentReader(indexReader);
final Filter parentsFilter = new FixedBitSetCachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("parent", "true"))));
- final FixedBitSet parentBits = (FixedBitSet) parentsFilter.getDocIdSet(reader.getContext(), null);
+ final FixedBitSet parentBits = (FixedBitSet) parentsFilter.getDocIdSet(reader.getContext(), null).bits();
final NumericDocValues parentValues = reader.getNumericDocValues("parent_val");
final NumericDocValues childValues = reader.getNumericDocValues("child_val");
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestDocTermOrds.java Tue Oct 28 09:19:21 2014
@@ -27,7 +27,6 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
@@ -129,7 +128,7 @@ public class TestDocTermOrds extends Luc
// Sometimes swap in codec that impls ord():
if (random().nextInt(10) == 7) {
// Make sure terms index has ords:
- Codec codec = TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene41WithOrds"));
+ Codec codec = TestUtil.alwaysPostingsFormat(TestUtil.getPostingsFormatWithOrds(random()));
conf.setCodec(codec);
}
@@ -226,7 +225,7 @@ public class TestDocTermOrds extends Luc
// Sometimes swap in codec that impls ord():
if (random().nextInt(10) == 7) {
- Codec codec = TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene41WithOrds"));
+ Codec codec = TestUtil.alwaysPostingsFormat(TestUtil.getPostingsFormatWithOrds(random()));
conf.setCodec(codec);
}
Modified: lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java (original)
+++ lucene/dev/branches/lucene6005/lucene/misc/src/test/org/apache/lucene/uninverting/TestFieldCacheSortRandom.java Tue Oct 28 09:19:21 2014
@@ -52,6 +52,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -290,7 +291,7 @@ public class TestFieldCacheSortRandom ex
}
}
- return bits;
+ return new BitDocIdSet(bits);
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java Tue Oct 28 09:19:21 2014
@@ -30,7 +30,7 @@ import org.apache.lucene.search.DocIdSet
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.DocIdSetBuilder;
/**
* A container Filter that allows Boolean composition of Filters.
@@ -51,7 +51,7 @@ public class BooleanFilter extends Filte
*/
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) throws IOException {
- FixedBitSet res = null;
+ DocIdSetBuilder res = null;
final LeafReader reader = context.reader();
boolean hasShouldClauses = false;
@@ -61,7 +61,7 @@ public class BooleanFilter extends Filte
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi == null) continue;
if (res == null) {
- res = new FixedBitSet(reader.maxDoc());
+ res = new DocIdSetBuilder(reader.maxDoc());
}
res.or(disi);
}
@@ -73,8 +73,7 @@ public class BooleanFilter extends Filte
if (fc.getOccur() == Occur.MUST_NOT) {
if (res == null) {
assert !hasShouldClauses;
- res = new FixedBitSet(reader.maxDoc());
- res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs
+ res = new DocIdSetBuilder(reader.maxDoc(), true); // NOTE: may set bits on deleted docs
}
final DocIdSetIterator disi = getDISI(fc.getFilter(), context);
if (disi != null) {
@@ -90,7 +89,7 @@ public class BooleanFilter extends Filte
return null; // no documents can match
}
if (res == null) {
- res = new FixedBitSet(reader.maxDoc());
+ res = new DocIdSetBuilder(reader.maxDoc());
res.or(disi);
} else {
res.and(disi);
@@ -98,7 +97,10 @@ public class BooleanFilter extends Filte
}
}
- return BitsFilteredDocIdSet.wrap(res, acceptDocs);
+ if (res == null) {
+ return null;
+ }
+ return BitsFilteredDocIdSet.wrap(res.build(), acceptDocs);
}
private static DocIdSetIterator getDISI(Filter filter, LeafReaderContext context)
Modified: lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java Tue Oct 28 09:19:21 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.queries;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
@@ -27,19 +29,18 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.TermRangeFilter;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.QueryWrapperFilter;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermRangeFilter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
-import java.io.IOException;
-
public class BooleanFilterTest extends LuceneTestCase {
private Directory directory;
private LeafReader reader;
@@ -93,7 +94,7 @@ public class BooleanFilterTest extends L
return new Filter() {
@Override
public DocIdSet getDocIdSet(LeafReaderContext context, Bits acceptDocs) {
- return new FixedBitSet(context.reader().maxDoc());
+ return new BitDocIdSet(new FixedBitSet(context.reader().maxDoc()));
}
};
}
Modified: lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java (original)
+++ lucene/dev/branches/lucene6005/lucene/queries/src/test/org/apache/lucene/queries/TermsFilterTest.java Tue Oct 28 09:19:21 2014
@@ -47,6 +47,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -81,20 +82,20 @@ public class TermsFilterTest extends Luc
List<Term> terms = new ArrayList<>();
terms.add(new Term(fieldName, "19"));
- FixedBitSet bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
+ BitDocIdSet bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
assertNull("Must match nothing", bits);
terms.add(new Term(fieldName, "20"));
- bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
- assertEquals("Must match 1", 1, bits.cardinality());
+ bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
+ assertEquals("Must match 1", 1, bits.bits().cardinality());
terms.add(new Term(fieldName, "10"));
- bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
- assertEquals("Must match 2", 2, bits.cardinality());
+ bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
+ assertEquals("Must match 2", 2, bits.bits().cardinality());
terms.add(new Term(fieldName, "00"));
- bits = (FixedBitSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
- assertEquals("Must match 2", 2, bits.cardinality());
+ bits = (BitDocIdSet) termsFilter(random().nextBoolean(), terms).getDocIdSet(context, context.reader().getLiveDocs());
+ assertEquals("Must match 2", 2, bits.bits().cardinality());
reader.close();
rd.close();
@@ -126,8 +127,8 @@ public class TermsFilterTest extends Luc
if (context.reader().docFreq(new Term(fieldName, "content1")) == 0) {
assertNull(docIdSet);
} else {
- FixedBitSet bits = (FixedBitSet) docIdSet;
- assertTrue("Must be >= 0", bits.cardinality() >= 0);
+ BitDocIdSet bits = (BitDocIdSet) docIdSet;
+ assertTrue("Must be >= 0", bits.bits().cardinality() >= 0);
}
}
multi.close();
@@ -163,8 +164,8 @@ public class TermsFilterTest extends Luc
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(terms);
- FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
- assertEquals("Must be num fields - 1 since we skip only one field", num-1, bits.cardinality());
+ BitDocIdSet bits = (BitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
+ assertEquals("Must be num fields - 1 since we skip only one field", num-1, bits.bits().cardinality());
reader.close();
dir.close();
}
@@ -199,8 +200,8 @@ public class TermsFilterTest extends Luc
LeafReaderContext context = reader.leaves().get(0);
TermsFilter tf = new TermsFilter(new ArrayList<>(terms));
- FixedBitSet bits = (FixedBitSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
- assertEquals(context.reader().numDocs(), bits.cardinality());
+ BitDocIdSet bits = (BitDocIdSet) tf.getDocIdSet(context, context.reader().getLiveDocs());
+ assertEquals(context.reader().numDocs(), bits.bits().cardinality());
reader.close();
dir.close();
}
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsFormat.java Tue Oct 28 09:19:21 2014
@@ -81,7 +81,7 @@ public class IDVersionPostingsFormat ext
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new IDVersionPostingsWriter(state);
+ PostingsWriterBase postingsWriter = new IDVersionPostingsWriter(state.liveDocs);
boolean success = false;
try {
FieldsConsumer ret = new VersionBlockTreeTermsWriter(state,
@@ -99,7 +99,7 @@ public class IDVersionPostingsFormat ext
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new IDVersionPostingsReader(state);
+ PostingsReaderBase postingsReader = new IDVersionPostingsReader();
boolean success = false;
try {
FieldsProducer ret = new VersionBlockTreeTermsReader(postingsReader, state);
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java Tue Oct 28 09:19:21 2014
@@ -33,16 +33,11 @@ import org.apache.lucene.util.Accountabl
import org.apache.lucene.util.Bits;
final class IDVersionPostingsReader extends PostingsReaderBase {
- final SegmentReadState state;
-
- public IDVersionPostingsReader(SegmentReadState state) {
- this.state = state;
- }
@Override
- public void init(IndexInput termsIn) throws IOException {
+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
// Make sure we are talking to the matching postings writer
- CodecUtil.checkSegmentHeader(termsIn,
+ CodecUtil.checkIndexHeader(termsIn,
IDVersionPostingsWriter.TERMS_CODEC,
IDVersionPostingsWriter.VERSION_START,
IDVersionPostingsWriter.VERSION_CURRENT,
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java Tue Oct 28 09:19:21 2014
@@ -26,6 +26,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
final class IDVersionPostingsWriter extends PushPostingsWriterBase {
@@ -43,10 +44,10 @@ final class IDVersionPostingsWriter exte
private int lastPosition;
private long lastVersion;
- private final SegmentWriteState state;
+ private final Bits liveDocs;
- public IDVersionPostingsWriter(SegmentWriteState state) {
- this.state = state;
+ public IDVersionPostingsWriter(Bits liveDocs) {
+ this.liveDocs = liveDocs;
}
@Override
@@ -55,8 +56,8 @@ final class IDVersionPostingsWriter exte
}
@Override
- public void init(IndexOutput termsOut) throws IOException {
- CodecUtil.writeSegmentHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
+ CodecUtil.writeIndexHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
}
@Override
@@ -82,7 +83,7 @@ final class IDVersionPostingsWriter exte
@Override
public void startDoc(int docID, int termDocFreq) throws IOException {
// TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush:
- if (state.liveDocs != null && state.liveDocs.get(docID) == false) {
+ if (liveDocs != null && liveDocs.get(docID) == false) {
return;
}
if (lastDocID != -1) {
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java Tue Oct 28 09:19:21 2014
@@ -631,7 +631,7 @@ public final class IDVersionSegmentTerms
int cmp = 0;
- // TOOD: we should write our vLong backwards (MSB
+ // TODO: we should write our vLong backwards (MSB
// first) to get better sharing from the FST
// First compare up to valid seek frames:
@@ -645,7 +645,7 @@ public final class IDVersionSegmentTerms
}
arc = arcs[1+targetUpto];
assert arc.label == (target.bytes[target.offset + targetUpto] & 0xFF): "arc.label=" + (char) arc.label + " targetLabel=" + (char) (target.bytes[target.offset + targetUpto] & 0xFF);
- // TOOD: we could save the outputs in local
+ // TODO: we could save the outputs in local
// byte[][] instead of making new objs ever
// seek; but, often the FST doesn't have any
// shared bytes (but this could change if we
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsReader.java Tue Oct 28 09:19:21 2014
@@ -72,7 +72,7 @@ public final class VersionBlockTreeTerms
IndexInput indexIn = null;
try {
- int termsVersion = CodecUtil.checkSegmentHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
+ int termsVersion = CodecUtil.checkIndexHeader(in, VersionBlockTreeTermsWriter.TERMS_CODEC_NAME,
VersionBlockTreeTermsWriter.VERSION_START,
VersionBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
@@ -81,7 +81,7 @@ public final class VersionBlockTreeTerms
state.segmentSuffix,
VersionBlockTreeTermsWriter.TERMS_INDEX_EXTENSION);
indexIn = state.directory.openInput(indexFile, state.context);
- int indexVersion = CodecUtil.checkSegmentHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
+ int indexVersion = CodecUtil.checkIndexHeader(indexIn, VersionBlockTreeTermsWriter.TERMS_INDEX_CODEC_NAME,
VersionBlockTreeTermsWriter.VERSION_START,
VersionBlockTreeTermsWriter.VERSION_CURRENT,
state.segmentInfo.getId(), state.segmentSuffix);
@@ -94,7 +94,7 @@ public final class VersionBlockTreeTerms
CodecUtil.checksumEntireFile(indexIn);
// Have PostingsReader init itself
- postingsReader.init(in);
+ postingsReader.init(in, state);
// NOTE: data file is too costly to verify checksum against all the bytes on open,
// but for now we at least verify proper structure of the checksum footer: which looks
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java Tue Oct 28 09:19:21 2014
@@ -118,7 +118,7 @@ public final class VersionBlockTreeTerms
/** Extension of terms file */
static final String TERMS_EXTENSION = "tiv";
- final static String TERMS_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_DICT";
+ final static String TERMS_CODEC_NAME = "VersionBlockTreeTermsDict";
/** Initial terms format. */
public static final int VERSION_START = 1;
@@ -128,7 +128,7 @@ public final class VersionBlockTreeTerms
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tipv";
- final static String TERMS_INDEX_CODEC_NAME = "VERSION_BLOCK_TREE_TERMS_INDEX";
+ final static String TERMS_INDEX_CODEC_NAME = "VersionBlockTreeTermsIndex";
private final IndexOutput out;
private final IndexOutput indexOut;
@@ -199,20 +199,20 @@ public final class VersionBlockTreeTerms
fieldInfos = state.fieldInfos;
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
- CodecUtil.writeSegmentHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeIndexHeader(out, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
//DEBUG = state.segmentName.equals("_4a");
final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
- CodecUtil.writeSegmentHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.writeIndexHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;
// System.out.println("BTW.init seg=" + state.segmentName);
- postingsWriter.init(out); // have consumer write its format/header
+ postingsWriter.init(out, state); // have consumer write its format/header
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java Tue Oct 28 09:19:21 2014
@@ -27,6 +27,7 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Filter;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@@ -93,88 +94,84 @@ public class DuplicateFilter extends Fil
}
}
- private SparseFixedBitSet correctBits(LeafReader reader, Bits acceptDocs) throws IOException {
+ private DocIdSet correctBits(LeafReader reader, Bits acceptDocs) throws IOException {
SparseFixedBitSet bits = new SparseFixedBitSet(reader.maxDoc()); //assume all are INvalid
Terms terms = reader.fields().terms(fieldName);
- if (terms == null) {
- return bits;
- }
-
- TermsEnum termsEnum = terms.iterator(null);
- DocsEnum docs = null;
- while (true) {
- BytesRef currTerm = termsEnum.next();
- if (currTerm == null) {
- break;
- } else {
- docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
- int doc = docs.nextDoc();
- if (doc != DocIdSetIterator.NO_MORE_DOCS) {
- if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
- bits.set(doc);
- } else {
- int lastDoc = doc;
- while (true) {
- lastDoc = doc;
- doc = docs.nextDoc();
- if (doc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator(null);
+ DocsEnum docs = null;
+ while (true) {
+ BytesRef currTerm = termsEnum.next();
+ if (currTerm == null) {
+ break;
+ } else {
+ docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
+ int doc = docs.nextDoc();
+ if (doc != DocIdSetIterator.NO_MORE_DOCS) {
+ if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
+ bits.set(doc);
+ } else {
+ int lastDoc = doc;
+ while (true) {
+ lastDoc = doc;
+ doc = docs.nextDoc();
+ if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
}
+ bits.set(lastDoc);
}
- bits.set(lastDoc);
}
}
}
}
- return bits;
+ return new BitDocIdSet(bits, bits.approximateCardinality());
}
- private FixedBitSet fastBits(LeafReader reader, Bits acceptDocs) throws IOException {
+ private DocIdSet fastBits(LeafReader reader, Bits acceptDocs) throws IOException {
FixedBitSet bits = new FixedBitSet(reader.maxDoc());
bits.set(0, reader.maxDoc()); //assume all are valid
Terms terms = reader.fields().terms(fieldName);
- if (terms == null) {
- return bits;
- }
-
- TermsEnum termsEnum = terms.iterator(null);
- DocsEnum docs = null;
- while (true) {
- BytesRef currTerm = termsEnum.next();
- if (currTerm == null) {
- break;
- } else {
- if (termsEnum.docFreq() > 1) {
- // unset potential duplicates
- docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
- int doc = docs.nextDoc();
- if (doc != DocIdSetIterator.NO_MORE_DOCS) {
- if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator(null);
+ DocsEnum docs = null;
+ while (true) {
+ BytesRef currTerm = termsEnum.next();
+ if (currTerm == null) {
+ break;
+ } else {
+ if (termsEnum.docFreq() > 1) {
+ // unset potential duplicates
+ docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
+ int doc = docs.nextDoc();
+ if (doc != DocIdSetIterator.NO_MORE_DOCS) {
+ if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
+ doc = docs.nextDoc();
+ }
+ }
+
+ int lastDoc = -1;
+ while (true) {
+ lastDoc = doc;
+ bits.clear(lastDoc);
doc = docs.nextDoc();
+ if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
}
- }
-
- int lastDoc = -1;
- while (true) {
- lastDoc = doc;
- bits.clear(lastDoc);
- doc = docs.nextDoc();
- if (doc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
+
+ if (keepMode == KeepMode.KM_USE_LAST_OCCURRENCE) {
+ // restore the last bit
+ bits.set(lastDoc);
}
}
-
- if (keepMode == KeepMode.KM_USE_LAST_OCCURRENCE) {
- // restore the last bit
- bits.set(lastDoc);
- }
}
}
}
- return bits;
+ return new BitDocIdSet(bits);
}
public String getFieldName() {
Modified: lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java (original)
+++ lucene/dev/branches/lucene6005/lucene/sandbox/src/test/org/apache/lucene/search/TestTermAutomatonQuery.java Tue Oct 28 09:19:21 2014
@@ -46,6 +46,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -637,7 +638,7 @@ public class TestTermAutomatonQuery exte
}
}
- return bits;
+ return new BitDocIdSet(bits);
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/IntersectsPrefixTreeFilter.java Tue Oct 28 09:19:21 2014
@@ -19,11 +19,13 @@ package org.apache.lucene.spatial.prefix
import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import java.io.IOException;
@@ -71,7 +73,7 @@ public class IntersectsPrefixTreeFilter
@Override
protected DocIdSet finish() {
- return results;
+ return new BitDocIdSet(results);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/WithinPrefixTreeFilter.java Tue Oct 28 09:19:21 2014
@@ -17,22 +17,24 @@ package org.apache.lucene.spatial.prefix
* limitations under the License.
*/
-import com.spatial4j.core.context.SpatialContext;
-import com.spatial4j.core.distance.DistanceUtils;
-import com.spatial4j.core.shape.Circle;
-import com.spatial4j.core.shape.Point;
-import com.spatial4j.core.shape.Rectangle;
-import com.spatial4j.core.shape.Shape;
-import com.spatial4j.core.shape.SpatialRelation;
+import java.io.IOException;
+
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.spatial.prefix.tree.Cell;
import org.apache.lucene.spatial.prefix.tree.CellIterator;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
-import java.io.IOException;
+import com.spatial4j.core.context.SpatialContext;
+import com.spatial4j.core.distance.DistanceUtils;
+import com.spatial4j.core.shape.Circle;
+import com.spatial4j.core.shape.Point;
+import com.spatial4j.core.shape.Rectangle;
+import com.spatial4j.core.shape.Shape;
+import com.spatial4j.core.shape.SpatialRelation;
/**
* Finds docs where its indexed shape is {@link org.apache.lucene.spatial.query.SpatialOperation#IsWithin
@@ -134,7 +136,7 @@ public class WithinPrefixTreeFilter exte
@Override
protected DocIdSet finish() {
inside.andNot(outside);
- return inside;
+ return new BitDocIdSet(inside);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/lucene6005/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Tue Oct 28 09:19:21 2014
@@ -31,7 +31,6 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.ngram.EdgeNGramTokenFilter;
-import org.apache.lucene.analysis.ngram.Lucene43EdgeNGramTokenFilter;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -296,12 +295,7 @@ public class AnalyzingInfixSuggester ext
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
if (fieldName.equals("textgrams") && minPrefixChars > 0) {
// TODO: should use an EdgeNGramTokenFilterFactory here
- TokenFilter filter;
- if (matchVersion.onOrAfter(Version.LUCENE_4_4_0)) {
- filter = new EdgeNGramTokenFilter(components.getTokenStream(), 1, minPrefixChars);
- } else {
- filter = new Lucene43EdgeNGramTokenFilter(components.getTokenStream(), 1, minPrefixChars);
- }
+ TokenFilter filter = new EdgeNGramTokenFilter(components.getTokenStream(), 1, minPrefixChars);
return new TokenStreamComponents(components.getTokenizer(), filter);
} else {
return components;
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/ivy.xml?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/ivy.xml (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/ivy.xml Tue Oct 28 09:19:21 2014
@@ -25,15 +25,15 @@
JUnit4 ANT task only, no ANT.
This is used from build scripts for taskdefs.
-->
- <conf name="junit4-stdalone" />
+ <conf name="junit4-stdalone" transitive="false"/>
</configurations>
<dependencies>
<dependency org="org.apache.ant" name="ant" rev="${/org.apache.ant/ant}" conf="compile"/>
- <dependency org="junit" name="junit" rev="${/junit/junit}" transitive="false" conf="compile,junit4-stdalone"/>
- <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="${/com.carrotsearch.randomizedtesting/junit4-ant}" transitive="false" conf="compile,junit4-stdalone"/>
- <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="${/com.carrotsearch.randomizedtesting/randomizedtesting-runner}" transitive="false" conf="compile,junit4-stdalone"/>
+ <dependency org="junit" name="junit" rev="${/junit/junit}" conf="compile,junit4-stdalone"/>
+ <dependency org="com.carrotsearch.randomizedtesting" name="junit4-ant" rev="${/com.carrotsearch.randomizedtesting/junit4-ant}" conf="compile,junit4-stdalone"/>
+ <dependency org="com.carrotsearch.randomizedtesting" name="randomizedtesting-runner" rev="${/com.carrotsearch.randomizedtesting/randomizedtesting-runner}" conf="compile,junit4-stdalone"/>
<exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/>
</dependencies>
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankySegmentInfoFormat.java Tue Oct 28 09:19:21 2014
@@ -35,8 +35,8 @@ class CrankySegmentInfoFormat extends Se
}
@Override
- public SegmentInfo read(Directory directory, String segmentName, IOContext context) throws IOException {
- return delegate.read(directory, segmentName, context);
+ public SegmentInfo read(Directory directory, String segmentName, byte[] segmentID, IOContext context) throws IOException {
+ return delegate.read(directory, segmentName, segmentID, context);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Tue Oct 28 09:19:21 2014
@@ -38,8 +38,8 @@ import org.apache.lucene.codecs.blocktre
import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene41.Lucene41PostingsReader;
-import org.apache.lucene.codecs.lucene41.Lucene41PostingsWriter;
+import org.apache.lucene.codecs.lucene50.Lucene50PostingsReader;
+import org.apache.lucene.codecs.lucene50.Lucene50PostingsWriter;
import org.apache.lucene.codecs.memory.FSTOrdTermsReader;
import org.apache.lucene.codecs.memory.FSTOrdTermsWriter;
import org.apache.lucene.codecs.memory.FSTTermsReader;
@@ -92,7 +92,9 @@ public final class MockRandomPostingsFor
}
// we pull this before the seed intentionally: because its not consumed at runtime
- // (the skipInterval is written into postings header)
+ // (the skipInterval is written into postings header).
+ // NOTE: Currently not passed to postings writer.
+ // before, it was being passed in wrongly as acceptableOverhead!
int skipInterval = TestUtil.nextInt(seedRandom, minSkipInterval, 10);
if (LuceneTestCase.VERBOSE) {
@@ -117,7 +119,7 @@ public final class MockRandomPostingsFor
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state, skipInterval);
+ PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
final FieldsConsumer fields;
final int t1 = random.nextInt(5);
@@ -280,7 +282,7 @@ public final class MockRandomPostingsFor
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
+ PostingsReaderBase postingsReader = new Lucene50PostingsReader(state);
final FieldsProducer fields;
final int t1 = random.nextInt(5);
@@ -312,12 +314,7 @@ public final class MockRandomPostingsFor
boolean success = false;
try {
- fields = new BlockTreeTermsReader(state.directory,
- state.fieldInfos,
- state.segmentInfo,
- postingsReader,
- state.context,
- state.segmentSuffix);
+ fields = new BlockTreeTermsReader(postingsReader, state);
success = true;
} finally {
if (!success) {
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseCompoundFormatTestCase.java Tue Oct 28 09:19:21 2014
@@ -197,13 +197,14 @@ public abstract class BaseCompoundFormat
}
}
riw.close();
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
+ SegmentInfos infos = SegmentInfos.readLatestCommit(dir);
for (SegmentCommitInfo si : infos) {
if (si.info.getUseCompoundFile()) {
try (Directory cfsDir = si.info.getCodec().compoundFormat().getCompoundReader(dir, si.info, newIOContext(random()))) {
for (String cfsFile : cfsDir.listAll()) {
- try (IndexInput cfsIn = cfsDir.openInput(cfsFile, IOContext.DEFAULT)) {}
+ try (IndexInput cfsIn = cfsDir.openInput(cfsFile, IOContext.DEFAULT)) {
+ assert cfsIn != null;
+ }
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Tue Oct 28 09:19:21 2014
@@ -541,6 +541,34 @@ public abstract class BaseDocValuesForma
ireader.close();
directory.close();
}
+
+ public void testBytesMergeAwayAllValues() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ doc.add(new BinaryDocValuesField("field", new BytesRef("hi")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ iwriter.deleteDocuments(new Term("id", "1"));
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ BinaryDocValues dv = getOnlySegmentReader(ireader).getBinaryDocValues("field");
+ assertEquals(new BytesRef(), dv.get(0));
+
+ ireader.close();
+ directory.close();
+ }
public void testSortedBytes() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
@@ -2749,6 +2777,34 @@ public abstract class BaseDocValuesForma
directory.close();
}
+ public void testNumberMergeAwayAllValues() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ iwriter.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ doc.add(new NumericDocValuesField("field", 5));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ iwriter.deleteDocuments(new Term("id", "1"));
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ NumericDocValues dv = getOnlySegmentReader(ireader).getNumericDocValues("field");
+ assertEquals(0, dv.get(0));
+
+ ireader.close();
+ directory.close();
+ }
+
public void testTwoSortedNumber() throws IOException {
assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
@@ -2772,6 +2828,29 @@ public abstract class BaseDocValuesForma
directory.close();
}
+ public void testTwoSortedNumberSameValue() throws IOException {
+ assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
+ Document doc = new Document();
+ doc.add(new SortedNumericDocValuesField("dv", 11));
+ doc.add(new SortedNumericDocValuesField("dv", 11));
+ writer.addDocument(doc);
+ writer.close();
+
+ // Now search the index:
+ IndexReader reader = DirectoryReader.open(directory);
+ assert reader.leaves().size() == 1;
+ SortedNumericDocValues dv = reader.leaves().get(0).reader().getSortedNumericDocValues("dv");
+ dv.setDocument(0);
+ assertEquals(2, dv.count());
+ assertEquals(11, dv.valueAt(0));
+ assertEquals(11, dv.valueAt(1));
+
+ reader.close();
+ directory.close();
+ }
+
public void testTwoSortedNumberOneMissing() throws IOException {
assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric());
Directory directory = newDirectory();
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Tue Oct 28 09:19:21 2014
@@ -153,6 +153,35 @@ public abstract class BaseNormsFormatTes
}
}
+ public void testOutliers() throws Exception {
+ int iterations = atLeast(1);
+ final Random r = random();
+ for (int i = 0; i < iterations; i++) {
+ final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
+ doTestNormsVersusStoredFields(new LongProducer() {
+ @Override
+ long next() {
+ return r.nextInt(100) == 0 ? TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE) : commonValue;
+ }
+ });
+ }
+ }
+
+ public void testOutliers2() throws Exception {
+ int iterations = atLeast(1);
+ final Random r = random();
+ for (int i = 0; i < iterations; i++) {
+ final long commonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
+ final long uncommonValue = TestUtil.nextLong(r, Byte.MIN_VALUE, Byte.MAX_VALUE);
+ doTestNormsVersusStoredFields(new LongProducer() {
+ @Override
+ long next() {
+ return r.nextInt(100) == 0 ? uncommonValue : commonValue;
+ }
+ });
+ }
+ }
+
private void doTestNormsVersusStoredFields(LongProducer longs) throws Exception {
int numDocs = atLeast(500);
long norms[] = new long[numDocs];
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java Tue Oct 28 09:19:21 2014
@@ -47,11 +47,12 @@ public abstract class BaseSegmentInfoFor
public void testFiles() throws Exception {
Directory dir = newDirectory();
Codec codec = getCodec();
+ byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), StringHelper.randomId());
+ Collections.<String,String>emptyMap(), id);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
assertEquals(info.files(), info2.files());
dir.close();
}
@@ -60,8 +61,9 @@ public abstract class BaseSegmentInfoFor
public void testAddsSelfToFiles() throws Exception {
Directory dir = newDirectory();
Codec codec = getCodec();
+ byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), StringHelper.randomId());
+ Collections.<String,String>emptyMap(), id);
Set<String> originalFiles = Collections.singleton("_123.a");
info.setFiles(originalFiles);
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
@@ -70,7 +72,7 @@ public abstract class BaseSegmentInfoFor
assertTrue(modifiedFiles.containsAll(originalFiles));
assertTrue("did you forget to add yourself to files()", modifiedFiles.size() > originalFiles.size());
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
assertEquals(info.files(), info2.files());
dir.close();
}
@@ -79,14 +81,15 @@ public abstract class BaseSegmentInfoFor
public void testDiagnostics() throws Exception {
Directory dir = newDirectory();
Codec codec = getCodec();
+ byte id[] = StringHelper.randomId();
Map<String,String> diagnostics = new HashMap<>();
diagnostics.put("key1", "value1");
diagnostics.put("key2", "value2");
SegmentInfo info = new SegmentInfo(dir, getVersions()[0], "_123", 1, false, codec,
- diagnostics, StringHelper.randomId());
+ diagnostics, id);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
assertEquals(diagnostics, info2.getDiagnostics());
dir.close();
}
@@ -100,7 +103,7 @@ public abstract class BaseSegmentInfoFor
Collections.<String,String>emptyMap(), id);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
assertIDEquals(id, info2.getId());
dir.close();
}
@@ -110,11 +113,12 @@ public abstract class BaseSegmentInfoFor
Codec codec = getCodec();
for (Version v : getVersions()) {
Directory dir = newDirectory();
+ byte id[] = StringHelper.randomId();
SegmentInfo info = new SegmentInfo(dir, v, "_123", 1, false, codec,
- Collections.<String,String>emptyMap(), StringHelper.randomId());
+ Collections.<String,String>emptyMap(), id);
info.setFiles(Collections.<String>emptySet());
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
assertEquals(info2.getVersion(), v);
dir.close();
}
@@ -152,7 +156,7 @@ public abstract class BaseSegmentInfoFor
SegmentInfo info = new SegmentInfo(dir, version, name, docCount, isCompoundFile, codec, diagnostics, id);
info.setFiles(files);
codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
- SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, IOContext.DEFAULT);
+ SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, id, IOContext.DEFAULT);
assertEquals(info, info2);
dir.close();
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java Tue Oct 28 09:19:21 2014
@@ -32,11 +32,11 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.asserting.AssertingCodec;
import org.apache.lucene.codecs.asserting.AssertingDocValuesFormat;
import org.apache.lucene.codecs.asserting.AssertingPostingsFormat;
-import org.apache.lucene.codecs.blocktreeords.Ords41PostingsFormat;
+import org.apache.lucene.codecs.blockterms.LuceneFixedGap;
+import org.apache.lucene.codecs.blockterms.LuceneVarGapDocFreqInterval;
+import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval;
+import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings;
-import org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds;
-import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapDocFreqInterval;
-import org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.codecs.memory.FSTOrdPostingsFormat;
import org.apache.lucene.codecs.memory.FSTPostingsFormat;
@@ -127,15 +127,15 @@ public class RandomCodec extends Asserti
new FSTOrdPostingsFormat(),
new DirectPostingsFormat(LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : maxItemsPerBlock),
LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : lowFreqCutoff)),
- //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucene41Postings to be constructed
+ //TODO as a PostingsFormat which wraps others, we should allow TestBloomFilteredLucenePostings to be constructed
//with a choice of concrete PostingsFormats. Maybe useful to have a generic means of marking and dealing
//with such "wrapper" classes?
new TestBloomFilteredLucenePostings(),
new MockRandomPostingsFormat(random),
- new Ords41PostingsFormat(minItemsPerBlock, maxItemsPerBlock),
- new Lucene41WithOrds(TestUtil.nextInt(random, 1, 1000)),
- new Lucene41VarGapFixedInterval(TestUtil.nextInt(random, 1, 1000)),
- new Lucene41VarGapDocFreqInterval(TestUtil.nextInt(random, 1, 100), TestUtil.nextInt(random, 1, 1000)),
+ new BlockTreeOrdsPostingsFormat(minItemsPerBlock, maxItemsPerBlock),
+ new LuceneFixedGap(TestUtil.nextInt(random, 1, 1000)),
+ new LuceneVarGapFixedInterval(TestUtil.nextInt(random, 1, 1000)),
+ new LuceneVarGapDocFreqInterval(TestUtil.nextInt(random, 1, 100), TestUtil.nextInt(random, 1, 1000)),
new SimpleTextPostingsFormat(),
new AssertingPostingsFormat(),
new MemoryPostingsFormat(true, random.nextFloat()),
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java Tue Oct 28 09:19:21 2014
@@ -56,7 +56,7 @@ public class RandomIndexWriter implement
public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
// Randomly calls Thread.yield so we mixup thread scheduling
final Random random = new Random(r.nextLong());
- return mockIndexWriter(dir, conf, new TestPoint() {
+ return mockIndexWriter(dir, conf, new TestPoint() {
@Override
public void apply(String message) {
if (random.nextInt(4) == 2)
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java Tue Oct 28 09:19:21 2014
@@ -800,11 +800,12 @@ public class MockDirectoryWrapper extend
if (LuceneTestCase.VERBOSE) {
System.out.println("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
}
- SegmentInfos sis = new SegmentInfos();
+ SegmentInfos sis;
try {
- sis.read(in, file);
+ sis = SegmentInfos.readCommit(in, file);
} catch (IOException ioe) {
// OK: likely some of the .si files were deleted
+ sis = new SegmentInfos();
}
try {
@@ -859,7 +860,7 @@ public class MockDirectoryWrapper extend
extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
}
- assert false : "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras;
+ throw new RuntimeException("unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras);
}
DirectoryReader ir1 = DirectoryReader.open(this);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/BaseDocIdSetTestCase.java Tue Oct 28 09:19:21 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import static org.apache.lucene.util.BaseBitSetTestCase.randomSet;
+
import java.io.IOException;
import java.util.BitSet;
@@ -29,31 +31,6 @@ public abstract class BaseDocIdSetTestCa
/** Create a copy of the given {@link BitSet} which has <code>length</code> bits. */
public abstract T copyOf(BitSet bs, int length) throws IOException;
- /** Create a random set which has <code>numBitsSet</code> of its <code>numBits</code> bits set. */
- protected static BitSet randomSet(int numBits, int numBitsSet) {
- assert numBitsSet <= numBits;
- final BitSet set = new BitSet(numBits);
- if (numBitsSet == numBits) {
- set.set(0, numBits);
- } else {
- for (int i = 0; i < numBitsSet; ++i) {
- while (true) {
- final int o = random().nextInt(numBits);
- if (!set.get(o)) {
- set.set(o);
- break;
- }
- }
- }
- }
- return set;
- }
-
- /** Same as {@link #randomSet(int, int)} but given a load factor. */
- protected static BitSet randomSet(int numBits, float percentSet) {
- return randomSet(numBits, (int) (percentSet * numBits));
- }
-
/** Test length=0. */
public void testNoBit() throws IOException {
final BitSet bs = new BitSet(1);
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Tue Oct 28 09:19:21 2014
@@ -79,7 +79,6 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.LiveIndexWriterConfig;
@@ -388,6 +387,8 @@ public abstract class LuceneTestCase ext
*/
public static final int RANDOM_MULTIPLIER = systemPropertyAsInt("tests.multiplier", 1);
+ public static final boolean TEST_ASSERTS_ENABLED = systemPropertyAsBoolean("tests.asserts", true);
+
/** TODO: javadoc? */
public static final String DEFAULT_LINE_DOCS_FILE = "europarl.lines.txt.gz";
@@ -2467,4 +2468,13 @@ public abstract class LuceneTestCase ext
public static Path createTempFile() throws IOException {
return createTempFile("tempFile", ".tmp");
}
+
+ /** True if assertions (-ea) are enabled (at least for this class). */
+ public static final boolean assertsAreEnabled;
+
+ static {
+ boolean enabled = false;
+ assert enabled = true; // Intentional side-effect!!!
+ assertsAreEnabled = enabled;
+ }
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/RunListenerPrintReproduceInfo.java Tue Oct 28 09:19:21 2014
@@ -177,6 +177,12 @@ public final class RunListenerPrintRepro
}
}
+ if (LuceneTestCase.assertsAreEnabled) {
+ addVmOpt(b, "tests.asserts", "true");
+ } else {
+ addVmOpt(b, "tests.asserts", "false");
+ }
+
addVmOpt(b, "tests.file.encoding", System.getProperty("file.encoding"));
System.err.println(b.toString());
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleAssertionsRequired.java Tue Oct 28 09:19:21 2014
@@ -31,11 +31,22 @@ public class TestRuleAssertionsRequired
@Override
public void evaluate() throws Throwable {
try {
- assert false;
- String msg = "Test class requires enabled assertions, enable globally (-ea)" +
- " or for Solr/Lucene subpackages only: " + description.getClassName();
- System.err.println(msg);
- throw new Exception(msg);
+ // Make sure -ea matches -Dtests.asserts, to catch accidental mis-use:
+ if (LuceneTestCase.assertsAreEnabled != LuceneTestCase.TEST_ASSERTS_ENABLED) {
+ String msg = "Assertions mismatch: ";
+ if (LuceneTestCase.assertsAreEnabled) {
+ msg += "-ea was specified";
+ } else {
+ msg += "-ea was not specified";
+ }
+ if (LuceneTestCase.TEST_ASSERTS_ENABLED) {
+ msg += " but -Dtests.asserts=true";
+ } else {
+ msg += " but -Dtests.asserts=false";
+ }
+ System.err.println(msg);
+ throw new Exception(msg);
+ }
} catch (AssertionError e) {
// Ok, enabled.
}
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Tue Oct 28 09:19:21 2014
@@ -45,9 +45,11 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.asserting.AssertingCodec;
-import org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat;
-import org.apache.lucene.codecs.lucene410.Lucene410DocValuesFormat;
+import org.apache.lucene.codecs.blockterms.LuceneFixedGap;
+import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene50.Lucene50Codec;
+import org.apache.lucene.codecs.lucene50.Lucene50DocValuesFormat;
+import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -746,7 +748,7 @@ public final class TestUtil {
* Returns the actual default postings format (e.g. LuceneMNPostingsFormat for this version of Lucene.
*/
public static PostingsFormat getDefaultPostingsFormat() {
- return new Lucene41PostingsFormat();
+ return new Lucene50PostingsFormat();
}
/**
@@ -754,14 +756,25 @@ public final class TestUtil {
* @lucene.internal this may disappear at any time
*/
public static PostingsFormat getDefaultPostingsFormat(int minItemsPerBlock, int maxItemsPerBlock) {
- return new Lucene41PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ return new Lucene50PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ }
+
+ /** Returns a random postings format that supports term ordinals */
+ public static PostingsFormat getPostingsFormatWithOrds(Random r) {
+ switch (r.nextInt(2)) {
+ case 0: return new LuceneFixedGap();
+ case 1: return new BlockTreeOrdsPostingsFormat();
+ // TODO: these don't actually support ords!
+ //case 2: return new FSTOrdPostingsFormat();
+ default: throw new AssertionError();
+ }
}
/**
* Returns the actual default docvalues format (e.g. LuceneMNDocValuesFormat for this version of Lucene.
*/
public static DocValuesFormat getDefaultDocValuesFormat() {
- return new Lucene410DocValuesFormat();
+ return new Lucene50DocValuesFormat();
}
// TODO: generalize all 'test-checks-for-crazy-codecs' to
Modified: lucene/dev/branches/lucene6005/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1634823&r1=1634822&r2=1634823&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/lucene6005/lucene/test-framework/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Tue Oct 28 09:19:21 2014
@@ -15,8 +15,8 @@
org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat
org.apache.lucene.codecs.ramonly.RAMOnlyPostingsFormat
-org.apache.lucene.codecs.lucene41ords.Lucene41WithOrds
-org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapFixedInterval
-org.apache.lucene.codecs.lucene41vargap.Lucene41VarGapDocFreqInterval
+org.apache.lucene.codecs.blockterms.LuceneFixedGap
+org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval
+org.apache.lucene.codecs.blockterms.LuceneVarGapDocFreqInterval
org.apache.lucene.codecs.bloom.TestBloomFilteredLucenePostings
org.apache.lucene.codecs.asserting.AssertingPostingsFormat