You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/27 15:37:54 UTC
svn commit: r1634549 - in /lucene/dev/trunk/lucene:
core/src/java/org/apache/lucene/index/
misc/src/java/org/apache/lucene/index/sorter/
misc/src/test/org/apache/lucene/index/sorter/
Author: mikemccand
Date: Mon Oct 27 14:37:53 2014
New Revision: 1634549
URL: http://svn.apache.org/r1634549
Log:
improve index sorter tests (rename the two different readers); improve javadocs
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1634549&r1=1634548&r2=1634549&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Mon Oct 27 14:37:53 2014
@@ -30,8 +30,8 @@ import org.apache.lucene.index.MultiDocV
/**
* This class forces a composite reader (eg a {@link
- * MultiReader} or {@link DirectoryReader}) to emulate an
- * atomic reader. This requires implementing the postings
+ * MultiReader} or {@link DirectoryReader}) to emulate a
+ * {@link LeafReader}. This requires implementing the postings
* APIs on-the-fly, using the static methods in {@link
* MultiFields}, {@link MultiDocValues}, by stepping through
* the sub-readers to merge fields/terms, appending docs, etc.
@@ -40,7 +40,7 @@ import org.apache.lucene.index.MultiDocV
* performance hit. If this is important to your use case,
* you'll get better performance by gathering the sub readers using
* {@link IndexReader#getContext()} to get the
- * atomic leaves and then operate per-LeafReader,
+ * leaves and then operate per-LeafReader,
* instead of using this class.
*/
public final class SlowCompositeReaderWrapper extends LeafReader {
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java?rev=1634549&r1=1634548&r2=1634549&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java Mon Oct 27 14:37:53 2014
@@ -746,7 +746,7 @@ public class SortingLeafReader extends F
/** Expert: same as {@link #wrap(org.apache.lucene.index.LeafReader, Sort)} but operates directly on a {@link Sorter.DocMap}. */
static LeafReader wrap(LeafReader reader, Sorter.DocMap docMap) {
if (docMap == null) {
- // the reader is already sorter
+ // the reader is already sorted
return reader;
}
if (reader.maxDoc() != docMap.size()) {
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java?rev=1634549&r1=1634548&r2=1634549&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java Mon Oct 27 14:37:53 2014
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.search.Sort;
@@ -40,13 +41,15 @@ public class IndexSortingTest extends So
@BeforeClass
public static void beforeClassSorterUtilTest() throws Exception {
+ // NOTE: index was created by by super's @BeforeClass
+
// only read the values of the undeleted documents, since after addIndexes,
// the deleted ones will be dropped from the index.
- Bits liveDocs = reader.getLiveDocs();
+ Bits liveDocs = unsortedReader.getLiveDocs();
List<Integer> values = new ArrayList<>();
- for (int i = 0; i < reader.maxDoc(); i++) {
+ for (int i = 0; i < unsortedReader.maxDoc(); i++) {
if (liveDocs == null || liveDocs.get(i)) {
- values.add(Integer.valueOf(reader.document(i).get(ID_FIELD)));
+ values.add(Integer.valueOf(unsortedReader.document(i).get(ID_FIELD)));
}
}
int idx = random().nextInt(SORT.length);
@@ -68,9 +71,10 @@ public class IndexSortingTest extends So
Directory target = newDirectory();
IndexWriter writer = new IndexWriter(target, newIndexWriterConfig(null));
- reader = SortingLeafReader.wrap(reader, sorter);
+ IndexReader reader = SortingLeafReader.wrap(unsortedReader, sorter);
writer.addIndexes(reader);
writer.close();
+ // NOTE: also closes unsortedReader
reader.close();
dir.close();
@@ -79,8 +83,8 @@ public class IndexSortingTest extends So
TestUtil.checkIndex(dir);
// set reader for tests
- reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
- assertFalse("index should not have deletions", reader.hasDeletions());
+ sortedReader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
+ assertFalse("index should not have deletions", sortedReader.hasDeletions());
}
}
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1634549&r1=1634548&r2=1634549&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Mon Oct 27 14:37:53 2014
@@ -167,7 +167,8 @@ public abstract class SorterTestBase ext
}
protected static Directory dir;
- protected static LeafReader reader;
+ protected static LeafReader unsortedReader;
+ protected static LeafReader sortedReader;
protected static Integer[] sortedValues;
private static Document doc(final int id, PositionsTokenStream positions) {
@@ -190,8 +191,8 @@ public abstract class SorterTestBase ext
return doc;
}
- /** Creates an index for sorting. */
- public static void createIndex(Directory dir, int numDocs, Random random) throws IOException {
+ /** Creates an unsorted index; subclasses then sort this index and open sortedReader. */
+ private static void createIndex(Directory dir, int numDocs, Random random) throws IOException {
List<Integer> ids = new ArrayList<>();
for (int i = 0; i < numDocs; i++) {
ids.add(Integer.valueOf(i * 10));
@@ -230,19 +231,20 @@ public abstract class SorterTestBase ext
int numDocs = atLeast(20);
createIndex(dir, numDocs, random());
- reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
+ unsortedReader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
}
@AfterClass
public static void afterClassSorterTestBase() throws Exception {
- reader.close();
+ unsortedReader.close();
+ sortedReader.close();
dir.close();
}
@Test
public void testBinaryDocValuesField() throws Exception {
- BinaryDocValues dv = reader.getBinaryDocValues(BINARY_DV_FIELD);
- for (int i = 0; i < reader.maxDoc(); i++) {
+ BinaryDocValues dv = sortedReader.getBinaryDocValues(BINARY_DV_FIELD);
+ for (int i = 0; i < sortedReader.maxDoc(); i++) {
final BytesRef bytes = dv.get(i);
assertEquals("incorrect binary DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
}
@@ -250,7 +252,7 @@ public abstract class SorterTestBase ext
@Test
public void testDocsAndPositionsEnum() throws Exception {
- TermsEnum termsEnum = reader.terms(DOC_POSITIONS_FIELD).iterator(null);
+ TermsEnum termsEnum = sortedReader.terms(DOC_POSITIONS_FIELD).iterator(null);
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(DOC_POSITIONS_TERM)));
DocsAndPositionsEnum sortedPositions = termsEnum.docsAndPositions(null, null);
int doc;
@@ -310,8 +312,8 @@ public abstract class SorterTestBase ext
@Test
public void testDocsEnum() throws Exception {
- Bits mappedLiveDocs = randomLiveDocs(reader.maxDoc());
- TermsEnum termsEnum = reader.terms(DOCS_ENUM_FIELD).iterator(null);
+ Bits mappedLiveDocs = randomLiveDocs(sortedReader.maxDoc());
+ TermsEnum termsEnum = sortedReader.terms(DOCS_ENUM_FIELD).iterator(null);
assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(DOCS_ENUM_TERM)));
DocsEnum docs = termsEnum.docs(mappedLiveDocs, null);
@@ -319,12 +321,12 @@ public abstract class SorterTestBase ext
int prev = -1;
while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
assertTrue("document " + doc + " marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(doc));
- assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(sortedReader.document(doc).get(ID_FIELD)));
while (++prev < doc) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
- while (++prev < reader.maxDoc()) {
+ while (++prev < sortedReader.maxDoc()) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
@@ -337,20 +339,20 @@ public abstract class SorterTestBase ext
prev = -1;
while ((doc = docs.advance(doc + 1)) != DocIdSetIterator.NO_MORE_DOCS) {
assertTrue("document " + doc + " marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(doc));
- assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(sortedReader.document(doc).get(ID_FIELD)));
while (++prev < doc) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
- while (++prev < reader.maxDoc()) {
+ while (++prev < sortedReader.maxDoc()) {
assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs == null || mappedLiveDocs.get(prev));
}
}
@Test
public void testNormValues() throws Exception {
- NumericDocValues dv = reader.getNormValues(NORMS_FIELD);
- int maxDoc = reader.maxDoc();
+ NumericDocValues dv = sortedReader.getNormValues(NORMS_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
assertEquals("incorrect norm value for doc " + i, sortedValues[i].intValue(), dv.get(i));
}
@@ -358,8 +360,8 @@ public abstract class SorterTestBase ext
@Test
public void testNumericDocValuesField() throws Exception {
- NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ NumericDocValues dv = sortedReader.getNumericDocValues(NUMERIC_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
assertEquals("incorrect numeric DocValues for doc " + i, sortedValues[i].intValue(), dv.get(i));
}
@@ -367,8 +369,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedDocValuesField() throws Exception {
- SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedDocValues dv = sortedReader.getSortedDocValues(SORTED_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
final BytesRef bytes = dv.get(i);
assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
@@ -377,8 +379,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedSetDocValuesField() throws Exception {
- SortedSetDocValues dv = reader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedSetDocValues dv = sortedReader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
dv.setDocument(i);
BytesRef bytes = dv.lookupOrd(dv.nextOrd());
@@ -392,8 +394,8 @@ public abstract class SorterTestBase ext
@Test
public void testSortedNumericDocValuesField() throws Exception {
- SortedNumericDocValues dv = reader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
- int maxDoc = reader.maxDoc();
+ SortedNumericDocValues dv = sortedReader.getSortedNumericDocValues(SORTED_NUMERIC_DV_FIELD);
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
dv.setDocument(i);
assertEquals(2, dv.count());
@@ -405,9 +407,9 @@ public abstract class SorterTestBase ext
@Test
public void testTermVectors() throws Exception {
- int maxDoc = reader.maxDoc();
+ int maxDoc = sortedReader.maxDoc();
for (int i = 0; i < maxDoc; i++) {
- Terms terms = reader.getTermVector(i, TERM_VECTORS_FIELD);
+ Terms terms = sortedReader.getTermVector(i, TERM_VECTORS_FIELD);
assertNotNull("term vectors not found for doc " + i + " field [" + TERM_VECTORS_FIELD + "]", terms);
assertEquals("incorrect term vector for doc " + i, sortedValues[i].toString(), terms.iterator(null).next().utf8ToString());
}
Modified: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java?rev=1634549&r1=1634548&r2=1634549&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java (original)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingLeafReaderTest.java Mon Oct 27 14:37:53 2014
@@ -30,15 +30,16 @@ public class SortingLeafReaderTest exten
@BeforeClass
public static void beforeClassSortingLeafReaderTest() throws Exception {
+ // NOTE: index was created by by super's @BeforeClass
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
Sort sort = new Sort(new SortField(NUMERIC_DV_FIELD, SortField.Type.INT));
- final Sorter.DocMap docMap = new Sorter(sort).sort(reader);
+ final Sorter.DocMap docMap = new Sorter(sort).sort(unsortedReader);
// Sorter.compute also sorts the values
- NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD);
- sortedValues = new Integer[reader.maxDoc()];
- for (int i = 0; i < reader.maxDoc(); ++i) {
+ NumericDocValues dv = unsortedReader.getNumericDocValues(NUMERIC_DV_FIELD);
+ sortedValues = new Integer[unsortedReader.maxDoc()];
+ for (int i = 0; i < unsortedReader.maxDoc(); ++i) {
sortedValues[docMap.oldToNew(i)] = (int)dv.get(i);
}
if (VERBOSE) {
@@ -47,11 +48,11 @@ public class SortingLeafReaderTest exten
}
// sort the index by id (as integer, in NUMERIC_DV_FIELD)
- reader = SortingLeafReader.wrap(reader, sort);
+ sortedReader = SortingLeafReader.wrap(unsortedReader, sort);
if (VERBOSE) {
System.out.print("mapped-deleted-docs: ");
- Bits mappedLiveDocs = reader.getLiveDocs();
+ Bits mappedLiveDocs = sortedReader.getLiveDocs();
for (int i = 0; i < mappedLiveDocs.length(); i++) {
if (!mappedLiveDocs.get(i)) {
System.out.print(i + " ");
@@ -60,12 +61,12 @@ public class SortingLeafReaderTest exten
System.out.println();
}
- TestUtil.checkReader(reader);
+ TestUtil.checkReader(sortedReader);
}
public void testBadSort() throws Exception {
try {
- SortingLeafReader.wrap(reader, Sort.RELEVANCE);
+ SortingLeafReader.wrap(sortedReader, Sort.RELEVANCE);
fail("Didn't get expected exception");
} catch (IllegalArgumentException e) {
assertEquals("Cannot sort an index with a Sort that refers to the relevance score", e.getMessage());