You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2018/12/07 18:17:22 UTC
[1/4] lucene-solr:master: Revert "Add the ability to set the number
of hits to track accurately"
Repository: lucene-solr
Updated Branches:
refs/heads/master 84d5015ee -> 9f29ed075
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 5bfd9d5..3857a97 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -355,206 +355,166 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingStringFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- assertEquals(1, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- } else {
- // docID 0 is missing:
- assertEquals(1, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(2, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ // docID 0 is missing:
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedStringFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", reverse);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", false);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1l, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingStringLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- if (reverse) {
- assertEquals(1, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- assertEquals(2, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(1, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- }
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedStringLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", reverse);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", false);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1l, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testBasicLong() throws Exception {
@@ -637,213 +597,11 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingLongFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- } else {
- // docID 0 has no value
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingMultiValuedLongFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 27));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 24));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingLongLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- // docID 0 is missing
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(7, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- }
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingMultiValuedLongLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 65));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- doc.add(new SortedNumericDocValuesField("foo", 74));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testBasicInt() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ SortField sortField = new SortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -852,9 +610,8 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", -1));
- w.addDocument(doc);
+ // missing
+ w.addDocument(new Document());
w.commit();
doc = new Document();
@@ -866,8 +623,7 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(-1, values.longValue());
+ // docID 0 has no value
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
@@ -877,32 +633,32 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testBasicMultiValuedInt() throws Exception {
+ public void testMissingMultiValuedLongFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(new NumericDocValuesField("id", 3));
doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 34));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
+ // missing
doc = new Document();
doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", -1));
- doc.add(new SortedNumericDocValuesField("foo", 34));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(new NumericDocValuesField("id", 2));
doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 22));
- doc.add(new SortedNumericDocValuesField("foo", 27));
+ doc.add(new SortedNumericDocValuesField("foo", 24));
w.addDocument(doc);
w.forceMerge(1);
@@ -921,212 +677,496 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testMissingIntFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ public void testMissingLongLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- } else {
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedLongLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 65));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ doc.add(new SortedNumericDocValuesField("foo", 74));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicInt() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", -1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicMultiValuedInt() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", -1));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 22));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingIntFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedIntFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 187667));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 187667));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingIntLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- // docID 0 is missing
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(7, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- }
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedIntLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 6372));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 6372));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 8));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 8));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicDouble() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", -1.0));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicMultiValuedDouble() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingDoubleFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedDoubleFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
- public void testBasicDouble() throws Exception {
+ public void testMissingDoubleLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -1135,9 +1175,8 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", -1.0));
- w.addDocument(doc);
+ // missing
+ w.addDocument(new Document());
w.commit();
doc = new Document();
@@ -1150,41 +1189,41 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
assertEquals(0, values.nextDoc());
- assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(1, values.nextDoc());
assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
+ assertEquals(1, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
- public void testBasicMultiValuedDouble() throws Exception {
+ public void testMissingMultiValuedDoubleLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
+ // missing
doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
+ doc.add(new NumericDocValuesField("id", 3));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new NumericDocValuesField("id", 1));
doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
w.addDocument(doc);
w.forceMerge(1);
@@ -1203,207 +1242,6 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testMissingDoubleFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- } else {
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingMultiValuedDoubleFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingDoubleLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(1, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(1, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- }
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
- }
-
- public void testMissingMultiValuedDoubleLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
- }
-
public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1484,204 +1322,164 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingFloatFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(1, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- } else {
- assertEquals(1, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedFloatFirst() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingFloatLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- if (reverse) {
- assertEquals(1, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(2, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(1, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- }
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
}
public void testMissingMultiValuedFloatLast() throws Exception {
- for (boolean reverse : new boolean[] {true, false}) {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- if (reverse) {
- assertEquals(0, values.nextDoc());
- assertEquals(3, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(1, values.longValue());
- } else {
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- }
- r.close();
- w.close();
- dir.close();
- }
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
}
public void testRandom1() throws IOException {
[4/4] lucene-solr:master: LUCENE-8592: Fix index sorting corruption
due to numeric overflow
Posted by ji...@apache.org.
LUCENE-8592: Fix index sorting corruption due to numeric overflow
The merge sort of sorted segments can produce an invalid
sort if the sort field is an Integer/Long that uses reverse order and contains values equal to
Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug.
Indices affected by the bug can be detected by running the CheckIndex command on a
distribution that contains the fix (7.6+).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f29ed07
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f29ed07
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f29ed07
Branch: refs/heads/master
Commit: 9f29ed0757eae12d8311ffd6891f7032370ea39a
Parents: da7919f
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:16:42 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:16:42 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 9 +-
.../index/TestBackwardsCompatibility.java | 27 +
.../lucene/index/sorted-invalid.7.5.0.zip | Bin 0 -> 468799 bytes
.../org/apache/lucene/index/MultiSorter.java | 30 +-
.../apache/lucene/index/TestIndexSorting.java | 1738 ++++++++++--------
5 files changed, 1020 insertions(+), 784 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 14b4961..007d35b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -314,7 +314,14 @@ Bug fixes
* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
to the same doc in the same updates package looses an update if the reset comes before
- the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+ the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
+
+* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
+ can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
+ values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
+ (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
+ detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
+ (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 6f28797..44911db 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1717,6 +1717,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dir.close();
}
}
+
+ /**
+ * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
+ * before https://issues.apache.org/jira/browse/LUCENE-8592.
+ */
+ public void testSortedIndexWithInvalidSort() throws Exception {
+ Path path = createTempDir("sorted");
+ String name = "sorted-invalid.7.5.0.zip";
+ InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
+ assertNotNull("Sorted index index " + name + " not found", resource);
+ TestUtil.unzip(resource, path);
+
+ Directory dir = FSDirectory.open(path);
+
+ DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(1, reader.leaves().size());
+ Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
+ assertNotNull(sort);
+ assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
+ reader.close();
+ CheckIndex.Status status = TestUtil.checkIndex(dir);
+ assertEquals(1, status.segmentInfos.size());
+ assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
+ assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
+ "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
+ dir.close();
+ }
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
new file mode 100644
index 0000000..30c68a3
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index b484228..d81ac75 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,17 +42,18 @@ final class MultiSorter {
SortField fields[] = sort.getSort();
final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
+ final int[] reverseMuls = new int[fields.length];
for(int i=0;i<fields.length;i++) {
comparables[i] = getComparableProviders(readers, fields[i]);
+ reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
-
int leafCount = readers.size();
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
for(int i=0;i<comparables.length;i++) {
- int cmp = a.values[i].compareTo(b.values[i]);
+ int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@@ -146,14 +147,13 @@ final class MultiSorter {
/** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
private interface ComparableProvider {
- public Comparable getComparable(int docID) throws IOException;
+ Comparable getComparable(int docID) throws IOException;
}
/** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
- final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
- missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+ missingOrd = Integer.MAX_VALUE;
} else {
- missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+ missingOrd = Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
}
if (readerDocID == docID) {
// translate segment's ord to global ord space:
- return reverseMul * (int) globalOrds.get(readerValues.ordValue());
+ return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
} else {
return missingOrd;
}
@@ -238,9 +238,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * values.longValue();
+ return values.longValue();
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -279,9 +279,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * (int) values.longValue();
+ return (int) values.longValue();
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -320,9 +320,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * Double.longBitsToDouble(values.longValue());
+ return Double.longBitsToDouble(values.longValue());
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -361,9 +361,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * Float.intBitsToFloat((int) values.longValue());
+ return Float.intBitsToFloat((int) values.longValue());
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
[3/4] lucene-solr:master: LUCENE-8592: Fix index sorting corruption
due to numeric overflow
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 3857a97..5bfd9d5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -355,166 +355,206 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingStringFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- // docID 0 is missing:
- assertEquals(1, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(2, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ } else {
+ // docID 0 is missing:
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedStringFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", false);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", reverse);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingStringLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(1, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedStringLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", false);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", reverse);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testBasicLong() throws Exception {
@@ -597,11 +637,213 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingLongFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ // docID 0 has no value
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedLongFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 24));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingLongLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ // docID 0 is missing
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedLongLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 65));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ doc.add(new SortedNumericDocValuesField("foo", 74));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testBasicInt() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -610,8 +852,9 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", -1));
+ w.addDocument(doc);
w.commit();
doc = new Document();
@@ -623,7 +866,8 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- // docID 0 has no value
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
@@ -633,195 +877,32 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testMissingMultiValuedLongFirst() throws Exception {
+ public void testBasicMultiValuedInt() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(new NumericDocValuesField("id", 3));
doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 27));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
doc = new Document();
doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", -1));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(new NumericDocValuesField("id", 2));
doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 24));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingLongLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingMultiValuedLongLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 65));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- doc.add(new SortedNumericDocValuesField("foo", 74));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicInt() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", -1));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(-1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicMultiValuedInt() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", -1));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 22));
- doc.add(new SortedNumericDocValuesField("foo", 27));
+ doc.add(new SortedNumericDocValuesField("foo", 22));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
w.addDocument(doc);
w.forceMerge(1);
@@ -841,332 +922,211 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingIntFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedIntFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 187667));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 187667));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingIntLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ // docID 0 is missing
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedIntLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 6372));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 8));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicDouble() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", -1.0));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicMultiValuedDouble() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingDoubleFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingMultiValuedDoubleFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 6372));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 8));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
- public void testMissingDoubleLast() throws Exception {
+ public void testBasicDouble() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -1175,8 +1135,9 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", -1.0));
+ w.addDocument(doc);
w.commit();
doc = new Document();
@@ -1189,41 +1150,41 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
assertEquals(0, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
- public void testMissingMultiValuedDoubleLast() throws Exception {
+ public void testBasicMultiValuedDouble() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new NumericDocValuesField("id", 2));
doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
w.addDocument(doc);
w.forceMerge(1);
@@ -1242,6 +1203,207 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testMissingDoubleFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedDoubleFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingDoubleLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedDoubleLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1322,164 +1484,204 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingFloatFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedFloatFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingFloatLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(1, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedFloatLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testRandom1() throws IOException {
[2/4] lucene-solr:master: Revert "Add the ability to set the number
of hits to track accurately"
Posted by ji...@apache.org.
Revert "Add the ability to set the number of hits to track accurately"
This reverts commit 84d5015eee00992e3c0c5185e6146ed48fe3dd25.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/da7919f7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/da7919f7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/da7919f7
Branch: refs/heads/master
Commit: da7919f7a4b6c0cc951ab078b7d21122f0669c78
Parents: 84d5015
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:15:44 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:15:44 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 9 +-
.../index/TestBackwardsCompatibility.java | 27 -
.../lucene/index/sorted-invalid.7.5.0.zip | Bin 468799 -> 0 bytes
.../org/apache/lucene/index/MultiSorter.java | 30 +-
.../apache/lucene/index/TestIndexSorting.java | 1744 ++++++++----------
5 files changed, 787 insertions(+), 1023 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 007d35b..14b4961 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -314,14 +314,7 @@ Bug fixes
* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
to the same doc in the same updates package looses an update if the reset comes before
- the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
-
-* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
- can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
- values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
- (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
- detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
- (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
+ the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 44911db..6f28797 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1717,33 +1717,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dir.close();
}
}
-
- /**
- * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
- * before https://issues.apache.org/jira/browse/LUCENE-8592.
- */
- public void testSortedIndexWithInvalidSort() throws Exception {
- Path path = createTempDir("sorted");
- String name = "sorted-invalid.7.5.0.zip";
- InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
- assertNotNull("Sorted index index " + name + " not found", resource);
- TestUtil.unzip(resource, path);
-
- Directory dir = FSDirectory.open(path);
-
- DirectoryReader reader = DirectoryReader.open(dir);
- assertEquals(1, reader.leaves().size());
- Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
- assertNotNull(sort);
- assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
- reader.close();
- CheckIndex.Status status = TestUtil.checkIndex(dir);
- assertEquals(1, status.segmentInfos.size());
- assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
- assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
- "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
- dir.close();
- }
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
deleted file mode 100644
index 30c68a3..0000000
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip and /dev/null differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index d81ac75..b484228 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,18 +42,17 @@ final class MultiSorter {
SortField fields[] = sort.getSort();
final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
- final int[] reverseMuls = new int[fields.length];
for(int i=0;i<fields.length;i++) {
comparables[i] = getComparableProviders(readers, fields[i]);
- reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
+
int leafCount = readers.size();
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
for(int i=0;i<comparables.length;i++) {
- int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
+ int cmp = a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@@ -147,13 +146,14 @@ final class MultiSorter {
/** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
private interface ComparableProvider {
- Comparable getComparable(int docID) throws IOException;
+ public Comparable getComparable(int docID) throws IOException;
}
/** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
+ final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
- missingOrd = Integer.MAX_VALUE;
+ missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
} else {
- missingOrd = Integer.MIN_VALUE;
+ missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
}
if (readerDocID == docID) {
// translate segment's ord to global ord space:
- return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
+ return reverseMul * (int) globalOrds.get(readerValues.ordValue());
} else {
return missingOrd;
}
@@ -238,9 +238,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return values.longValue();
+ return reverseMul * values.longValue();
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -279,9 +279,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return (int) values.longValue();
+ return reverseMul * (int) values.longValue();
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -320,9 +320,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return Double.longBitsToDouble(values.longValue());
+ return reverseMul * Double.longBitsToDouble(values.longValue());
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -361,9 +361,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return Float.intBitsToFloat((int) values.longValue());
+ return reverseMul * Float.intBitsToFloat((int) values.longValue());
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};