You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ji...@apache.org on 2018/12/07 18:32:01 UTC
[1/3] lucene-solr:branch_7x: LUCENE-8592: Fix index sorting
corruption due to numeric overflow
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x f1759301c -> d9cd9f78b
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16a76883/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 2d8d48d..60d36c1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -354,166 +354,206 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingStringFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- // docID 0 is missing:
- assertEquals(1, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(2, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ } else {
+ // docID 0 is missing:
+ assertEquals(1, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedStringFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", false);
- sortField.setMissingValue(SortField.STRING_FIRST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", reverse);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingStringLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.STRING);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- SortedDocValues values = leaf.getSortedDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals("mmm", values.binaryValue().utf8ToString());
- assertEquals(1, values.nextDoc());
- assertEquals("zzz", values.binaryValue().utf8ToString());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ SortedDocValues values = leaf.getSortedDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ assertEquals(2, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals("mmm", values.binaryValue().utf8ToString());
+ assertEquals(1, values.nextDoc());
+ assertEquals("zzz", values.binaryValue().utf8ToString());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedStringLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedSetSortField("foo", false);
- sortField.setMissingValue(SortField.STRING_LAST);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", reverse);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
- doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1l, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2l, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3l, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testBasicLong() throws Exception {
@@ -596,11 +636,213 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingLongFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ // docID 0 has no value
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedLongFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 24));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingLongLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ // docID 0 is missing
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedLongLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 65));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ doc.add(new SortedNumericDocValuesField("foo", 74));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testBasicInt() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -609,8 +851,9 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", -1));
+ w.addDocument(doc);
w.commit();
doc = new Document();
@@ -622,7 +865,8 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- // docID 0 has no value
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
assertEquals(1, values.nextDoc());
assertEquals(7, values.longValue());
assertEquals(2, values.nextDoc());
@@ -632,32 +876,32 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testMissingMultiValuedLongFirst() throws Exception {
+ public void testBasicMultiValuedInt() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
doc.add(new NumericDocValuesField("id", 3));
doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 27));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
doc = new Document();
doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", -1));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
w.addDocument(doc);
w.commit();
doc = new Document();
doc.add(new NumericDocValuesField("id", 2));
doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 24));
+ doc.add(new SortedNumericDocValuesField("foo", 22));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
w.addDocument(doc);
w.forceMerge(1);
@@ -676,496 +920,212 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
- public void testMissingLongLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ public void testMissingIntFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingMultiValuedLongLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 65));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- doc.add(new SortedNumericDocValuesField("foo", 74));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicInt() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", -1));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(-1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicMultiValuedInt() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", -1));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 22));
- doc.add(new SortedNumericDocValuesField("foo", 27));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingIntFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedIntFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 187667));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 187667));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 34));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingIntLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ // docID 0 is missing
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(7, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedIntLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", 18));
- doc.add(new SortedNumericDocValuesField("foo", 6372));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", 7));
- doc.add(new SortedNumericDocValuesField("foo", 8));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicDouble() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", -1.0));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- r.close();
- w.close();
- dir.close();
- }
-
- public void testBasicMultiValuedDouble() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingDoubleFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- w.addDocument(new Document());
- w.commit();
-
- doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
- w.addDocument(doc);
- w.forceMerge(1);
-
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- r.close();
- w.close();
- dir.close();
- }
-
- public void testMissingMultiValuedDoubleFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
-
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
-
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
- w.addDocument(doc);
- w.forceMerge(1);
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 6372));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 8));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
- public void testMissingDoubleLast() throws Exception {
+ public void testBasicDouble() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
@@ -1174,8 +1134,9 @@ public class TestIndexSorting extends LuceneTestCase {
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", -1.0));
+ w.addDocument(doc);
w.commit();
doc = new Document();
@@ -1188,41 +1149,41 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
assertEquals(0, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
- public void testMissingMultiValuedDoubleLast() throws Exception {
+ public void testBasicMultiValuedDouble() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new NumericDocValuesField("id", 2));
doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
w.addDocument(doc);
w.forceMerge(1);
@@ -1241,6 +1202,207 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testMissingDoubleFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedDoubleFirst() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingDoubleLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
+ public void testMissingMultiValuedDoubleLast() throws Exception {
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
+ }
+
public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1321,164 +1483,204 @@ public class TestIndexSorting extends LuceneTestCase {
}
public void testMissingFloatFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(2, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ } else {
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedFloatFirst() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingFloatLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 18.0f));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- w.addDocument(new Document());
- w.commit();
+ // missing
+ w.addDocument(new Document());
+ w.commit();
- doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new FloatDocValuesField("foo", 7.0f));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(1, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ if (reverse) {
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ }
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testMissingMultiValuedFloatLast() throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.POSITIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
- iwc.setIndexSort(indexSort);
- IndexWriter w = new IndexWriter(dir, iwc);
- Document doc = new Document();
- doc.add(new NumericDocValuesField("id", 2));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
- w.addDocument(doc);
- // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
- w.commit();
+ for (boolean reverse : new boolean[] {true, false}) {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
- // missing
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 3));
- w.addDocument(doc);
- w.commit();
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
- doc = new Document();
- doc.add(new NumericDocValuesField("id", 1));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
- doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
- w.addDocument(doc);
- w.forceMerge(1);
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ w.addDocument(doc);
+ w.forceMerge(1);
- DirectoryReader r = DirectoryReader.open(w);
- LeafReader leaf = getOnlyLeafReader(r);
- assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("id");
- assertEquals(0, values.nextDoc());
- assertEquals(1, values.longValue());
- assertEquals(1, values.nextDoc());
- assertEquals(2, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(3, values.longValue());
- r.close();
- w.close();
- dir.close();
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ if (reverse) {
+ assertEquals(0, values.nextDoc());
+ assertEquals(3, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(1, values.longValue());
+ } else {
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ }
+ r.close();
+ w.close();
+ dir.close();
+ }
}
public void testRandom1() throws IOException {
[2/3] lucene-solr:branch_7x: LUCENE-8592: Fix index sorting
corruption due to numeric overflow
Posted by ji...@apache.org.
LUCENE-8592: Fix index sorting corruption due to numeric overflow
The merge sort of sorted segments can produce an invalid
sort if the sort field is an Integer/Long that uses reverse order and contains values equal to
Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug.
Indices affected by the bug can be detected by running the CheckIndex command on a
distribution that contains the fix (7.6+).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/16a76883
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/16a76883
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/16a76883
Branch: refs/heads/branch_7x
Commit: 16a76883eba7a6c70a70b634fbe6cf89712f2d97
Parents: f175930
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:16:42 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:18:06 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 9 +-
.../index/TestBackwardsCompatibility.java | 27 +
.../lucene/index/sorted-invalid.7.5.0.zip | Bin 0 -> 468799 bytes
.../org/apache/lucene/index/MultiSorter.java | 30 +-
.../apache/lucene/index/TestIndexSorting.java | 1744 ++++++++++--------
5 files changed, 1023 insertions(+), 787 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16a76883/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6041fdf..1ad7a6c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -104,7 +104,14 @@ Bug fixes
* LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
to the same doc in the same updates package looses an update if the reset comes before
- the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+ the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
+
+* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
+ can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
+ values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
+ (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
+ detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
+ (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
New Features
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16a76883/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 9c44f48..9962901 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1641,6 +1641,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
dir.close();
}
}
+
+ /**
+ * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
+ * before https://issues.apache.org/jira/browse/LUCENE-8592.
+ */
+ public void testSortedIndexWithInvalidSort() throws Exception {
+ Path path = createTempDir("sorted");
+ String name = "sorted-invalid.7.5.0.zip";
+ InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
+ assertNotNull("Sorted index index " + name + " not found", resource);
+ TestUtil.unzip(resource, path);
+
+ Directory dir = FSDirectory.open(path);
+
+ DirectoryReader reader = DirectoryReader.open(dir);
+ assertEquals(1, reader.leaves().size());
+ Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
+ assertNotNull(sort);
+ assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
+ reader.close();
+ CheckIndex.Status status = TestUtil.checkIndex(dir);
+ assertEquals(1, status.segmentInfos.size());
+ assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
+ assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
+ "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
+ dir.close();
+ }
static long getValue(BinaryDocValues bdv) throws IOException {
BytesRef term = bdv.binaryValue();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16a76883/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
new file mode 100644
index 0000000..30c68a3
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16a76883/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index b484228..d81ac75 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,17 +42,18 @@ final class MultiSorter {
SortField fields[] = sort.getSort();
final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
+ final int[] reverseMuls = new int[fields.length];
for(int i=0;i<fields.length;i++) {
comparables[i] = getComparableProviders(readers, fields[i]);
+ reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
}
-
int leafCount = readers.size();
PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
@Override
public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
for(int i=0;i<comparables.length;i++) {
- int cmp = a.values[i].compareTo(b.values[i]);
+ int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
if (cmp != 0) {
return cmp < 0;
}
@@ -146,14 +147,13 @@ final class MultiSorter {
/** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
private interface ComparableProvider {
- public Comparable getComparable(int docID) throws IOException;
+ Comparable getComparable(int docID) throws IOException;
}
/** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
- final int reverseMul = sortField.getReverse() ? -1 : 1;
final SortField.Type sortType = Sorter.getSortFieldType(sortField);
switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
- missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+ missingOrd = Integer.MAX_VALUE;
} else {
- missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+ missingOrd = Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
}
if (readerDocID == docID) {
// translate segment's ord to global ord space:
- return reverseMul * (int) globalOrds.get(readerValues.ordValue());
+ return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
} else {
return missingOrd;
}
@@ -238,9 +238,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * values.longValue();
+ return values.longValue();
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -279,9 +279,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * (int) values.longValue();
+ return (int) values.longValue();
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -320,9 +320,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * Double.longBitsToDouble(values.longValue());
+ return Double.longBitsToDouble(values.longValue());
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
@@ -361,9 +361,9 @@ final class MultiSorter {
readerDocID = values.advance(docID);
}
if (readerDocID == docID) {
- return reverseMul * Float.intBitsToFloat((int) values.longValue());
+ return Float.intBitsToFloat((int) values.longValue());
} else {
- return reverseMul * missingValue;
+ return missingValue;
}
}
};
[3/3] lucene-solr:branch_7x: LUCENE-8592: switch the corrupted sorted
index to a 7.6 version (instead of 7x).
Posted by ji...@apache.org.
LUCENE-8592: switch the corrupted sorted index to a 7.6 version (instead of 7x).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d9cd9f78
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d9cd9f78
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d9cd9f78
Branch: refs/heads/branch_7x
Commit: d9cd9f78b1182125a7fb02d724608d5f355df785
Parents: 16a7688
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:29:40 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:30:30 2018 +0100
----------------------------------------------------------------------
.../lucene/index/sorted-invalid.7.5.0.zip | Bin 468799 -> 908443 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d9cd9f78/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
index 30c68a3..615afdf 100644
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ