You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2016/11/15 21:23:06 UTC
[1/2] lucene-solr:master: LUCENE-7537: Index time sorting now
supports multi-valued sorts using selectors (MIN, MAX, etc.)
Repository: lucene-solr
Updated Branches:
refs/heads/master 7af454ad7 -> 6c3c6bc37
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 3e72a7e..1bcac1a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -61,6 +61,8 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
@@ -115,6 +117,49 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testBasicMultiValuedString() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedSetSortField("foo", false));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("aaa")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("bcg")));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("pppp")));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
public void testMissingStringFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -152,6 +197,51 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testMissingMultiValuedStringFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", false);
+ sortField.setMissingValue(SortField.STRING_FIRST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
public void testMissingStringLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -189,6 +279,50 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testMissingMultiValuedStringLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedSetSortField("foo", false);
+ sortField.setMissingValue(SortField.STRING_LAST);
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+ doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1l, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2l, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3l, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
public void testBasicLong() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -226,6 +360,48 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testBasicMultiValuedLong() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.LONG));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 35));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", -1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 22));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
public void testMissingLongFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -255,24 +431,472 @@ public class TestIndexSorting extends LuceneTestCase {
NumericDocValues values = leaf.getNumericDocValues("foo");
// docID 0 has no value
assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedLongFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 24));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingLongLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedLongLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+ sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 65));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ doc.add(new SortedNumericDocValuesField("foo", 74));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicInt() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", -1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicMultiValuedInt() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", -1));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 22));
+ doc.add(new SortedNumericDocValuesField("foo", 27));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingIntFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(1, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(18, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedIntFirst() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 187667));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 34));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingIntLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 18));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ w.addDocument(new Document());
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("foo", 7));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(18, values.longValue());
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingMultiValuedIntLast() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+ sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ Sort indexSort = new Sort(sortField);
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", 18));
+ doc.add(new SortedNumericDocValuesField("foo", 6372));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ // missing
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", 7));
+ doc.add(new SortedNumericDocValuesField("foo", 8));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicDouble() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 18.0));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", -1.0));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new DoubleDocValuesField("foo", 7.0));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(1, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ r.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testBasicMultiValuedDouble() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
+ w.addDocument(doc);
+ // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
+ w.addDocument(doc);
+ w.commit();
+
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
+ w.addDocument(doc);
+ w.forceMerge(1);
+
+ DirectoryReader r = DirectoryReader.open(w);
+ LeafReader leaf = getOnlyLeafReader(r);
+ assertEquals(3, leaf.maxDoc());
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
+ assertEquals(1, values.nextDoc());
+ assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
}
- public void testMissingLongLast() throws Exception {
+ public void testMissingDoubleFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.LONG);
- sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
+ doc.add(new DoubleDocValuesField("foo", 18.0));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
@@ -282,7 +906,7 @@ public class TestIndexSorting extends LuceneTestCase {
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
+ doc.add(new DoubleDocValuesField("foo", 7.0));
w.addDocument(doc);
w.forceMerge(1);
@@ -290,63 +914,69 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
r.close();
w.close();
dir.close();
}
- public void testBasicInt() throws Exception {
+ public void testMissingMultiValuedDoubleFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
+ // missing
doc = new Document();
- doc.add(new NumericDocValuesField("foo", -1));
+ doc.add(new NumericDocValuesField("id", 1));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
+ NumericDocValues values = leaf.getNumericDocValues("id");
assertEquals(0, values.nextDoc());
- assertEquals(-1, values.longValue());
+ assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
+ assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
}
- public void testMissingIntFirst() throws Exception {
+ public void testMissingDoubleLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+ SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
+ doc.add(new DoubleDocValuesField("foo", 18.0));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
@@ -356,7 +986,7 @@ public class TestIndexSorting extends LuceneTestCase {
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
+ doc.add(new DoubleDocValuesField("foo", 7.0));
w.addDocument(doc);
w.forceMerge(1);
@@ -364,71 +994,79 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
+ assertEquals(0, values.nextDoc());
+ assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
- assertEquals(7, values.longValue());
- assertEquals(2, values.nextDoc());
- assertEquals(18, values.longValue());
+ assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
- public void testMissingIntLast() throws Exception {
+ public void testMissingMultiValuedDoubleLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.INT);
- sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+ sortField.setMissingValue(Double.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new NumericDocValuesField("foo", 18));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
// missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new NumericDocValuesField("foo", 7));
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
+ NumericDocValues values = leaf.getNumericDocValues("id");
assertEquals(0, values.nextDoc());
- assertEquals(7, values.longValue());
+ assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(18, values.longValue());
- assertEquals(NO_MORE_DOCS, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
}
- public void testBasicDouble() throws Exception {
+ public void testBasicFloat() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+ Sort indexSort = new Sort(new SortField("foo", SortField.Type.FLOAT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
+ doc.add(new FloatDocValuesField("foo", 18.0f));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
doc = new Document();
- doc.add(new DoubleDocValuesField("foo", -1.0));
+ doc.add(new FloatDocValuesField("foo", -1.0f));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
+ doc.add(new FloatDocValuesField("foo", 7.0f));
w.addDocument(doc);
w.forceMerge(1);
@@ -437,62 +1075,68 @@ public class TestIndexSorting extends LuceneTestCase {
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
assertEquals(0, values.nextDoc());
- assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(-1.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
r.close();
w.close();
dir.close();
}
- public void testMissingDoubleFirst() throws Exception {
+ public void testBasicMultiValuedFloat() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.NEGATIVE_INFINITY);
- Sort indexSort = new Sort(sortField);
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.FLOAT));
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(29.0f)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
- // missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(-1.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(34.0f)));
+ w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
+ NumericDocValues values = leaf.getNumericDocValues("id");
+ assertEquals(0, values.nextDoc());
+ assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
}
- public void testMissingDoubleLast() throws Exception {
+ public void testMissingFloatFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
- sortField.setMissingValue(Double.POSITIVE_INFINITY);
+ SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 18.0));
+ doc.add(new FloatDocValuesField("foo", 18.0f));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
@@ -502,7 +1146,7 @@ public class TestIndexSorting extends LuceneTestCase {
w.commit();
doc = new Document();
- doc.add(new DoubleDocValuesField("foo", 7.0));
+ doc.add(new FloatDocValuesField("foo", 7.0f));
w.addDocument(doc);
w.forceMerge(1);
@@ -510,58 +1154,64 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(0, values.nextDoc());
- assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
assertEquals(1, values.nextDoc());
- assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
+ assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.nextDoc());
+ assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
r.close();
w.close();
dir.close();
}
- public void testBasicFloat() throws Exception {
+ public void testMissingMultiValuedFloatFirst() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- Sort indexSort = new Sort(new SortField("foo", SortField.Type.FLOAT));
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
+ sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
+ doc.add(new NumericDocValuesField("id", 3));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
+ // missing
doc = new Document();
- doc.add(new FloatDocValuesField("foo", -1.0f));
+ doc.add(new NumericDocValuesField("id", 1));
w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
+ NumericDocValues values = leaf.getNumericDocValues("id");
assertEquals(0, values.nextDoc());
- assertEquals(-1.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(2, values.longValue());
assertEquals(2, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
}
- public void testMissingFloatFirst() throws Exception {
+ public void testMissingFloatLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
SortField sortField = new SortField("foo", SortField.Type.FLOAT);
- sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+ sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
@@ -584,47 +1234,55 @@ public class TestIndexSorting extends LuceneTestCase {
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
NumericDocValues values = leaf.getNumericDocValues("foo");
- assertEquals(1, values.nextDoc());
+ assertEquals(0, values.nextDoc());
assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(2, values.nextDoc());
+ assertEquals(1, values.nextDoc());
assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(NO_MORE_DOCS, values.nextDoc());
r.close();
w.close();
dir.close();
}
- public void testMissingFloatLast() throws Exception {
+ public void testMissingMultiValuedFloatLast() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
- SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+ SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
sortField.setMissingValue(Float.POSITIVE_INFINITY);
Sort indexSort = new Sort(sortField);
iwc.setIndexSort(indexSort);
IndexWriter w = new IndexWriter(dir, iwc);
Document doc = new Document();
- doc.add(new FloatDocValuesField("foo", 18.0f));
+ doc.add(new NumericDocValuesField("id", 2));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
w.addDocument(doc);
// so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
w.commit();
// missing
- w.addDocument(new Document());
+ doc = new Document();
+ doc.add(new NumericDocValuesField("id", 3));
+ w.addDocument(doc);
w.commit();
doc = new Document();
- doc.add(new FloatDocValuesField("foo", 7.0f));
+ doc.add(new NumericDocValuesField("id", 1));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+ doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
w.addDocument(doc);
w.forceMerge(1);
DirectoryReader r = DirectoryReader.open(w);
LeafReader leaf = getOnlyLeafReader(r);
assertEquals(3, leaf.maxDoc());
- NumericDocValues values = leaf.getNumericDocValues("foo");
+ NumericDocValues values = leaf.getNumericDocValues("id");
assertEquals(0, values.nextDoc());
- assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+ assertEquals(1, values.longValue());
assertEquals(1, values.nextDoc());
- assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
- assertEquals(NO_MORE_DOCS, values.nextDoc());
+ assertEquals(2, values.longValue());
+ assertEquals(2, values.nextDoc());
+ assertEquals(3, values.longValue());
r.close();
w.close();
dir.close();
@@ -703,6 +1361,58 @@ public class TestIndexSorting extends LuceneTestCase {
dir.close();
}
+ public void testMultiValuedRandom1() throws IOException {
+ boolean withDeletes = random().nextBoolean();
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.LONG));
+ iwc.setIndexSort(indexSort);
+ IndexWriter w = new IndexWriter(dir, iwc);
+ final int numDocs = atLeast(1000);
+ final FixedBitSet deleted = new FixedBitSet(numDocs);
+ for (int i = 0; i < numDocs; ++i) {
+ Document doc = new Document();
+ int num = random().nextInt(10);
+ for (int j = 0; j < num; j++) {
+ doc.add(new SortedNumericDocValuesField("foo", random().nextInt(2000)));
+ }
+ doc.add(new StringField("id", Integer.toString(i), Store.YES));
+ doc.add(new NumericDocValuesField("id", i));
+ w.addDocument(doc);
+ if (random().nextInt(5) == 0) {
+ w.getReader().close();
+ } else if (random().nextInt(30) == 0) {
+ w.forceMerge(2);
+ } else if (random().nextInt(4) == 0) {
+ final int id = TestUtil.nextInt(random(), 0, i);
+ deleted.set(id);
+ w.deleteDocuments(new Term("id", Integer.toString(id)));
+ }
+ }
+
+ DirectoryReader reader = w.getReader();
+ // Now check that the index is consistent
+ IndexSearcher searcher = newSearcher(reader);
+ for (int i = 0; i < numDocs; ++i) {
+ TermQuery termQuery = new TermQuery(new Term("id", Integer.toString(i)));
+ final TopDocs topDocs = searcher.search(termQuery, 1);
+ if (deleted.get(i)) {
+ assertEquals(0, topDocs.totalHits);
+ } else {
+ assertEquals(1, topDocs.totalHits);
+ NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
+ assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
+ assertEquals(i, values.longValue());
+ Document document = reader.document(topDocs.scoreDocs[0].doc);
+ assertEquals(Integer.toString(i), document.get("id"));
+ }
+ }
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
static class UpdateRunnable implements Runnable {
private final int numDocs;
@@ -766,7 +1476,7 @@ public class TestIndexSorting extends LuceneTestCase {
final int numDocs = atLeast(100);
Thread[] threads = new Thread[2];
-
+
final AtomicInteger updateCount = new AtomicInteger(atLeast(1000));
final CountDownLatch latch = new CountDownLatch(1);
for (int i = 0; i < threads.length; ++i) {
@@ -1008,13 +1718,13 @@ public class TestIndexSorting extends LuceneTestCase {
}
static final class NormsSimilarity extends Similarity {
-
+
private final Similarity in;
-
+
public NormsSimilarity(Similarity in) {
this.in = in;
}
-
+
@Override
public long computeNorm(FieldInvertState state) {
if (state.getName().equals("norms")) {
@@ -1023,39 +1733,39 @@ public class TestIndexSorting extends LuceneTestCase {
return in.computeNorm(state);
}
}
-
+
@Override
public SimWeight computeWeight(float boost, CollectionStatistics collectionStats, TermStatistics... termStats) {
return in.computeWeight(boost, collectionStats, termStats);
}
-
+
@Override
public SimScorer simScorer(SimWeight weight, LeafReaderContext context) throws IOException {
return in.simScorer(weight, context);
}
-
+
}
-
+
static final class PositionsTokenStream extends TokenStream {
-
+
private final CharTermAttribute term;
private final PayloadAttribute payload;
private final OffsetAttribute offset;
-
+
private int pos, off;
-
+
public PositionsTokenStream() {
term = addAttribute(CharTermAttribute.class);
payload = addAttribute(PayloadAttribute.class);
offset = addAttribute(OffsetAttribute.class);
}
-
+
@Override
public boolean incrementToken() throws IOException {
if (pos == 0) {
return false;
}
-
+
clearAttributes();
term.append("#all#");
payload.setPayload(new BytesRef(Integer.toString(pos)));
@@ -1064,7 +1774,7 @@ public class TestIndexSorting extends LuceneTestCase {
++off;
return true;
}
-
+
void setId(int id) {
pos = id / 10 + 1;
off = 0;
@@ -1105,10 +1815,10 @@ public class TestIndexSorting extends LuceneTestCase {
doc.add(norms);
doc.add(new BinaryDocValuesField("binary", new BytesRef(Integer.toString(id))));
doc.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(id))));
- doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef(Integer.toString(id))));
- doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef(Integer.toString(id + 1))));
- doc.add(new SortedNumericDocValuesField("sorted_numeric", id));
- doc.add(new SortedNumericDocValuesField("sorted_numeric", id + 1));
+ doc.add(new SortedSetDocValuesField("multi_valued_string", new BytesRef(Integer.toString(id))));
+ doc.add(new SortedSetDocValuesField("multi_valued_string", new BytesRef(Integer.toString(id + 1))));
+ doc.add(new SortedNumericDocValuesField("multi_valued_numeric", id));
+ doc.add(new SortedNumericDocValuesField("multi_valued_numeric", id + 1));
doc.add(new Field("term_vectors", Integer.toString(id), TERM_VECTORS_TYPE));
byte[] bytes = new byte[4];
NumericUtils.intToSortableBytes(id, bytes, 0);
@@ -1121,7 +1831,7 @@ public class TestIndexSorting extends LuceneTestCase {
// We add document alread in ID order for the first writer:
Directory dir1 = newFSDirectory(createTempDir());
-
+
Random random1 = new Random(seed);
IndexWriterConfig iwc1 = newIndexWriterConfig(random1, a);
iwc1.setSimilarity(new NormsSimilarity(iwc1.getSimilarity())); // for testing norms field
@@ -1138,7 +1848,7 @@ public class TestIndexSorting extends LuceneTestCase {
// We shuffle documents, but set index sort, for the second writer:
Directory dir2 = newFSDirectory(createTempDir());
-
+
Random random2 = new Random(seed);
IndexWriterConfig iwc2 = newIndexWriterConfig(random2, a);
iwc2.setSimilarity(new NormsSimilarity(iwc2.getSimilarity())); // for testing norms field
@@ -1179,10 +1889,16 @@ public class TestIndexSorting extends LuceneTestCase {
private static final class RandomDoc {
public final int id;
public final int intValue;
+ public final int[] intValues;
public final long longValue;
+ public final long[] longValues;
public final float floatValue;
+ public final float[] floatValues;
public final double doubleValue;
+ public final double[] doubleValues;
public final byte[] bytesValue;
+ public final byte[][] bytesValues;
+
public RandomDoc(int id) {
this.id = id;
@@ -1192,16 +1908,28 @@ public class TestIndexSorting extends LuceneTestCase {
doubleValue = random().nextDouble();
bytesValue = new byte[TestUtil.nextInt(random(), 1, 50)];
random().nextBytes(bytesValue);
+
+ int numValues = random().nextInt(10);
+ intValues = new int[numValues];
+ longValues = new long[numValues];
+ floatValues = new float[numValues];
+ doubleValues = new double[numValues];
+ bytesValues = new byte[numValues][];
+ for (int i = 0; i < numValues; i++) {
+ intValues[i] = random().nextInt();
+ longValues[i] = random().nextLong();
+ floatValues[i] = random().nextFloat();
+ doubleValues[i] = random().nextDouble();
+ bytesValues[i] = new byte[TestUtil.nextInt(random(), 1, 50)];
+ random().nextBytes(bytesValue);
+ }
}
}
- private static Sort randomSort() {
- int numFields = TestUtil.nextInt(random(), 1, 3);
- SortField[] sortFields = new SortField[numFields];
- for(int i=0;i<numFields-1;i++) {
- boolean reversed = random().nextBoolean();
- SortField sortField;
- switch(random().nextInt(5)) {
+ private static SortField randomIndexSortField() {
+ boolean reversed = random().nextBoolean();
+ SortField sortField;
+ switch(random().nextInt(10)) {
case 0:
sortField = new SortField("int", SortField.Type.INT, reversed);
if (random().nextBoolean()) {
@@ -1209,32 +1937,73 @@ public class TestIndexSorting extends LuceneTestCase {
}
break;
case 1:
+ sortField = new SortedNumericSortField("multi_valued_int", SortField.Type.INT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextInt());
+ }
+ break;
+ case 2:
sortField = new SortField("long", SortField.Type.LONG, reversed);
if (random().nextBoolean()) {
sortField.setMissingValue(random().nextLong());
}
break;
- case 2:
+ case 3:
+ sortField = new SortedNumericSortField("multi_valued_long", SortField.Type.LONG, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextLong());
+ }
+ break;
+ case 4:
sortField = new SortField("float", SortField.Type.FLOAT, reversed);
if (random().nextBoolean()) {
sortField.setMissingValue(random().nextFloat());
}
break;
- case 3:
+ case 5:
+ sortField = new SortedNumericSortField("multi_valued_float", SortField.Type.FLOAT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextFloat());
+ }
+ break;
+ case 6:
sortField = new SortField("double", SortField.Type.DOUBLE, reversed);
if (random().nextBoolean()) {
sortField.setMissingValue(random().nextDouble());
}
break;
- case 4:
+ case 7:
+ sortField = new SortedNumericSortField("multi_valued_double", SortField.Type.DOUBLE, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextDouble());
+ }
+ break;
+ case 8:
sortField = new SortField("bytes", SortField.Type.STRING, reversed);
if (random().nextBoolean()) {
sortField.setMissingValue(SortField.STRING_LAST);
}
break;
+ case 9:
+ sortField = new SortedSetSortField("multi_valued_bytes", reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(SortField.STRING_LAST);
+ }
+ break;
default:
- throw new AssertionError();
- }
+ sortField = null;
+ fail();
+ }
+ return sortField;
+ }
+
+
+ private static Sort randomSort() {
+ // at least 2
+ int numFields = TestUtil.nextInt(random(), 2, 4);
+ SortField[] sortFields = new SortField[numFields];
+ for(int i=0;i<numFields-1;i++) {
+ SortField sortField = randomIndexSortField();
sortFields[i] = sortField;
}
@@ -1294,6 +2063,27 @@ public class TestIndexSorting extends LuceneTestCase {
doc.add(new DoubleDocValuesField("double", docValues.doubleValue));
doc.add(new FloatDocValuesField("float", docValues.floatValue));
doc.add(new SortedDocValuesField("bytes", new BytesRef(docValues.bytesValue)));
+
+ for (int value : docValues.intValues) {
+ doc.add(new SortedNumericDocValuesField("multi_valued_int", value));
+ }
+
+ for (long value : docValues.longValues) {
+ doc.add(new SortedNumericDocValuesField("multi_valued_long", value));
+ }
+
+ for (float value : docValues.floatValues) {
+ doc.add(new SortedNumericDocValuesField("multi_valued_float", NumericUtils.floatToSortableInt(value)));
+ }
+
+ for (double value : docValues.doubleValues) {
+ doc.add(new SortedNumericDocValuesField("multi_valued_double", NumericUtils.doubleToSortableLong(value)));
+ }
+
+ for (byte[] value : docValues.bytesValues) {
+ doc.add(new SortedSetDocValuesField("multi_valued_bytes", new BytesRef(value)));
+ }
+
w1.addDocument(doc);
w2.addDocument(doc);
if (random().nextDouble() < deleteChance) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
index 49d19ae..ae5416f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseSegmentInfoFormatTestCase.java
@@ -28,6 +28,8 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.MockDirectoryWrapper;
@@ -167,6 +169,78 @@ public abstract class BaseSegmentInfoFormatTestCase extends BaseIndexFileFormatT
return true;
}
+ private SortField randomIndexSortField() {
+ boolean reversed = random().nextBoolean();
+ SortField sortField;
+ switch(random().nextInt(10)) {
+ case 0:
+ sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.INT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextInt());
+ }
+ break;
+ case 1:
+ sortField = new SortedNumericSortField(TestUtil.randomSimpleString(random()), SortField.Type.INT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextInt());
+ }
+ break;
+
+ case 2:
+ sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.LONG, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextLong());
+ }
+ break;
+ case 3:
+ sortField = new SortedNumericSortField(TestUtil.randomSimpleString(random()), SortField.Type.LONG, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextLong());
+ }
+ break;
+ case 4:
+ sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.FLOAT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextFloat());
+ }
+ break;
+ case 5:
+ sortField = new SortedNumericSortField(TestUtil.randomSimpleString(random()), SortField.Type.FLOAT, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextFloat());
+ }
+ break;
+ case 6:
+ sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.DOUBLE, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextDouble());
+ }
+ break;
+ case 7:
+ sortField = new SortedNumericSortField(TestUtil.randomSimpleString(random()), SortField.Type.DOUBLE, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(random().nextDouble());
+ }
+ break;
+ case 8:
+ sortField = new SortField(TestUtil.randomSimpleString(random()), SortField.Type.STRING, reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(SortField.STRING_LAST);
+ }
+ break;
+ case 9:
+ sortField = new SortedSetSortField(TestUtil.randomSimpleString(random()), reversed);
+ if (random().nextBoolean()) {
+ sortField.setMissingValue(SortField.STRING_LAST);
+ }
+ break;
+ default:
+ sortField = null;
+ fail();
+ }
+ return sortField;
+ }
+
/** Test sort */
public void testSort() throws IOException {
assumeTrue("test requires a codec that can read/write index sort", supportsIndexSort());
@@ -180,22 +254,7 @@ public abstract class BaseSegmentInfoFormatTestCase extends BaseIndexFileFormatT
final int numSortFields = TestUtil.nextInt(random(), 1, 3);
SortField[] sortFields = new SortField[numSortFields];
for (int j = 0; j < numSortFields; ++j) {
- sortFields[j] = new SortField(
- TestUtil.randomSimpleString(random()),
- random().nextBoolean() ? SortField.Type.LONG : SortField.Type.STRING,
- random().nextBoolean());
- if (random().nextBoolean()) {
- switch (sortFields[j].getType()) {
- case LONG:
- sortFields[j].setMissingValue(random().nextLong());
- break;
- case STRING:
- sortFields[j].setMissingValue(random().nextBoolean() ? SortField.STRING_FIRST : SortField.STRING_LAST);
- break;
- default:
- fail();
- }
- }
+ sortFields[j] = randomIndexSortField();
}
sort = new Sort(sortFields);
}
[2/2] lucene-solr:master: LUCENE-7537: Index time sorting now
supports multi-valued sorts using selectors (MIN, MAX, etc.)
Posted by mi...@apache.org.
LUCENE-7537: Index time sorting now supports multi-valued sorts using selectors (MIN, MAX, etc.)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6c3c6bc3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6c3c6bc3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6c3c6bc3
Branch: refs/heads/master
Commit: 6c3c6bc3797307efa13cae06778d41f24a26bccb
Parents: 7af454a
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Nov 15 16:22:51 2016 -0500
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Nov 15 16:22:51 2016 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../simpletext/SimpleTextSegmentInfoFormat.java | 148 ++-
.../lucene62/Lucene62SegmentInfoFormat.java | 118 +-
.../apache/lucene/index/IndexWriterConfig.java | 3 +-
.../org/apache/lucene/index/MultiSorter.java | 68 +-
.../java/org/apache/lucene/index/Sorter.java | 102 +-
.../lucene/search/SortedNumericSortField.java | 5 +
.../apache/lucene/index/TestIndexSorting.java | 1030 ++++++++++++++++--
.../index/BaseSegmentInfoFormatTestCase.java | 91 +-
9 files changed, 1308 insertions(+), 260 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a6c6dbe..bdc118b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -90,6 +90,9 @@ Improvements
which can be overridden to return false to eek out more speed in some cases.
(Timothy M. Rodriguez, David Smiley)
+* LUCENE-7537: Index time sorting now supports multi-valued sorts
+ using selectors (MIN, MAX, etc.) (Jim Ferenczi via Mike McCandless)
+
Other
* LUCENE-7546: Fixed references to benchmark wikipedia data and the Jenkins line-docs file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
index 146e92a..3d38d72 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSegmentInfoFormat.java
@@ -33,9 +33,14 @@ import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -64,6 +69,7 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
final static BytesRef SI_SORT = new BytesRef(" sort ");
final static BytesRef SI_SORT_FIELD = new BytesRef(" field ");
final static BytesRef SI_SORT_TYPE = new BytesRef(" type ");
+ final static BytesRef SI_SELECTOR_TYPE = new BytesRef(" selector ");
final static BytesRef SI_SORT_REVERSE = new BytesRef(" reverse ");
final static BytesRef SI_SORT_MISSING = new BytesRef(" missing ");
@@ -158,6 +164,8 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
final String typeAsString = readString(SI_SORT_TYPE.length, scratch);
final SortField.Type type;
+ SortedSetSelector.Type selectorSet = null;
+ SortedNumericSelector.Type selectorNumeric = null;
switch (typeAsString) {
case "string":
type = SortField.Type.STRING;
@@ -174,6 +182,26 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
case "float":
type = SortField.Type.FLOAT;
break;
+ case "multi_valued_string":
+ type = SortField.Type.STRING;
+ selectorSet = readSetSelector(input, scratch);
+ break;
+ case "multi_valued_long":
+ type = SortField.Type.LONG;
+ selectorNumeric = readNumericSelector(input, scratch);
+ break;
+ case "multi_valued_int":
+ type = SortField.Type.INT;
+ selectorNumeric = readNumericSelector(input, scratch);
+ break;
+ case "multi_valued_double":
+ type = SortField.Type.DOUBLE;
+ selectorNumeric = readNumericSelector(input, scratch);
+ break;
+ case "multi_valued_float":
+ type = SortField.Type.FLOAT;
+ selectorNumeric = readNumericSelector(input, scratch);
+ break;
default:
throw new CorruptIndexException("unable to parse sort type string: " + typeAsString, input);
}
@@ -245,7 +273,13 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
default:
throw new AssertionError();
}
- sortField[i] = new SortField(field, type, reverse);
+ if (selectorSet != null) {
+ sortField[i] = new SortedSetSortField(field, reverse);
+ } else if (selectorNumeric != null) {
+ sortField[i] = new SortedNumericSortField(field, type, reverse);
+ } else {
+ sortField[i] = new SortField(field, type, reverse);
+ }
if (missingValue != null) {
sortField[i].setMissingValue(missingValue);
}
@@ -265,6 +299,38 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
private String readString(int offset, BytesRefBuilder scratch) {
return new String(scratch.bytes(), offset, scratch.length()-offset, StandardCharsets.UTF_8);
}
+
+ private SortedSetSelector.Type readSetSelector(IndexInput input, BytesRefBuilder scratch) throws IOException {
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_SELECTOR_TYPE);
+ final String selectorAsString = readString(SI_SELECTOR_TYPE.length, scratch);
+ switch (selectorAsString) {
+ case "min":
+ return SortedSetSelector.Type.MIN;
+ case "middle_min":
+ return SortedSetSelector.Type.MIDDLE_MIN;
+ case "middle_max":
+ return SortedSetSelector.Type.MIDDLE_MAX;
+ case "max":
+ return SortedSetSelector.Type.MAX;
+ default:
+ throw new CorruptIndexException("unable to parse SortedSetSelector type: " + selectorAsString, input);
+ }
+ }
+
+ private SortedNumericSelector.Type readNumericSelector(IndexInput input, BytesRefBuilder scratch) throws IOException {
+ SimpleTextUtil.readLine(input, scratch);
+ assert StringHelper.startsWith(scratch.get(), SI_SELECTOR_TYPE);
+ final String selectorAsString = readString(SI_SELECTOR_TYPE.length, scratch);
+ switch (selectorAsString) {
+ case "min":
+ return SortedNumericSelector.Type.MIN;
+ case "max":
+ return SortedNumericSelector.Type.MAX;
+ default:
+ throw new CorruptIndexException("unable to parse SortedNumericSelector type: " + selectorAsString, input);
+ }
+ }
@Override
public void write(Directory dir, SegmentInfo si, IOContext ioContext) throws IOException {
@@ -352,29 +418,93 @@ public class SimpleTextSegmentInfoFormat extends SegmentInfoFormat {
SimpleTextUtil.writeNewline(output);
SimpleTextUtil.write(output, SI_SORT_TYPE);
- final String sortType;
- switch (sortField.getType()) {
+ final String sortTypeString;
+ final SortField.Type sortType;
+ final boolean multiValued;
+ if (sortField instanceof SortedSetSortField) {
+ sortType = SortField.Type.STRING;
+ multiValued = true;
+ } else if (sortField instanceof SortedNumericSortField) {
+ sortType = ((SortedNumericSortField) sortField).getNumericType();
+ multiValued = true;
+ } else {
+ sortType = sortField.getType();
+ multiValued = false;
+ }
+ switch (sortType) {
case STRING:
- sortType = "string";
+ if (multiValued) {
+ sortTypeString = "multi_valued_string";
+ } else {
+ sortTypeString = "string";
+ }
break;
case LONG:
- sortType = "long";
+ if (multiValued) {
+ sortTypeString = "multi_valued_long";
+ } else {
+ sortTypeString = "long";
+ }
break;
case INT:
- sortType = "int";
+ if (multiValued) {
+ sortTypeString = "multi_valued_int";
+ } else {
+ sortTypeString = "int";
+ }
break;
case DOUBLE:
- sortType = "double";
+ if (multiValued) {
+ sortTypeString = "multi_valued_double";
+ } else {
+ sortTypeString = "double";
+ }
break;
case FLOAT:
- sortType = "float";
+ if (multiValued) {
+ sortTypeString = "multi_valued_float";
+ } else {
+ sortTypeString = "float";
+ }
break;
default:
throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
}
- SimpleTextUtil.write(output, sortType, scratch);
+ SimpleTextUtil.write(output, sortTypeString, scratch);
SimpleTextUtil.writeNewline(output);
+ if (sortField instanceof SortedSetSortField) {
+ SortedSetSelector.Type selector = ((SortedSetSortField) sortField).getSelector();
+ final String selectorString;
+ if (selector == SortedSetSelector.Type.MIN) {
+ selectorString = "min";
+ } else if (selector == SortedSetSelector.Type.MIDDLE_MIN) {
+ selectorString = "middle_min";
+ } else if (selector == SortedSetSelector.Type.MIDDLE_MAX) {
+ selectorString = "middle_max";
+ } else if (selector == SortedSetSelector.Type.MAX) {
+ selectorString = "max";
+ } else {
+ throw new IllegalStateException("Unexpected SortedSetSelector type selector: " + selector);
+ }
+ SimpleTextUtil.write(output, SI_SELECTOR_TYPE);
+ SimpleTextUtil.write(output, selectorString, scratch);
+ SimpleTextUtil.writeNewline(output);
+ } else if (sortField instanceof SortedNumericSortField) {
+ SortedNumericSelector.Type selector = ((SortedNumericSortField) sortField).getSelector();
+ final String selectorString;
+ if (selector == SortedNumericSelector.Type.MIN) {
+ selectorString = "min";
+ } else if (selector == SortedNumericSelector.Type.MAX) {
+ selectorString = "max";
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSelector type selector: " + selector);
+ }
+ SimpleTextUtil.write(output, SI_SELECTOR_TYPE);
+ SimpleTextUtil.write(output, selectorString, scratch);
+ SimpleTextUtil.writeNewline(output);
+ }
+
SimpleTextUtil.write(output, SI_SORT_REVERSE);
SimpleTextUtil.write(output, Boolean.toString(sortField.getReverse()), scratch);
SimpleTextUtil.writeNewline(output);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
index 1ee5258..da6e395 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene62/Lucene62SegmentInfoFormat.java
@@ -29,6 +29,10 @@ import org.apache.lucene.index.SegmentInfo; // javadocs
import org.apache.lucene.index.SegmentInfos; // javadocs
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.Directory;
@@ -69,7 +73,7 @@ import org.apache.lucene.util.Version;
* addIndexes), etc.</li>
* <li>Files is a list of files referred to by this segment.</li>
* </ul>
- *
+ *
* @see SegmentInfos
* @lucene.experimental
*/
@@ -78,7 +82,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
/** Sole constructor. */
public Lucene62SegmentInfoFormat() {
}
-
+
@Override
public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene62SegmentInfoFormat.SI_EXTENSION);
@@ -91,13 +95,13 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
Lucene62SegmentInfoFormat.VERSION_CURRENT,
segmentID, "");
final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
-
+
final int docCount = input.readInt();
if (docCount < 0) {
throw new CorruptIndexException("invalid docCount: " + docCount, input);
}
final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
-
+
final Map<String,String> diagnostics = input.readMapOfStrings();
final Set<String> files = input.readSetOfStrings();
final Map<String,String> attributes = input.readMapOfStrings();
@@ -110,6 +114,8 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
String fieldName = input.readString();
int sortTypeID = input.readVInt();
SortField.Type sortType;
+ SortedSetSelector.Type sortedSetSelector = null;
+ SortedNumericSelector.Type sortedNumericSelector = null;
switch(sortTypeID) {
case 0:
sortType = SortField.Type.STRING;
@@ -126,6 +132,43 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
case 4:
sortType = SortField.Type.FLOAT;
break;
+ case 5:
+ sortType = SortField.Type.STRING;
+ byte selector = input.readByte();
+ if (selector == 0) {
+ sortedSetSelector = SortedSetSelector.Type.MIN;
+ } else if (selector == 1) {
+ sortedSetSelector = SortedSetSelector.Type.MAX;
+ } else if (selector == 2) {
+ sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;
+ } else if (selector == 3) {
+ sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;
+ } else {
+ throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input);
+ }
+ break;
+ case 6:
+ byte type = input.readByte();
+ if (type == 0) {
+ sortType = SortField.Type.LONG;
+ } else if (type == 1) {
+ sortType = SortField.Type.INT;
+ } else if (type == 2) {
+ sortType = SortField.Type.DOUBLE;
+ } else if (type == 3) {
+ sortType = SortField.Type.FLOAT;
+ } else {
+ throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input);
+ }
+ byte numericSelector = input.readByte();
+ if (numericSelector == 0) {
+ sortedNumericSelector = SortedNumericSelector.Type.MIN;
+ } else if (numericSelector == 1) {
+ sortedNumericSelector = SortedNumericSelector.Type.MAX;
+ } else {
+ throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input);
+ }
+ break;
default:
throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input);
}
@@ -139,7 +182,13 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
throw new CorruptIndexException("invalid index sort reverse: " + b, input);
}
- sortFields[i] = new SortField(fieldName, sortType, reverse);
+ if (sortedSetSelector != null) {
+ sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector);
+ } else if (sortedNumericSelector != null) {
+ sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector);
+ } else {
+ sortFields[i] = new SortField(fieldName, sortType, reverse);
+ }
Object missingValue;
b = input.readByte();
@@ -194,7 +243,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
} else {
indexSort = null;
}
-
+
si = new SegmentInfo(dir, version, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
si.setFiles(files);
} catch (Throwable exception) {
@@ -213,8 +262,8 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
try (IndexOutput output = dir.createOutput(fileName, ioContext)) {
// Only add the file once we've successfully created it, else IFD assert can trip:
si.addFile(fileName);
- CodecUtil.writeIndexHeader(output,
- Lucene62SegmentInfoFormat.CODEC_NAME,
+ CodecUtil.writeIndexHeader(output,
+ Lucene62SegmentInfoFormat.CODEC_NAME,
Lucene62SegmentInfoFormat.VERSION_CURRENT,
si.getId(),
"");
@@ -245,6 +294,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
output.writeVInt(numSortFields);
for (int i = 0; i < numSortFields; ++i) {
SortField sortField = indexSort.getSort()[i];
+ SortField.Type sortType = sortField.getType();
output.writeString(sortField.getField());
int sortTypeID;
switch (sortField.getType()) {
@@ -263,10 +313,55 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
case FLOAT:
sortTypeID = 4;
break;
+ case CUSTOM:
+ if (sortField instanceof SortedSetSortField) {
+ sortTypeID = 5;
+ sortType = SortField.Type.STRING;
+ } else if (sortField instanceof SortedNumericSortField) {
+ sortTypeID = 6;
+ sortType = ((SortedNumericSortField) sortField).getNumericType();
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSortField " + sortField);
+ }
+ break;
default:
throw new IllegalStateException("Unexpected sort type: " + sortField.getType());
}
output.writeVInt(sortTypeID);
+ if (sortTypeID == 5) {
+ SortedSetSortField ssf = (SortedSetSortField) sortField;
+ if (ssf.getSelector() == SortedSetSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) {
+ output.writeByte((byte) 2);
+ } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedSetSelector type: " + ssf.getSelector());
+ }
+ } else if (sortTypeID == 6) {
+ SortedNumericSortField snsf = (SortedNumericSortField) sortField;
+ if (snsf.getNumericType() == SortField.Type.LONG) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getNumericType() == SortField.Type.INT) {
+ output.writeByte((byte) 1);
+ } else if (snsf.getNumericType() == SortField.Type.DOUBLE) {
+ output.writeByte((byte) 2);
+ } else if (snsf.getNumericType() == SortField.Type.FLOAT) {
+ output.writeByte((byte) 3);
+ } else {
+ throw new IllegalStateException("Unexpected SortedNumericSelector type: " + snsf.getNumericType());
+ }
+ if (snsf.getSelector() == SortedNumericSelector.Type.MIN) {
+ output.writeByte((byte) 0);
+ } else if (snsf.getSelector() == SortedNumericSelector.Type.MAX) {
+ output.writeByte((byte) 1);
+ } else {
+ throw new IllegalStateException("Unexpected sorted numeric selector type: " + snsf.getSelector());
+ }
+ }
output.writeByte((byte) (sortField.getReverse() ? 0 : 1));
// write missing value
@@ -274,7 +369,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
if (missingValue == null) {
output.writeByte((byte) 0);
} else {
- switch(sortField.getType()) {
+ switch(sortType) {
case STRING:
if (missingValue == SortField.STRING_LAST) {
output.writeByte((byte) 1);
@@ -305,7 +400,7 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
}
}
}
-
+
CodecUtil.writeFooter(output);
}
}
@@ -314,5 +409,6 @@ public class Lucene62SegmentInfoFormat extends SegmentInfoFormat {
public final static String SI_EXTENSION = "si";
static final String CODEC_NAME = "Lucene62SegmentInfo";
static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_MULTI_VALUED_SORT = 1;
+ static final int VERSION_CURRENT = VERSION_MULTI_VALUED_SORT;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
index 368259a..4f642ee 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriterConfig.java
@@ -468,7 +468,8 @@ public final class IndexWriterConfig extends LiveIndexWriterConfig {
*/
public IndexWriterConfig setIndexSort(Sort sort) {
for(SortField sortField : sort.getSort()) {
- if (ALLOWED_INDEX_SORT_TYPES.contains(sortField.getType()) == false) {
+ final SortField.Type sortType = Sorter.getSortFieldType(sortField);
+ if (ALLOWED_INDEX_SORT_TYPES.contains(sortType) == false) {
throw new IllegalArgumentException("invalid SortField type: must be one of " + ALLOWED_INDEX_SORT_TYPES + " but got: " + sortField);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index ee969c7..5ca6b65 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -141,33 +141,25 @@ final class MultiSorter {
private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
ComparableProvider[] providers = new ComparableProvider[readers.size()];
+ final int reverseMul = sortField.getReverse() ? -1 : 1;
+ final SortField.Type sortType = Sorter.getSortFieldType(sortField);
- switch(sortField.getType()) {
+ switch(sortType) {
case STRING:
{
// this uses the efficient segment-local ordinal map:
final SortedDocValues[] values = new SortedDocValues[readers.size()];
for(int i=0;i<readers.size();i++) {
- SortedDocValues v = readers.get(i).getSortedDocValues(sortField.getField());
- if (v == null) {
- v = DocValues.emptySorted();
- }
- values[i] = v;
+ final SortedDocValues sorted = Sorter.getOrWrapSorted(readers.get(i), sortField);
+ values[i] = sorted;
}
MultiDocValues.OrdinalMap ordinalMap = MultiDocValues.OrdinalMap.build(null, values, PackedInts.DEFAULT);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
- missingOrd = Integer.MAX_VALUE;
+ missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
} else {
- missingOrd = Integer.MIN_VALUE;
- }
-
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
+ missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -205,13 +197,6 @@ final class MultiSorter {
case LONG:
{
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
final Long missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Long) sortField.getMissingValue();
@@ -220,8 +205,8 @@ final class MultiSorter {
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
- final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
-
+ final NumericDocValues values = Sorter.getOrWrapNumeric(readers.get(readerIndex), sortField);
+
providers[readerIndex] = new ComparableProvider() {
// used only by assert:
int lastDocID = -1;
@@ -243,7 +228,7 @@ final class MultiSorter {
if (readerDocID == docID) {
return reverseMul * values.longValue();
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -253,13 +238,6 @@ final class MultiSorter {
case INT:
{
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
final Integer missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Integer) sortField.getMissingValue();
@@ -268,7 +246,7 @@ final class MultiSorter {
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
- final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+ final NumericDocValues values = Sorter.getOrWrapNumeric(readers.get(readerIndex), sortField);
providers[readerIndex] = new ComparableProvider() {
// used only by assert:
@@ -291,7 +269,7 @@ final class MultiSorter {
if (readerDocID == docID) {
return reverseMul * (int) values.longValue();
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -301,13 +279,6 @@ final class MultiSorter {
case DOUBLE:
{
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
final Double missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Double) sortField.getMissingValue();
@@ -316,7 +287,7 @@ final class MultiSorter {
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
- final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+ final NumericDocValues values = Sorter.getOrWrapNumeric(readers.get(readerIndex), sortField);
providers[readerIndex] = new ComparableProvider() {
// used only by assert:
@@ -339,7 +310,7 @@ final class MultiSorter {
if (readerDocID == docID) {
return reverseMul * Double.longBitsToDouble(values.longValue());
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
@@ -349,13 +320,6 @@ final class MultiSorter {
case FLOAT:
{
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
final Float missingValue;
if (sortField.getMissingValue() != null) {
missingValue = (Float) sortField.getMissingValue();
@@ -364,7 +328,7 @@ final class MultiSorter {
}
for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
- final NumericDocValues values = DocValues.getNumeric(readers.get(readerIndex), sortField.getField());
+ final NumericDocValues values = Sorter.getOrWrapNumeric(readers.get(readerIndex), sortField);
providers[readerIndex] = new ComparableProvider() {
// used only by assert:
@@ -387,7 +351,7 @@ final class MultiSorter {
if (readerDocID == docID) {
return reverseMul * Float.intBitsToFloat((int) values.longValue());
} else {
- return missingValue;
+ return reverseMul * missingValue;
}
}
};
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/java/org/apache/lucene/index/Sorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Sorter.java b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
index 75816ba..e647435 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Sorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
@@ -25,6 +25,10 @@ import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.SortedNumericSelector;
+import org.apache.lucene.search.SortedNumericSortField;
+import org.apache.lucene.search.SortedSetSelector;
+import org.apache.lucene.search.SortedSetSortField;
import org.apache.lucene.util.TimSorter;
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
@@ -200,18 +204,55 @@ final class Sorter {
};
}
+ /** Returns the native sort type for {@link SortedSetSortField} and {@link SortedNumericSortField},
+ * {@link SortField#getType()} otherwise */
+ static SortField.Type getSortFieldType(SortField sortField) {
+ if (sortField instanceof SortedSetSortField) {
+ return SortField.Type.STRING;
+ } else if (sortField instanceof SortedNumericSortField) {
+ return ((SortedNumericSortField) sortField).getNumericType();
+ } else {
+ return sortField.getType();
+ }
+ }
+
+ /** Wraps a {@link SortedNumericDocValues} as a single-valued view if the field is an instance of {@link SortedNumericSortField},
+ * returns {@link NumericDocValues} for the field otherwise. */
+ static NumericDocValues getOrWrapNumeric(LeafReader reader, SortField sortField) throws IOException {
+ if (sortField instanceof SortedNumericSortField) {
+ SortedNumericSortField sf = (SortedNumericSortField) sortField;
+ return SortedNumericSelector.wrap(DocValues.getSortedNumeric(reader, sf.getField()), sf.getSelector(), sf.getNumericType());
+ } else {
+ return DocValues.getNumeric(reader, sortField.getField());
+ }
+ }
+
+ /** Wraps a {@link SortedSetDocValues} as a single-valued view if the field is an instance of {@link SortedSetSortField},
+ * returns {@link SortedDocValues} for the field otherwise. */
+ static SortedDocValues getOrWrapSorted(LeafReader reader, SortField sortField) throws IOException {
+ if (sortField instanceof SortedSetSortField) {
+ SortedSetSortField sf = (SortedSetSortField) sortField;
+ return SortedSetSelector.wrap(DocValues.getSortedSet(reader, sf.getField()), sf.getSelector());
+ } else {
+ return DocValues.getSorted(reader, sortField.getField());
+ }
+ }
+
/** We cannot use the {@link FieldComparator} API because that API requires that you send it docIDs in order. Note that this API
* allocates arrays[maxDoc] to hold the native values needed for comparison, but 1) they are transient (only alive while sorting this one
* segment), and 2) in the typical index sorting case, they are only used to sort newly flushed segments, which will be smaller than
* merged segments. */
private static DocComparator getDocComparator(LeafReader reader, SortField sortField) throws IOException {
- int maxDoc = reader.maxDoc();
-
- switch(sortField.getType()) {
+ final int maxDoc = reader.maxDoc();
+ final int reverseMul = sortField.getReverse() ? -1 : 1;
+ final SortField.Type sortType = getSortFieldType(sortField);
+
+ switch(sortType) {
case STRING:
{
+ final SortedDocValues sorted = getOrWrapSorted(reader, sortField);
final int missingOrd;
if (sortField.getMissingValue() == SortField.STRING_LAST) {
missingOrd = Integer.MAX_VALUE;
@@ -221,18 +262,10 @@ final class Sorter {
final int[] ords = new int[reader.maxDoc()];
Arrays.fill(ords, missingOrd);
- SortedDocValues sorted = DocValues.getSorted(reader, sortField.getField());
int docID;
while ((docID = sorted.nextDoc()) != NO_MORE_DOCS) {
ords[docID] = sorted.ordValue();
}
-
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
return new DocComparator() {
@Override
@@ -244,9 +277,8 @@ final class Sorter {
case LONG:
{
+ final NumericDocValues dvs = getOrWrapNumeric(reader, sortField);
long[] values = new long[maxDoc];
- NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
-
if (sortField.getMissingValue() != null) {
Arrays.fill(values, (Long) sortField.getMissingValue());
}
@@ -258,13 +290,6 @@ final class Sorter {
values[docID] = dvs.longValue();
}
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
return new DocComparator() {
@Override
public int compare(int docID1, int docID2) {
@@ -275,13 +300,12 @@ final class Sorter {
case INT:
{
+ final NumericDocValues dvs = getOrWrapNumeric(reader, sortField);
int[] values = new int[maxDoc];
- NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
-
if (sortField.getMissingValue() != null) {
Arrays.fill(values, (Integer) sortField.getMissingValue());
}
-
+
while (true) {
int docID = dvs.nextDoc();
if (docID == NO_MORE_DOCS) {
@@ -290,13 +314,6 @@ final class Sorter {
values[docID] = (int) dvs.longValue();
}
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
return new DocComparator() {
@Override
public int compare(int docID1, int docID2) {
@@ -307,9 +324,8 @@ final class Sorter {
case DOUBLE:
{
+ final NumericDocValues dvs = getOrWrapNumeric(reader, sortField);
double[] values = new double[maxDoc];
- NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
-
if (sortField.getMissingValue() != null) {
Arrays.fill(values, (Double) sortField.getMissingValue());
}
@@ -320,13 +336,6 @@ final class Sorter {
}
values[docID] = Double.longBitsToDouble(dvs.longValue());
}
-
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
return new DocComparator() {
@Override
@@ -338,9 +347,8 @@ final class Sorter {
case FLOAT:
{
+ final NumericDocValues dvs = getOrWrapNumeric(reader, sortField);
float[] values = new float[maxDoc];
- NumericDocValues dvs = DocValues.getNumeric(reader, sortField.getField());
-
if (sortField.getMissingValue() != null) {
Arrays.fill(values, (Float) sortField.getMissingValue());
}
@@ -352,13 +360,6 @@ final class Sorter {
values[docID] = Float.intBitsToFloat((int) dvs.longValue());
}
- final int reverseMul;
- if (sortField.getReverse()) {
- reverseMul = -1;
- } else {
- reverseMul = 1;
- }
-
return new DocComparator() {
@Override
public int compare(int docID1, int docID2) {
@@ -371,7 +372,7 @@ final class Sorter {
throw new IllegalArgumentException("unhandled SortField.getType()=" + sortField.getType());
}
}
-
+
/**
* Returns a mapping from the old document ID to its new location in the
* sorted index. Implementations can use the auxiliary
@@ -386,7 +387,6 @@ final class Sorter {
*/
DocMap sort(LeafReader reader) throws IOException {
SortField fields[] = sort.getSort();
- final int reverseMul[] = new int[fields.length];
final DocComparator comparators[] = new DocComparator[fields.length];
for (int i = 0; i < fields.length; i++) {
@@ -424,7 +424,7 @@ final class Sorter {
public String toString() {
return getID();
}
-
+
static final Scorer FAKESCORER = new Scorer(null) {
float score;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c3c6bc3/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
index 22d7457..5b1492d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortedNumericSortField.java
@@ -82,6 +82,11 @@ public class SortedNumericSortField extends SortField {
this.selector = selector;
this.type = type;
}
+
+ /** Returns the numeric type in use for this sort */
+ public SortField.Type getNumericType() {
+ return type;
+ }
/** Returns the selector in use for this sort */
public SortedNumericSelector.Type getSelector() {