You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/12/11 09:27:30 UTC

[01/47] lucene-solr:jira/http2: LUCENE-8591: add LegacyBM25Similarity#getDiscountOverlaps

Repository: lucene-solr
Updated Branches:
  refs/heads/jira/http2 0ecb8110e -> 925fb154f


LUCENE-8591: add LegacyBM25Similarity#getDiscountOverlaps

Signed-off-by: Adrien Grand <jp...@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b24af10d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b24af10d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b24af10d

Branch: refs/heads/jira/http2
Commit: b24af10d59b15d4b79418c9d7af958aa0ac7c39a
Parents: f0db05b
Author: Luca Cavanna <ja...@users.noreply.github.com>
Authored: Wed Dec 5 16:08:13 2018 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 7 12:31:22 2018 +0100

----------------------------------------------------------------------
 .../lucene/search/similarity/LegacyBM25Similarity.java       | 8 ++++++++
 1 file changed, 8 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b24af10d/lucene/misc/src/java/org/apache/lucene/search/similarity/LegacyBM25Similarity.java
----------------------------------------------------------------------
diff --git a/lucene/misc/src/java/org/apache/lucene/search/similarity/LegacyBM25Similarity.java b/lucene/misc/src/java/org/apache/lucene/search/similarity/LegacyBM25Similarity.java
index 58091a7..300c1c0 100644
--- a/lucene/misc/src/java/org/apache/lucene/search/similarity/LegacyBM25Similarity.java
+++ b/lucene/misc/src/java/org/apache/lucene/search/similarity/LegacyBM25Similarity.java
@@ -89,6 +89,14 @@ public final class LegacyBM25Similarity extends Similarity {
     bm25Similarity.setDiscountOverlaps(v);
   }
 
+  /**
+   * Returns true if overlap tokens are discounted from the document's length.
+   * @see #setDiscountOverlaps
+   */
+  public boolean getDiscountOverlaps() {
+    return bm25Similarity.getDiscountOverlaps();
+  }
+
   @Override
   public String toString() {
     return bm25Similarity.toString();


[04/47] lucene-solr:jira/http2: Add the ability to set the number of hits to track accurately

Posted by da...@apache.org.
Add the ability to set the number of hits to track accurately

In Lucene 8 searches can skip non-competitive hits if the total hit count is not requested.
It is also possible to track the number of hits up to a certain threshold. This is a trade off to speed up searches while still being able to know a lower bound of the total hit count. This change adds the ability to set this threshold directly in the track_total_hits search option. A boolean value (true, false) indicates whether the total hit count should be tracked in the response. When set as an integer this option allows to compute a lower bound of the total hits while preserving the ability to skip non-competitive hits when enough matches have been collected.

Relates #33028


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/84d5015e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/84d5015e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/84d5015e

Branch: refs/heads/jira/http2
Commit: 84d5015eee00992e3c0c5185e6146ed48fe3dd25
Parents: 17fca05
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:11:59 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:11:59 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |    9 +-
 .../index/TestBackwardsCompatibility.java       |   27 +
 .../lucene/index/sorted-invalid.7.5.0.zip       |  Bin 0 -> 468799 bytes
 .../org/apache/lucene/index/MultiSorter.java    |   30 +-
 .../apache/lucene/index/TestIndexSorting.java   | 1738 ++++++++++--------
 5 files changed, 1020 insertions(+), 784 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84d5015e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 14b4961..007d35b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -314,7 +314,14 @@ Bug fixes
 
 * LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
   to the same doc in the same updates package looses an update if the reset comes before
-  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
+
+* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
+  can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
+  values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
+  (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
+  detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
+  (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
 
 New Features
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84d5015e/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 6f28797..44911db 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1717,6 +1717,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       dir.close();
     }
   }
+
+  /**
+   * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
+   * before https://issues.apache.org/jira/browse/LUCENE-8592.
+   */
+  public void testSortedIndexWithInvalidSort() throws Exception {
+    Path path = createTempDir("sorted");
+    String name = "sorted-invalid.7.5.0.zip";
+    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
+    assertNotNull("Sorted index index " + name + " not found", resource);
+    TestUtil.unzip(resource, path);
+
+    Directory dir = FSDirectory.open(path);
+
+    DirectoryReader reader = DirectoryReader.open(dir);
+    assertEquals(1, reader.leaves().size());
+    Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
+    assertNotNull(sort);
+    assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
+    reader.close();
+    CheckIndex.Status status = TestUtil.checkIndex(dir);
+    assertEquals(1, status.segmentInfos.size());
+    assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
+    assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
+        "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
+    dir.close();
+  }
   
   static long getValue(BinaryDocValues bdv) throws IOException {
     BytesRef term = bdv.binaryValue();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84d5015e/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
new file mode 100644
index 0000000..30c68a3
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84d5015e/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index b484228..d81ac75 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,17 +42,18 @@ final class MultiSorter {
 
     SortField fields[] = sort.getSort();
     final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
+    final int[] reverseMuls = new int[fields.length];
     for(int i=0;i<fields.length;i++) {
       comparables[i] = getComparableProviders(readers, fields[i]);
+      reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
     }
-
     int leafCount = readers.size();
 
     PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
         @Override
         public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
           for(int i=0;i<comparables.length;i++) {
-            int cmp = a.values[i].compareTo(b.values[i]);
+            int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
             if (cmp != 0) {
               return cmp < 0;
             }
@@ -146,14 +147,13 @@ final class MultiSorter {
 
   /** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
   private interface ComparableProvider {
-    public Comparable getComparable(int docID) throws IOException;
+    Comparable getComparable(int docID) throws IOException;
   }
 
   /** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
   private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
 
     ComparableProvider[] providers = new ComparableProvider[readers.size()];
-    final int reverseMul = sortField.getReverse() ? -1 : 1;
     final SortField.Type sortType = Sorter.getSortFieldType(sortField);
 
     switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
         OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
         final int missingOrd;
         if (sortField.getMissingValue() == SortField.STRING_LAST) {
-          missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+          missingOrd = Integer.MAX_VALUE;
         } else {
-          missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+          missingOrd = Integer.MIN_VALUE;
         }
 
         for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
                 }
                 if (readerDocID == docID) {
                   // translate segment's ord to global ord space:
-                  return reverseMul * (int) globalOrds.get(readerValues.ordValue());
+                  return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
                 } else {
                   return missingOrd;
                 }
@@ -238,9 +238,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * values.longValue();
+                  return values.longValue();
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -279,9 +279,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * (int) values.longValue();
+                  return (int) values.longValue();
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -320,9 +320,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * Double.longBitsToDouble(values.longValue());
+                  return Double.longBitsToDouble(values.longValue());
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -361,9 +361,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * Float.intBitsToFloat((int) values.longValue());
+                  return Float.intBitsToFloat((int) values.longValue());
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };


[03/47] lucene-solr:jira/http2: Add the ability to set the number of hits to track accurately

Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/84d5015e/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 3857a97..5bfd9d5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -355,166 +355,206 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingStringFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.STRING);
-    sortField.setMissingValue(SortField.STRING_FIRST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+      sortField.setMissingValue(SortField.STRING_FIRST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    SortedDocValues values = leaf.getSortedDocValues("foo");
-    // docID 0 is missing:
-    assertEquals(1, values.nextDoc());
-    assertEquals("mmm", values.binaryValue().utf8ToString());
-    assertEquals(2, values.nextDoc());
-    assertEquals("zzz", values.binaryValue().utf8ToString());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      SortedDocValues values = leaf.getSortedDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+        assertEquals(1, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+      } else {
+        // docID 0 is missing:
+        assertEquals(1, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+        assertEquals(2, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedStringFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedSetSortField("foo", false);
-    sortField.setMissingValue(SortField.STRING_FIRST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedSetSortField("foo", reverse);
+      sortField.setMissingValue(SortField.STRING_FIRST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1l, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2l, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3l, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1l, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3l, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingStringLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.STRING);
-    sortField.setMissingValue(SortField.STRING_LAST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+      sortField.setMissingValue(SortField.STRING_LAST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    SortedDocValues values = leaf.getSortedDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals("mmm", values.binaryValue().utf8ToString());
-    assertEquals(1, values.nextDoc());
-    assertEquals("zzz", values.binaryValue().utf8ToString());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      SortedDocValues values = leaf.getSortedDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+        assertEquals(2, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+        assertEquals(1, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedStringLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedSetSortField("foo", false);
-    sortField.setMissingValue(SortField.STRING_LAST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedSetSortField("foo", reverse);
+      sortField.setMissingValue(SortField.STRING_LAST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1l, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2l, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3l, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1l, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3l, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testBasicLong() throws Exception {
@@ -597,11 +637,213 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingLongFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        // docID 0 has no value
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedLongFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 27));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 24));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingLongLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        // docID 0 is missing
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedLongLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 65));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 34));
+      doc.add(new SortedNumericDocValuesField("foo", 74));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testBasicInt() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -610,8 +852,9 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
-    w.addDocument(new Document());
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", -1));
+    w.addDocument(doc);
     w.commit();
 
     doc = new Document();
@@ -623,7 +866,8 @@ public class TestIndexSorting extends LuceneTestCase {
     LeafReader leaf = getOnlyLeafReader(r);
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
-    // docID 0 has no value
+    assertEquals(0, values.nextDoc());
+    assertEquals(-1, values.longValue());
     assertEquals(1, values.nextDoc());
     assertEquals(7, values.longValue());
     assertEquals(2, values.nextDoc());
@@ -633,195 +877,32 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
-  public void testMissingMultiValuedLongFirst() throws Exception {
+  public void testBasicMultiValuedInt() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
     doc.add(new NumericDocValuesField("id", 3));
     doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 27));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", -1));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 2));
     doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 24));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingLongLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    w.addDocument(new Document());
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(18, values.longValue());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingMultiValuedLongLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 65));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    doc.add(new SortedNumericDocValuesField("foo", 74));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicInt() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", -1));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(-1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(18, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicMultiValuedInt() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", -1));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 22));
-    doc.add(new SortedNumericDocValuesField("foo", 27));
+    doc.add(new SortedNumericDocValuesField("foo", 22));
+    doc.add(new SortedNumericDocValuesField("foo", 27));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -841,332 +922,211 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingIntFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(18, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedIntFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 187667));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 187667));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 34));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingIntLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(18, values.longValue());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        // docID 0 is missing
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedIntLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 6372));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 8));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicDouble() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 18.0));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", -1.0));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 7.0));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicMultiValuedDouble() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingDoubleFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 18.0));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    w.addDocument(new Document());
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 7.0));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingMultiValuedDoubleFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 6372));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 8));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
-  public void testMissingDoubleLast() throws Exception {
+  public void testBasicDouble() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -1175,8 +1135,9 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
-    w.addDocument(new Document());
+    doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", -1.0));
+    w.addDocument(doc);
     w.commit();
 
     doc = new Document();
@@ -1189,41 +1150,41 @@ public class TestIndexSorting extends LuceneTestCase {
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
     assertEquals(0, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
     assertEquals(1, values.nextDoc());
+    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(2, values.nextDoc());
     assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
     r.close();
     w.close();
     dir.close();
   }
 
-  public void testMissingMultiValuedDoubleLast() throws Exception {
+  public void testBasicMultiValuedDouble() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new NumericDocValuesField("id", 2));
     doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -1242,6 +1203,207 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
+  public void testMissingDoubleFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 18.0));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 7.0));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(2, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedDoubleFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingDoubleLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 18.0));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 7.0));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(2, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedDoubleLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
   public void testBasicFloat() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1322,164 +1484,204 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingFloatFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 18.0f));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 18.0f));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 7.0f));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 7.0f));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(2, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedFloatFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingFloatLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 18.0f));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 18.0f));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 7.0f));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 7.0f));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(1, values.nextDoc());
-    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(2, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedFloatLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testRandom1() throws IOException {


[26/47] lucene-solr:jira/http2: SOLR-12801: harden CollectionsAPISolrJTest

Posted by da...@apache.org.
SOLR-12801: harden CollectionsAPISolrJTest


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/65bfb163
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/65bfb163
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/65bfb163

Branch: refs/heads/jira/http2
Commit: 65bfb1639ce1b9f19b052953e453cf36fbf07b58
Parents: 1184fbe
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 00:21:28 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/65bfb163/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index ef19728..0925eee 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -130,7 +130,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
           .build()
           .process(cluster.getSolrClient());
 
-      for (int i = 0; i < 10; i++) {
+      for (int i = 0; i < 30; i++) {
         Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
         if (m != null) break;
         Thread.sleep(10);


[17/47] lucene-solr:jira/http2: SOLR-12801: don't do single index check here on windows.

Posted by da...@apache.org.
SOLR-12801: don't do single index check here on windows.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b94911b2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b94911b2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b94911b2

Branch: refs/heads/jira/http2
Commit: b94911b260d8411cde755e2aa2c18694b3f9a825
Parents: e25d685
Author: markrmiller <ma...@apache.org>
Authored: Sat Dec 8 11:40:00 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../test/org/apache/solr/handler/TestReplicationHandler.java   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b94911b2/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index e3a9377..f1a9a86 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.lucene.util.TestUtil;
@@ -899,7 +900,10 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
         assertVersions(masterClient, slaveClient);
         
         checkForSingleIndex(masterJetty);
-        checkForSingleIndex(slaveJetty);
+        
+        if (!Constants.WINDOWS) {
+          checkForSingleIndex(slaveJetty);
+        }
         
         if (random().nextBoolean()) {
           // move the slave ahead


[24/47] lucene-solr:jira/http2: SOLR-12801: harden CollectionsAPIDistributedZkTest

Posted by da...@apache.org.
SOLR-12801: harden CollectionsAPIDistributedZkTest


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/906e32af
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/906e32af
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/906e32af

Branch: refs/heads/jira/http2
Commit: 906e32af1384fe5ece66beb57e8f8619afd1807b
Parents: 65bfb16
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 00:28:15 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../CollectionsAPIDistributedZkTest.java            | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/906e32af/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
index d019dd8..42a2e40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java
@@ -474,20 +474,20 @@ public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase {
         .add("id", "7")
         .add("id", "8")
         .commit(cluster.getSolrClient(), collectionName);
+    long numFound = 0;
     TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
     while (!timeOut.hasTimedOut()) {
-      try {
-        long numFound = cluster.getSolrClient().query(collectionName, new SolrQuery("*:*")).getResults().getNumFound();
-        assertEquals(3, numFound);
+
+      numFound = cluster.getSolrClient().query(collectionName, new SolrQuery("*:*")).getResults().getNumFound();
+      if (numFound == 3) {
         break;
-      } catch (Exception e) {
-        // Query node can have stale clusterstate
-        log.info("Error when query " + collectionName, e);
-        Thread.sleep(500);
       }
+
+      Thread.sleep(500);
     }
+    
     if (timeOut.hasTimedOut()) {
-      fail("Timeout on query " + collectionName);
+      fail("Timeout waiting to see 3 found, instead saw " + numFound + " for collection " + collectionName);
     }
 
     checkNoTwoShardsUseTheSameIndexDir();


[29/47] lucene-solr:jira/http2: SOLR-12801: Harden CollectionsAPISolrJTest#testCreateCollWithDefaultClusterPropertiesNewFormat.

Posted by da...@apache.org.
SOLR-12801: Harden CollectionsAPISolrJTest#testCreateCollWithDefaultClusterPropertiesNewFormat.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6f8d5987
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6f8d5987
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6f8d5987

Branch: refs/heads/jira/http2
Commit: 6f8d598781bed64d17d077040a4afd9f51a32b98
Parents: 665517d
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 20:52:03 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 20:52:03 2018 -0600

----------------------------------------------------------------------
 .../src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f8d5987/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 166748e..b685aa8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -213,7 +213,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
           .build()
           .process(cluster.getSolrClient());
 
-      for (int i = 0; i < 10; i++) {
+      for (int i = 0; i < 300; i++) {
         Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
         if (m != null) break;
         Thread.sleep(10);


[18/47] lucene-solr:jira/http2: SOLR-12923: Harden TestSimTriggerIntegration.testNodeMarkersRegistration.

Posted by da...@apache.org.
SOLR-12923: Harden TestSimTriggerIntegration.testNodeMarkersRegistration.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4133ce4c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4133ce4c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4133ce4c

Branch: refs/heads/jira/http2
Commit: 4133ce4cc167a80ed20f7763de4b4a8ab5d0316f
Parents: 3eb2612
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 15:10:12 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../autoscaling/sim/TestSimTriggerIntegration.java    | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4133ce4c/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 1b177f9..8f37ddb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -20,6 +20,7 @@ package org.apache.solr.cloud.autoscaling.sim;
 import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
 import static org.apache.solr.cloud.autoscaling.ScheduledTriggers.DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS;
 
+import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -66,6 +67,7 @@ import org.apache.solr.common.util.TimeSource;
 import org.apache.solr.core.SolrResourceLoader;
 import org.apache.solr.util.LogLevel;
 import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.KeeperException;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -848,7 +850,19 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
     cluster.getTimeSource().sleep(5000);
     // verify that a znode does NOT exist - there's no nodeLost trigger,
     // so the new overseer cleaned up existing nodeLost markers
+    
     String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader;
+    
+    TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+    timeout.waitFor("Path " + pathLost + " exists", () -> {
+      try {
+        return !cluster.getDistribStateManager().hasData(pathLost);
+      } catch (IOException | KeeperException | InterruptedException e) {
+        e.printStackTrace();
+        throw new RuntimeException(e);
+      }
+    });
+
     assertFalse("Path " + pathLost + " exists", cluster.getDistribStateManager().hasData(pathLost));
 
     listener.reset();


[37/47] lucene-solr:jira/http2: SOLR-12923: Harden TestSimTriggerIntegration.

Posted by da...@apache.org.
SOLR-12923: Harden TestSimTriggerIntegration.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a1657616
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a1657616
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a1657616

Branch: refs/heads/jira/http2
Commit: a1657616bb596702aee8c8bac582ef1f86f55d7d
Parents: 56b9591
Author: markrmiller <ma...@apache.org>
Authored: Mon Dec 10 08:46:54 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Dec 10 10:02:58 2018 -0600

----------------------------------------------------------------------
 .../solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a1657616/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index c4ec122..0227cda 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -210,7 +210,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
 
     String newNode = cluster.simAddNode();
 
-    if (!triggerFiredLatch.await(220000 / SPEED, TimeUnit.MILLISECONDS)) {
+    if (!triggerFiredLatch.await(320000 / SPEED, TimeUnit.MILLISECONDS)) {
       fail("Both triggers should have fired by now");
     }
 
@@ -438,7 +438,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
     }
 
     String newNode = cluster.simAddNode();
-    boolean await = triggerFiredLatch.await(60000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFiredLatch.await(90000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     assertTrue(triggerFired.get());
     TriggerEvent nodeAddedEvent = events.iterator().next();


[44/47] lucene-solr:jira/http2: SOLR-13045: Harden TestSimPolicyCloud

Posted by da...@apache.org.
SOLR-13045: Harden TestSimPolicyCloud

This commit fixes a race condition in SimClusterStateProvider, fixing
several fails in TestSimPolicyCloud.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f89f109e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f89f109e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f89f109e

Branch: refs/heads/jira/http2
Commit: f89f109ec1d1b9a19034d17345d41d3dd2c34852
Parents: af6e15e
Author: Jason Gerlowski <ge...@apache.org>
Authored: Fri Dec 7 11:07:42 2018 -0500
Committer: Jason Gerlowski <ge...@apache.org>
Committed: Mon Dec 10 17:39:12 2018 -0500

----------------------------------------------------------------------
 .../cloud/autoscaling/sim/SimCloudManager.java  |   2 +-
 .../sim/SimClusterStateProvider.java            | 139 ++++++++++---------
 .../autoscaling/sim/TestSimExtremeIndexing.java |   2 -
 .../autoscaling/sim/TestSimPolicyCloud.java     |   6 +-
 4 files changed, 75 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f89f109e/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index dad9463..fd622c1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -838,7 +838,7 @@ public class SimCloudManager implements SolrCloudManager {
           results.add("success", "");
           break;
         case ADDROLE:
-          nodeStateProvider.simAddNodeValue(req.getParams().get("node"), "nodeRole", req.getParams().get("role"));
+          nodeStateProvider.simSetNodeValue(req.getParams().get("node"), "nodeRole", req.getParams().get("role"));
           break;
         case CREATESHARD:
           try {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f89f109e/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index d81d92c..6197c41 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -691,79 +691,84 @@ public class SimClusterStateProvider implements ClusterStateProvider {
 
   private void simRunLeaderElection(String collection, Slice s, boolean saveState) throws Exception {
     AtomicBoolean stateChanged = new AtomicBoolean(Boolean.FALSE);
-    Replica leader = s.getLeader();
-    if (leader == null || !liveNodes.contains(leader.getNodeName())) {
-      log.trace("Running leader election for {} / {}", collection, s.getName());
-      if (s.getReplicas().isEmpty()) { // no replicas - punt
-        log.trace("-- no replicas in {} / {}", collection, s.getName());
-        return;
-      }
-      ActionThrottle lt = getThrottle(collection, s.getName());
-      synchronized (lt) {
-        // collect all active and live
-        List<ReplicaInfo> active = new ArrayList<>();
-        AtomicBoolean alreadyHasLeader = new AtomicBoolean(false);
-        s.getReplicas().forEach(r -> {
-          // find our ReplicaInfo for this replica
-          ReplicaInfo ri = getReplicaInfo(r);
-          if (ri == null) {
-            throw new IllegalStateException("-- could not find ReplicaInfo for replica " + r);
-          }
-          synchronized (ri) {
-            if (r.isActive(liveNodes.get())) {
-              if (ri.getVariables().get(ZkStateReader.LEADER_PROP) != null) {
-                log.trace("-- found existing leader {} / {}: {}, {}", collection, s.getName(), ri, r);
-                alreadyHasLeader.set(true);
-                return;
-              } else {
-                active.add(ri);
-              }
-            } else { // if it's on a node that is not live mark it down
-              log.trace("-- replica not active on live nodes: {}, {}", liveNodes.get(), r);
-              if (!liveNodes.contains(r.getNodeName())) {
-                ri.getVariables().put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
-                ri.getVariables().remove(ZkStateReader.LEADER_PROP);
-                stateChanged.set(true);
+    lock.lockInterruptibly();
+    try {
+      Replica leader = s.getLeader();
+      if (leader == null || !liveNodes.contains(leader.getNodeName())) {
+        log.trace("Running leader election for {} / {}", collection, s.getName());
+        if (s.getReplicas().isEmpty()) { // no replicas - punt
+          log.trace("-- no replicas in {} / {}", collection, s.getName());
+          return;
+        }
+        ActionThrottle lt = getThrottle(collection, s.getName());
+        synchronized (lt) {
+          // collect all active and live
+          List<ReplicaInfo> active = new ArrayList<>();
+          AtomicBoolean alreadyHasLeader = new AtomicBoolean(false);
+          s.getReplicas().forEach(r -> {
+            // find our ReplicaInfo for this replica
+            ReplicaInfo ri = getReplicaInfo(r);
+            if (ri == null) {
+              throw new IllegalStateException("-- could not find ReplicaInfo for replica " + r);
+            }
+            synchronized (ri) {
+              if (r.isActive(liveNodes.get())) {
+                if (ri.getVariables().get(ZkStateReader.LEADER_PROP) != null) {
+                  log.trace("-- found existing leader {} / {}: {}, {}", collection, s.getName(), ri, r);
+                  alreadyHasLeader.set(true);
+                  return;
+                } else {
+                  active.add(ri);
+                }
+              } else { // if it's on a node that is not live mark it down
+                log.trace("-- replica not active on live nodes: {}, {}", liveNodes.get(), r);
+                if (!liveNodes.contains(r.getNodeName())) {
+                  ri.getVariables().put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
+                  ri.getVariables().remove(ZkStateReader.LEADER_PROP);
+                  stateChanged.set(true);
+                }
               }
             }
+          });
+          if (alreadyHasLeader.get()) {
+            log.trace("-- already has leader {} / {}: {}", collection, s.getName(), s);
+            return;
           }
-        });
-        if (alreadyHasLeader.get()) {
-          log.trace("-- already has leader {} / {}: {}", collection, s.getName(), s);
-          return;
-        }
-        if (active.isEmpty()) {
-          log.warn("Can't find any active replicas for {} / {}: {}", collection, s.getName(), s);
-          log.debug("-- liveNodes: {}", liveNodes.get());
-          return;
-        }
-        // pick first active one
-        ReplicaInfo ri = null;
-        for (ReplicaInfo a : active) {
-          if (!a.getType().equals(Replica.Type.PULL)) {
-            ri = a;
-            break;
+          if (active.isEmpty()) {
+            log.warn("Can't find any active replicas for {} / {}: {}", collection, s.getName(), s);
+            log.debug("-- liveNodes: {}", liveNodes.get());
+            return;
           }
+          // pick first active one
+          ReplicaInfo ri = null;
+          for (ReplicaInfo a : active) {
+            if (!a.getType().equals(Replica.Type.PULL)) {
+              ri = a;
+              break;
+            }
+          }
+          if (ri == null) {
+            log.warn("-- can't find any suitable replica type for {} / {}: {}", collection, s.getName(), s);
+            return;
+          }
+          // now mark the leader election throttle
+          lt.minimumWaitBetweenActions();
+          lt.markAttemptingAction();
+          synchronized (ri) {
+            ri.getVariables().put(ZkStateReader.LEADER_PROP, "true");
+          }
+          log.debug("-- elected new leader for {} / {} (currentVersion={}): {}", collection,
+              s.getName(), clusterStateVersion, ri);
+          stateChanged.set(true);
         }
-        if (ri == null) {
-          log.warn("-- can't find any suitable replica type for {} / {}: {}", collection, s.getName(), s);
-          return;
-        }
-        // now mark the leader election throttle
-        lt.minimumWaitBetweenActions();
-        lt.markAttemptingAction();
-        synchronized (ri) {
-          ri.getVariables().put(ZkStateReader.LEADER_PROP, "true");
-        }
-        log.debug("-- elected new leader for {} / {} (currentVersion={}): {}", collection,
-            s.getName(), clusterStateVersion, ri);
-        stateChanged.set(true);
+      } else {
+        log.trace("-- already has leader for {} / {}", collection, s.getName());
       }
-    } else {
-      log.trace("-- already has leader for {} / {}", collection, s.getName());
-    }
-    if (stateChanged.get() || saveState) {
-      collectionsStatesRef.set(null);
+    } finally {
+      if (stateChanged.get() || saveState) {
+        collectionsStatesRef.set(null);
+      }
+      lock.unlock();
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f89f109e/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java
index 6c5f9e6..15d676b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExtremeIndexing.java
@@ -23,7 +23,6 @@ import java.util.concurrent.TimeUnit;
 
 import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
 
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -52,7 +51,6 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
 @TimeoutSuite(millis = 48 * 3600 * 1000)
 @LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.ComputePlanAction=INFO;org.apache.solr.cloud.autoscaling.ExecutePlanAction=DEBUG;org.apache.solr.cloud.autoscaling.ScheduledTriggers=DEBUG")
 //@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.CloudTestUtils=TRACE")
-@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // this test can fail to elect a leader, seems to be common among sim tests
 public class TestSimExtremeIndexing extends SimSolrCloudTestCase {
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f89f109e/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
index e70cefb..eb251010 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimPolicyCloud.java
@@ -108,7 +108,6 @@ public class TestSimPolicyCloud extends SimSolrCloudTestCase {
 
   }
 
-  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
   public void testCreateCollectionAddReplica() throws Exception  {
     SolrClient solrClient = cluster.simGetSolrClient();
     String nodeId = cluster.getSimClusterStateProvider().simGetRandomNode();
@@ -134,8 +133,7 @@ public class TestSimPolicyCloud extends SimSolrCloudTestCase {
 
     getCollectionState(collectionName).forEachReplica((s, replica) -> assertEquals(nodeId, replica.getNodeName()));
   }
-  
-  @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+
   public void testCreateCollectionSplitShard() throws Exception  {
     SolrClient solrClient = cluster.simGetSolrClient();
     String firstNode = cluster.getSimClusterStateProvider().simGetRandomNode();
@@ -294,7 +292,7 @@ public class TestSimPolicyCloud extends SimSolrCloudTestCase {
     assertEquals(3, coll.getSlice("s3").getReplicas().size());
     coll.forEachReplica(verifyReplicas);
   }
-  @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
+  
   public void testCreateCollectionAddShardUsingPolicy() throws Exception {
     SolrClient solrClient = cluster.simGetSolrClient();
     String nodeId = cluster.getSimClusterStateProvider().simGetRandomNode();


[30/47] lucene-solr:jira/http2: SOLR-12801: Try to harden SolrRrdBackendFactoryTest.

Posted by da...@apache.org.
SOLR-12801: Try to harden SolrRrdBackendFactoryTest.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f47976a2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f47976a2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f47976a2

Branch: refs/heads/jira/http2
Commit: f47976a219a823460cad655f151eeb3a2cd4606d
Parents: 6f8d598
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 21:59:01 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 22:55:19 2018 -0600

----------------------------------------------------------------------
 .../solr/metrics/rrd/SolrRrdBackendFactoryTest.java   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f47976a2/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
index fb1f55d..936c41a 100644
--- a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
@@ -87,13 +87,13 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     List<Pair<String, Long>> list = factory.list(100);
     assertEquals(list.toString(), 1, list.size());
     assertEquals(list.toString(), "foo", list.get(0).first());
-    timeSource.sleep(2000);
+    timeSource.sleep(4000);
     // there should be one sync data
     assertEquals(solrClient.docs.toString(), 1, solrClient.docs.size());
     String id = SolrRrdBackendFactory.ID_PREFIX + SolrRrdBackendFactory.ID_SEP + "foo";
     SolrInputDocument doc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     long timestamp = (Long)doc.getFieldValue("timestamp_l");
-    timeSource.sleep(2000);
+    timeSource.sleep(4000);
     SolrInputDocument newDoc = solrClient.docs.get(CollectionAdminParams.SYSTEM_COLL).get(id);
     assertEquals(newDoc.toString(), newDoc, doc);
     // make sure the update doesn't race with the sampling boundaries
@@ -157,15 +157,15 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
     rowCount = fd.getRowCount();
     one = fd.getValues("one");
     assertEquals("one: " + dump, 102, one.length);
-    assertEquals(dump, Double.NaN, one[0], 0.00001);
-    assertEquals(dump, Double.NaN, one[101], 0.00001);
+    assertEquals(dump, Double.NaN, one[0], 0.01);
+    assertEquals(dump, Double.NaN, one[101], 0.01);
     for (int i = 1; i < 101; i++) {
-      assertEquals(dump, 1.0, one[i], 0.00001);
+      assertEquals(dump, 1.0, one[i], 0.01);
     }
     two = fd.getValues("two");
-    assertEquals("two: " + dump, Double.NaN, two[101], 0.00001);
+    assertEquals("two: " + dump, Double.NaN, two[101], 0.001);
     for (int i = 1; i < 101; i++) {
-      assertEquals(dump, 100.0, two[i], 0.00001);
+      assertEquals(dump, 100.0, two[i], 0.001);
     }
 
     db.close();


[42/47] lucene-solr:jira/http2: LUCENE-8599: Use sparse bitset to store docs in SingleValueDocValuesFieldUpdates

Posted by da...@apache.org.
LUCENE-8599: Use sparse bitset to store docs in SingleValueDocValuesFieldUpdates

Using a sparse bitset in SingleValueDocValuesFieldUdpates allows storing
which documents have an update much more efficient and prevents the need
to sort the docs array altogether that showed to be a significant bottleneck
in LUCENE-8598. Using the spares bitset yields another 10x performance improvement
in applying updates versus the changes proposed in LUCENE-8598.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ef61b547
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ef61b547
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ef61b547

Branch: refs/heads/jira/http2
Commit: ef61b547b175f9fbb63dd9fca78f7984f4f72be2
Parents: fdd9b55
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Dec 10 15:31:43 2018 +0100
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Dec 10 19:20:28 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  3 +
 .../lucene/index/DocValuesFieldUpdates.java     | 73 +++++++++++++++++---
 .../lucene/index/TestDocValuesFieldUpdates.java | 59 +++++++++++++++-
 3 files changed, 124 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef61b547/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4eaca5f..22e3e61 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -271,6 +271,9 @@ Optimizations
 * LUCENE-8598: Moved to the default accepted overhead ratio for packet ints in DocValuesFieldUpdats
   yields an up-to 4x performance improvement when applying doc values updates. (Simon Willnauer, Adrien Grand)
 
+* LUCENE-8599: Use sparse bitset to store docs in SingleValueDocValuesFieldUpdates. 
+  (Simon Willnauer, Adrien Grand)
+
 Other
 
 * LUCENE-8573: BKDWriter now uses FutureArrays#mismatch to compute shared prefixes.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef61b547/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 9ab3b7c..c514ad0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -18,10 +18,13 @@ package org.apache.lucene.index;
 
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BitSet;
+import org.apache.lucene.util.BitSetIterator;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.InPlaceMergeSorter;
 import org.apache.lucene.util.PriorityQueue;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.SparseFixedBitSet;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PagedMutable;
 
@@ -282,7 +285,6 @@ abstract class DocValuesFieldUpdates implements Accountable {
       throw new IllegalStateException("already finished");
     }
     finished = true;
-
     // shrink wrap
     if (size < docs.size()) {
       resize(size);
@@ -302,9 +304,9 @@ abstract class DocValuesFieldUpdates implements Accountable {
       }
     }.sort(0, size);
   }
-  
+
   /** Returns true if this instance contains any updates. */
-  synchronized final boolean any() {
+  synchronized boolean any() {
     return size > 0;
   }
 
@@ -316,7 +318,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
    * Adds an update that resets the documents value.
    * @param doc the doc to update
    */
-  final synchronized void reset(int doc) {
+  synchronized void reset(int doc) {
     addInternal(doc, HAS_NO_VALUE_MASK);
   }
   final synchronized int add(int doc) {
@@ -431,21 +433,42 @@ abstract class DocValuesFieldUpdates implements Accountable {
   }
 
   static abstract class SingleValueDocValuesFieldUpdates extends DocValuesFieldUpdates {
-
+    private final BitSet bitSet;
+    private BitSet hasNoValue;
+    private boolean hasAtLeastOneValue;
     protected SingleValueDocValuesFieldUpdates(int maxDoc, long delGen, String field, DocValuesType type) {
       super(maxDoc, delGen, field, type);
+      this.bitSet = new SparseFixedBitSet(maxDoc);
     }
 
     @Override
     void add(int doc, long value) {
       assert longValue() == value;
-      super.add(doc);
+      bitSet.set(doc);
+      this.hasAtLeastOneValue = true;
+      if (hasNoValue != null) {
+        hasNoValue.clear(doc);
+      }
     }
 
     @Override
     void add(int doc, BytesRef value) {
       assert binaryValue().equals(value);
-      super.add(doc);
+      bitSet.set(doc);
+      this.hasAtLeastOneValue = true;
+      if (hasNoValue != null) {
+        hasNoValue.clear(doc);
+      }
+    }
+
+    @Override
+    synchronized void reset(int doc) {
+      bitSet.set(doc);
+      this.hasAtLeastOneValue = true;
+      if (hasNoValue == null) {
+        hasNoValue = new SparseFixedBitSet(maxDoc);
+      }
+      hasNoValue.set(doc);
     }
 
     @Override
@@ -458,11 +481,28 @@ abstract class DocValuesFieldUpdates implements Accountable {
     protected abstract long longValue();
 
     @Override
+    synchronized boolean any() {
+      return super.any() || hasAtLeastOneValue;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+      return super.ramBytesUsed() + bitSet.ramBytesUsed();
+    }
+
+    @Override
     Iterator iterator() {
-      return new DocValuesFieldUpdates.AbstractIterator(size, docs, delGen) {
+      BitSetIterator iterator = new BitSetIterator(bitSet, maxDoc);
+      return new DocValuesFieldUpdates.Iterator() {
+
+        @Override
+        public int docID() {
+          return iterator.docID();
+        }
+
         @Override
-        protected void set(long idx) {
-          // nothing to do;
+        public int nextDoc() {
+          return iterator.nextDoc();
         }
 
         @Override
@@ -474,6 +514,19 @@ abstract class DocValuesFieldUpdates implements Accountable {
         BytesRef binaryValue() {
           return SingleValueDocValuesFieldUpdates.this.binaryValue();
         }
+
+        @Override
+        long delGen() {
+          return delGen;
+        }
+
+        @Override
+        boolean hasValue() {
+          if (hasNoValue != null) {
+            return hasNoValue.get(docID()) == false;
+          }
+          return true;
+        }
       };
     }
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ef61b547/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
index 6c60df3..a3a6e5a 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDocValuesFieldUpdates.java
@@ -21,6 +21,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
+import org.apache.lucene.index.NumericDocValuesFieldUpdates.SingleValueNumericDocValuesFieldUpdates;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.util.LuceneTestCase;
 
@@ -129,7 +130,63 @@ public class TestDocValuesFieldUpdates extends LuceneTestCase {
       }
       idx++;
     }
+  }
 
-
+  public void testSharedValueUpdates() {
+    int delGen = random().nextInt();
+    int maxDoc = 1 + random().nextInt(1000);
+    long value = random().nextLong();
+    SingleValueNumericDocValuesFieldUpdates update = new SingleValueNumericDocValuesFieldUpdates(delGen, "foo", maxDoc, value);
+    assertEquals(value, update.longValue());
+    Boolean[] values = new Boolean[maxDoc];
+    boolean any = false;
+    boolean noReset = random().nextBoolean(); // sometimes don't reset
+    for (int i = 0; i < maxDoc; i++) {
+      if (random().nextBoolean()) {
+        values[i] = Boolean.TRUE;
+        any = true;
+        update.add(i, value);
+      } else if (random().nextBoolean() && noReset == false) {
+        values[i] = null;
+        any = true;
+        update.reset(i);
+      } else {
+        values[i] = Boolean.FALSE;
+      }
+    }
+    if (noReset == false) {
+      for (int i = 0; i < values.length; i++) {
+        if (rarely()) {
+          if (values[i] == null) {
+            values[i] = Boolean.TRUE;
+            update.add(i, value);
+          } else if (values[i]) {
+            values[i] = null;
+            update.reset(i);
+          }
+        }
+      }
+    }
+    update.finish();
+    DocValuesFieldUpdates.Iterator iterator = update.iterator();
+    assertEquals(any, update.any());
+    assertEquals(delGen, iterator.delGen());
+    int index = 0;
+    while (iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+      int doc = iterator.docID();
+      if (index < iterator.docID()) {
+        for (;index < doc; index++) {
+          assertFalse(values[index]);
+        }
+      }
+      if (index == doc) {
+        if (values[index++] == null) {
+          assertFalse(iterator.hasValue());
+        } else {
+          assertTrue(iterator.hasValue());
+          assertEquals(value, iterator.longValue());
+        }
+      }
+    }
   }
 }


[39/47] lucene-solr:jira/http2: SOLR-13014: URI Too Long with large streaming expressions in SolrJ

Posted by da...@apache.org.
SOLR-13014: URI Too Long with large streaming expressions in SolrJ


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b6cbb849
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b6cbb849
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b6cbb849

Branch: refs/heads/jira/http2
Commit: b6cbb8494e0bc32ed57a94e45aecba9723133786
Parents: 0650f99
Author: Jan Høydahl <ja...@apache.org>
Authored: Mon Dec 10 16:05:29 2018 +0100
Committer: Jan Høydahl <ja...@apache.org>
Committed: Mon Dec 10 18:22:52 2018 +0100

----------------------------------------------------------------------
 solr/CHANGES.txt                                |  2 +
 .../client/solrj/io/stream/FacetStream.java     |  3 +-
 .../client/solrj/io/stream/RandomStream.java    |  3 +-
 .../client/solrj/io/stream/SearchStream.java    |  3 +-
 .../solrj/io/stream/SignificantTermsStream.java |  3 +-
 .../client/solrj/io/stream/StatsStream.java     |  5 +-
 .../solrj/io/stream/TimeSeriesStream.java       |  3 +-
 .../solrj/io/stream/StreamExpressionTest.java   | 66 ++++++++++++++++++++
 8 files changed, 81 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 68e17a0..ff00394 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -147,6 +147,8 @@ Bug Fixes
   
 * SOLR-12933: Fix SolrCloud distributed commit. (Mark Miller)
 
+* SOLR-13014: URI Too Long with large streaming expressions in SolrJ (janhoy) 
+
 Improvements
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
index ee4cb5d..b84967c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/FacetStream.java
@@ -28,6 +28,7 @@ import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -496,7 +497,7 @@ public class FacetStream extends TupleStream implements Expressible  {
     paramsLoc.set("json.facet", json);
     paramsLoc.set("rows", "0");
 
-    QueryRequest request = new QueryRequest(paramsLoc);
+    QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST);
     try {
       NamedList response = cloudSolrClient.request(request, collection);
       getTuples(response, buckets, metrics);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java
index 052fc30..6f04085 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/RandomStream.java
@@ -29,6 +29,7 @@ import java.util.Optional;
 import java.util.Random;
 import java.util.stream.Collectors;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
@@ -191,7 +192,7 @@ public class RandomStream extends TupleStream implements Expressible  {
     String sortField = "random_"+seed;
     params.add(SORT, sortField+" asc");
 
-    QueryRequest request = new QueryRequest(params);
+    QueryRequest request = new QueryRequest(params, SolrRequest.METHOD.POST);
     try {
       QueryResponse response = request.process(cloudSolrClient, collection);
       SolrDocumentList docs = response.getResults();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java
index 2eeeb78..a4ed996 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SearchStream.java
@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Optional;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
@@ -185,7 +186,7 @@ public class SearchStream extends TupleStream implements Expressible  {
     }
 
 
-    QueryRequest request = new QueryRequest(params);
+    QueryRequest request = new QueryRequest(params, SolrRequest.METHOD.POST);
     try {
       QueryResponse response = request.process(cloudSolrClient, collection);
       SolrDocumentList docs = response.getResults();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java
index 729ddb1..fbe893d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/SignificantTermsStream.java
@@ -30,6 +30,7 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.io.SolrClientCache;
 import org.apache.solr.client.solrj.io.Tuple;
@@ -398,7 +399,7 @@ public class SignificantTermsStream extends TupleStream implements Expressible{
       params.add("field", field);
       params.add("numTerms", String.valueOf(numTerms*5));
 
-      QueryRequest request= new QueryRequest(params);
+      QueryRequest request= new QueryRequest(params, SolrRequest.METHOD.POST);
       QueryResponse response = request.process(solrClient);
       NamedList res = response.getResponse();
       return res;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java
index d4df1e3..6c6df59 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/StatsStream.java
@@ -25,6 +25,7 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.stream.Collectors;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -208,7 +209,7 @@ public class StatsStream extends TupleStream implements Expressible  {
 
     Map<String, List<String>> shardsMap = (Map<String, List<String>>)streamContext.get("shards");
     if(shardsMap == null) {
-      QueryRequest request = new QueryRequest(paramsLoc);
+      QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST);
       CloudSolrClient cloudSolrClient = cache.getCloudSolrClient(zkHost);
       try {
         NamedList response = cloudSolrClient.request(request, collection);
@@ -226,7 +227,7 @@ public class StatsStream extends TupleStream implements Expressible  {
         paramsLoc.add("distrib", "true");
       }
 
-      QueryRequest request = new QueryRequest(paramsLoc);
+      QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST);
       try {
         NamedList response = client.request(request);
         this.tuple = getTuple(response);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
index dcd938e..13e72fa 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
@@ -30,6 +30,7 @@ import java.util.Optional;
 import java.util.Map.Entry;
 import java.util.stream.Collectors;
 
+import org.apache.solr.client.solrj.SolrRequest;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.CloudSolrClient.Builder;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -288,7 +289,7 @@ public class TimeSeriesStream extends TupleStream implements Expressible  {
     paramsLoc.set("json.facet", json);
     paramsLoc.set("rows", "0");
 
-    QueryRequest request = new QueryRequest(paramsLoc);
+    QueryRequest request = new QueryRequest(paramsLoc, SolrRequest.METHOD.POST);
     try {
       NamedList response = cloudSolrClient.request(request, collection);
       getTuples(response, field, metrics);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6cbb849/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 2de9c80..2725903 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -23,10 +23,13 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.io.ClassificationEvaluation;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -2669,6 +2672,69 @@ public class StreamExpressionTest extends SolrCloudTestCase {
     }
   }
 
+  @Test
+  public void tooLargeForGetRequest() throws IOException, SolrServerException {
+    // Test expressions which are larger than GET can handle
+    UpdateRequest updateRequest = new UpdateRequest();
+    for (int i = 0; i < 10; i++) {
+      updateRequest.add(id, "a"+i, "test_t", "a b c d m l");
+    }
+    for(int i=1; i<=50; i++) {
+      updateRequest.add(id, "id_"+(i),"test_dt", getDateString("2016", "5", "1"), "price_f", "400.00");
+    }
+    updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+    SolrClientCache cache = new SolrClientCache();
+    StreamContext streamContext = new StreamContext();
+    streamContext.setSolrClientCache(cache);
+    String longQuery = "\"id:(" + IntStream.range(0, 4000).mapToObj(i -> "a").collect(Collectors.joining(" ", "", "")) + ")\"";
+        
+    try {
+      assertSuccess("significantTerms("+COLLECTIONORALIAS+", q="+longQuery+", field=\"test_t\", limit=3, minTermLength=1, maxDocFreq=\".5\")", streamContext);
+      String expr = "timeseries("+COLLECTIONORALIAS+", q="+longQuery+", start=\"2013-01-01T01:00:00.000Z\", " +
+              "end=\"2016-12-01T01:00:00.000Z\", " +
+              "gap=\"+1YEAR\", " +
+              "field=\"test_dt\", " +
+              "format=\"yyyy\", " +
+              "count(*), sum(price_f), max(price_f), min(price_f))";
+      assertSuccess(expr, streamContext);
+      expr = "facet("
+                    +   "collection1, "
+                    +   "q="+longQuery+", "
+                    +   "fl=\"a_s,a_i,a_f\", "
+                    +   "sort=\"a_s asc\", "
+                    +   "buckets=\"a_s\", "
+                    +   "bucketSorts=\"sum(a_i) asc\", "
+                    +   "bucketSizeLimit=100, "
+                    +   "sum(a_i), sum(a_f), "
+                    +   "min(a_i), min(a_f), "
+                    +   "max(a_i), max(a_f), "
+                    +   "avg(a_i), avg(a_f), "
+                    +   "count(*)"
+                    + ")";
+      assertSuccess(expr, streamContext);
+      expr = "stats(" + COLLECTIONORALIAS + ", q="+longQuery+", sum(a_i), sum(a_f), min(a_i), min(a_f), max(a_i), max(a_f), avg(a_i), avg(a_f), count(*))"; 
+      assertSuccess(expr, streamContext);
+      expr = "search(" + COLLECTIONORALIAS + ", q="+longQuery+", fl=\"id,a_s,a_i,a_f\", sort=\"a_f asc, a_i asc\")";
+      assertSuccess(expr, streamContext);
+      expr = "random(" + COLLECTIONORALIAS + ", q="+longQuery+", rows=\"1000\", fl=\"id, a_i\")";
+      assertSuccess(expr, streamContext);
+    } finally {
+      cache.close();
+    }
+  }
+
+  private void assertSuccess(String expr, StreamContext streamContext) throws IOException {
+    ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+    paramsLoc.set("expr", expr);
+    paramsLoc.set("qt", "/stream");
+
+    String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+    TupleStream solrStream = new SolrStream(url, paramsLoc);
+    solrStream.setStreamContext(streamContext);
+    getTuples(solrStream);
+  }
+
   protected List<Tuple> getTuples(TupleStream tupleStream) throws IOException {
     List<Tuple> tuples = new ArrayList<Tuple>();
 


[46/47] lucene-solr:jira/http2: SOLR-12885: BinaryResponseWriter (javabin format) should directly copy from BytesRef to output

Posted by da...@apache.org.
SOLR-12885: BinaryResponseWriter (javabin format) should directly copy from BytesRef to output


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c651cbfd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c651cbfd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c651cbfd

Branch: refs/heads/jira/http2
Commit: c651cbfd9a806e0ea0fd7f0ca1f17e36306cd6c7
Parents: a2199c7
Author: Noble Paul <no...@apache.org>
Authored: Tue Dec 11 18:35:57 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Tue Dec 11 18:35:57 2018 +1100

----------------------------------------------------------------------
 .../java/org/apache/lucene/document/Field.java  |  14 ++-
 .../org/apache/lucene/document/StoredField.java |  13 ++
 .../org/apache/lucene/index/IndexableField.java |   7 ++
 solr/CHANGES.txt                                |   2 +
 .../solr/response/BinaryResponseWriter.java     | 121 ++++++++++++++++++-
 .../org/apache/solr/response/DocsStreamer.java  |   6 +-
 .../org/apache/solr/response/ResultContext.java |   2 +
 .../java/org/apache/solr/schema/FieldType.java  |   7 ++
 .../org/apache/solr/schema/IndexSchema.java     |  10 +-
 .../java/org/apache/solr/schema/StrField.java   |   6 +
 .../java/org/apache/solr/schema/TextField.java  |   5 +
 .../apache/solr/search/SolrDocumentFetcher.java |  24 +++-
 .../solr/response/TestBinaryResponseWriter.java |  45 ++++++-
 .../solr/common/ConditionalMapWriter.java       |   8 +-
 .../apache/solr/common/LinkedHashMapWriter.java |   2 +-
 .../java/org/apache/solr/common/MapWriter.java  |   5 +
 .../org/apache/solr/common/MapWriterMap.java    |   2 +-
 .../org/apache/solr/common/SolrDocument.java    |  18 ++-
 .../apache/solr/common/SolrDocumentBase.java    |   2 +-
 .../apache/solr/common/SolrInputDocument.java   |   8 +-
 .../common/util/ByteArrayUtf8CharSequence.java  |  19 ++-
 .../solr/common/util/FastOutputStream.java      |   2 +-
 .../apache/solr/common/util/JavaBinCodec.java   |  25 ++--
 .../org/apache/solr/common/util/NamedList.java  |   2 +-
 .../solr/common/util/Utf8CharSequence.java      |   2 +-
 .../solrj/embedded/SolrExampleJettyTest.java    |  65 ++++++++++
 26 files changed, 382 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/lucene/core/src/java/org/apache/lucene/document/Field.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/Field.java b/lucene/core/src/java/org/apache/lucene/document/Field.java
index 467fec7..2f0f13b 100644
--- a/lucene/core/src/java/org/apache/lucene/document/Field.java
+++ b/lucene/core/src/java/org/apache/lucene/document/Field.java
@@ -230,7 +230,7 @@ public class Field implements IndexableField {
    *         is null, or if the field's type is neither indexed() nor stored(), 
    *         or if indexed() is false but storeTermVectors() is true.
    */
-  public Field(String name, String value, IndexableFieldType type) {
+  public Field(String name, CharSequence value, IndexableFieldType type) {
     if (name == null) {
       throw new IllegalArgumentException("name must not be null");
     }
@@ -256,13 +256,19 @@ public class Field implements IndexableField {
    */
   @Override
   public String stringValue() {
-    if (fieldsData instanceof String || fieldsData instanceof Number) {
+    if (fieldsData instanceof CharSequence || fieldsData instanceof Number) {
       return fieldsData.toString();
     } else {
       return null;
     }
   }
-  
+
+  @Override
+  public CharSequence getCharSequenceValue() {
+    return fieldsData instanceof CharSequence ?
+        (CharSequence) fieldsData : stringValue();
+  }
+
   /**
    * The value of the field as a Reader, or null. If null, the String value or
    * binary value is used. Exactly one of stringValue(), readerValue(), and
@@ -446,7 +452,7 @@ public class Field implements IndexableField {
       return null;
     }
   }
-  
+
   /** Prints a Field for human consumption. */
   @Override
   public String toString() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/lucene/core/src/java/org/apache/lucene/document/StoredField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/StoredField.java b/lucene/core/src/java/org/apache/lucene/document/StoredField.java
index 7dc5a99..19fc231 100644
--- a/lucene/core/src/java/org/apache/lucene/document/StoredField.java
+++ b/lucene/core/src/java/org/apache/lucene/document/StoredField.java
@@ -127,6 +127,19 @@ public class StoredField extends Field {
     super(name, value, type);
   }
 
+  /**
+   * Expert: allows you to customize the {@link
+   * FieldType}.
+   * @param name field name
+   * @param value CharSequence value
+   * @param type custom {@link FieldType} for this field
+   * @throws IllegalArgumentException if the field name, value or type
+   *         is null.
+   */
+  public StoredField(String name, CharSequence value, FieldType type) {
+    super(name, value, type);
+  }
+
   // TODO: not great but maybe not a big problem?
   /**
    * Create a stored-only field with the given integer value.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
index f08eab5..4039f1f 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
@@ -61,6 +61,13 @@ public interface IndexableField {
   /** Non-null if this field has a string value */
   public String stringValue();
 
+  /**
+   * Non-null if this field has a string value
+   */
+  default CharSequence getCharSequenceValue() {
+    return stringValue();
+  }
+
   /** Non-null if this field has a Reader value */
   public Reader readerValue();
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ff00394..62370de 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -167,6 +167,8 @@ Improvements
 
 * SOLR-12833: Add configurable timeout to VersionBucket lock. (Jeffery Yuan, Mark Miller)
 
+* SOLR-12885: BinaryResponseWriter (javabin format) should directly copy from BytesRef to output (noble)
+
 Other Changes
 ----------------------
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
index 81172f3..be6317b 100644
--- a/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/BinaryResponseWriter.java
@@ -22,16 +22,20 @@ import java.io.OutputStream;
 import java.io.Writer;
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.function.Consumer;
 
 import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexableField;
 import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utf8CharSequence;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
@@ -40,8 +44,11 @@ import org.apache.solr.search.ReturnFields;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;
+
 
 public class BinaryResponseWriter implements BinaryQueryResponseWriter {
+//  public static boolean useUtf8CharSeq = true;
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   @Override
@@ -80,14 +87,32 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
 
     @Override
     public Object resolve(Object o, JavaBinCodec codec) throws IOException {
+      if (o instanceof StoredField) {
+        CharSequence val = ((StoredField) o).getCharSequenceValue();
+        if (val instanceof Utf8CharSequence) {
+          codec.writeUTF8Str((Utf8CharSequence) val);
+          return null;
+        }
+      }
       if (o instanceof ResultContext) {
         ReturnFields orig = returnFields;
         ResultContext res = (ResultContext)o;
         if(res.getReturnFields()!=null) {
           returnFields = res.getReturnFields();
         }
-        writeResults(res, codec);
+//        if (useUtf8CharSeq) {
+        ResultContext.READASBYTES.set(fieldName -> {
+          SchemaField fld = res.getRequest().getSchema().getFieldOrNull(fieldName);
+          return fld != null && fld.getType().isUtf8Field();
+        });
+
+        try {
+          writeResults(res, codec);
+        } finally {
+          ResultContext.READASBYTES.remove();
+        }
         returnFields = orig;
+
         return null; // null means we completely handled it
       }
       if (o instanceof DocList) {
@@ -176,4 +201,98 @@ public class BinaryResponseWriter implements BinaryQueryResponseWriter {
     }
   }
 
+  static class MaskCharSeqSolrDocument extends SolrDocument {
+    /**
+     * Get the value or collection of values for a given field.
+     */
+    @Override
+    public Object getFieldValue(String name) {
+      return convertCharSeq(_fields.get(name));
+    }
+
+    /**
+     * Get a collection of values for a given field name
+     */
+    @SuppressWarnings("unchecked")
+    @Override
+    public Collection<Object> getFieldValues(String name) {
+      Object v = _fields.get(name);
+      if (v instanceof Collection) {
+        return convertCharSeq((Collection<Object>) v);
+      }
+      if (v != null) {
+        ArrayList<Object> arr = new ArrayList<>(1);
+        arr.add(convertCharSeq(v));
+        return arr;
+      }
+      return null;
+    }
+
+    public Collection<Object> getRawFieldValues(String name) {
+      Object v = _fields.get(name);
+      if (v instanceof Collection) {
+        return (Collection<Object>) v;
+      }
+      if (v != null) {
+        ArrayList<Object> arr = new ArrayList<>(1);
+        arr.add(v);
+        return arr;
+      }
+      return null;
+    }
+
+
+    /**
+     * Iterate of String-&gt;Object keys
+     */
+    @Override
+    public Iterator<Entry<String, Object>> iterator() {
+      Iterator<Entry<String, Object>> it = _fields.entrySet().iterator();
+      return new Iterator<Entry<String, Object>>() {
+        @Override
+        public boolean hasNext() {
+          return it.hasNext();
+        }
+
+        @Override
+        public Entry<String, Object> next() {
+          return convertCharSeq(it.next());
+        }
+      };
+    }
+
+
+    ///////////////////////////////////////////////////////////////////
+    // Get the field values
+    ///////////////////////////////////////////////////////////////////
+
+    /**
+     * returns the first value for a field
+     */
+    @Override
+    public Object getFirstValue(String name) {
+      Object v = _fields.get(name);
+      if (v == null || !(v instanceof Collection)) return convertCharSeq(v);
+      Collection c = (Collection) v;
+      if (c.size() > 0) {
+        return convertCharSeq(c.iterator().next());
+      }
+      return null;
+    }
+
+    @Override
+    public Object get(Object key) {
+      return convertCharSeq(_fields.get(key));
+    }
+
+    public Object getRaw(Object key) {
+      return _fields.get(key);
+    }
+
+    @Override
+    public void forEach(Consumer<? super Entry<String, Object>> action) {
+      super.forEach(action);
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index 67892cb..3d1976e 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -147,8 +147,10 @@ public class DocsStreamer implements Iterator<SolrDocument> {
     // can't just use fields.wantsField(String)
     // because that doesn't include extra fields needed by transformers
     final Set<String> fieldNamesNeeded = fields.getLuceneFieldNames();
-    
-    final SolrDocument out = new SolrDocument();
+
+    final SolrDocument out = ResultContext.READASBYTES.get() == null ?
+        new SolrDocument() :
+        new BinaryResponseWriter.MaskCharSeqSolrDocument();
 
     // NOTE: it would be tempting to try and optimize this to loop over fieldNamesNeeded
     // when it's smaller then the IndexableField[] in the Document -- but that's actually *less* effecient

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/response/ResultContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/ResultContext.java b/solr/core/src/java/org/apache/solr/response/ResultContext.java
index a9aff25..ccc2c5d 100644
--- a/solr/core/src/java/org/apache/solr/response/ResultContext.java
+++ b/solr/core/src/java/org/apache/solr/response/ResultContext.java
@@ -17,6 +17,7 @@
 package org.apache.solr.response;
 
 import java.util.Iterator;
+import java.util.function.Predicate;
 
 import org.apache.lucene.search.Query;
 import org.apache.solr.common.SolrDocument;
@@ -54,6 +55,7 @@ public abstract class ResultContext {
   public Iterator<SolrDocument> getProcessedDocuments() {
     return new DocsStreamer(this);
   }
+  public static final ThreadLocal<Predicate<String>>  READASBYTES = new ThreadLocal<>();
 }
 
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 5919c9e..8bcf839 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -62,6 +62,7 @@ import org.apache.lucene.util.CharsRefBuilder;
 import org.apache.lucene.util.Version;
 import org.apache.solr.analysis.SolrAnalyzer;
 import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.common.IteratorWriter;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.util.Base64;
@@ -133,6 +134,8 @@ public abstract class FieldType extends FieldProperties {
     return false;
   }
 
+  public boolean isUtf8Field(){return false;}
+
   /**
    * Returns true if the fields' docValues should be used for obtaining stored value
    */
@@ -157,6 +160,10 @@ public abstract class FieldType extends FieldProperties {
 
   }
 
+  public boolean write(IteratorWriter.ItemWriter itemWriter) {
+    return false;
+  }
+
   // Handle additional arguments...
   protected void setArgs(IndexSchema schema, Map<String,String> args) {
     // default to STORED, INDEXED, OMIT_TF_POSITIONS and MULTIVALUED depending on schema version

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index b31f6e9..098b3f0 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -37,6 +37,7 @@ import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -139,6 +140,8 @@ public class IndexSchema {
   protected volatile DynamicField[] dynamicFields;
   public DynamicField[] getDynamicFields() { return dynamicFields; }
 
+  protected Map<String, SchemaField> dynamicFieldCache = new ConcurrentHashMap<>();
+
   private Analyzer indexAnalyzer;
   private Analyzer queryAnalyzer;
 
@@ -1195,9 +1198,14 @@ public class IndexSchema {
   public SchemaField getFieldOrNull(String fieldName) {
     SchemaField f = fields.get(fieldName);
     if (f != null) return f;
+    f = dynamicFieldCache.get(fieldName);
+    if (f != null) return f;
 
     for (DynamicField df : dynamicFields) {
-      if (df.matches(fieldName)) return df.makeSchemaField(fieldName);
+      if (df.matches(fieldName)) {
+        dynamicFieldCache.put(fieldName, f = df.makeSchemaField(fieldName));
+        break;
+      }
     }
 
     return f;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/schema/StrField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/StrField.java b/solr/core/src/java/org/apache/solr/schema/StrField.java
index d9b51d1..a8ec62c 100644
--- a/solr/core/src/java/org/apache/solr/schema/StrField.java
+++ b/solr/core/src/java/org/apache/solr/schema/StrField.java
@@ -68,6 +68,12 @@ public class StrField extends PrimitiveFieldType {
     return Collections.singletonList(fval);
   }
 
+
+  @Override
+  public boolean isUtf8Field() {
+    return true;
+  }
+
   @Override
   public SortField getSortField(SchemaField field,boolean reverse) {
     return getStringSort(field,reverse);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/schema/TextField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/TextField.java b/solr/core/src/java/org/apache/solr/schema/TextField.java
index 8920c53..0d44eb7 100644
--- a/solr/core/src/java/org/apache/solr/schema/TextField.java
+++ b/solr/core/src/java/org/apache/solr/schema/TextField.java
@@ -209,4 +209,9 @@ public class TextField extends FieldType {
   public Object unmarshalSortValue(Object value) {
     return unmarshalStringSortValue(value);
   }
+
+  @Override
+  public boolean isUtf8Field() {
+    return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
index 45c1cf2..4d5018d 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Predicate;
 import java.util.function.Supplier;
 
 import org.apache.commons.collections.CollectionUtils;
@@ -37,7 +38,10 @@ import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DocumentStoredFieldVisitor;
+import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.LazyDocument;
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
@@ -58,11 +62,13 @@ import org.apache.lucene.util.NumericUtils;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentBase;
 import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.ByteArrayUtf8CharSequence;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.response.DocsStreamer;
+import org.apache.solr.response.ResultContext;
+import org.apache.solr.schema.AbstractEnumField;
 import org.apache.solr.schema.BoolField;
 import org.apache.solr.schema.LatLonPointSpatialField;
-import org.apache.solr.schema.AbstractEnumField;
 import org.apache.solr.schema.NumberType;
 import org.apache.solr.schema.SchemaField;
 import org.slf4j.Logger;
@@ -291,6 +297,22 @@ public class SolrDocumentFetcher {
       //    When: toLoad is one single-valued field, no lazyFieldProducer
     }
 
+
+    @Override
+    public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException {
+      Predicate<String> readAsBytes = ResultContext.READASBYTES.get();
+      if (readAsBytes != null && readAsBytes.test(fieldInfo.name)) {
+        final FieldType ft = new FieldType(TextField.TYPE_STORED);
+        ft.setStoreTermVectors(fieldInfo.hasVectors());
+        ft.setOmitNorms(fieldInfo.omitsNorms());
+        ft.setIndexOptions(fieldInfo.getIndexOptions());
+        doc.add(new StoredField(fieldInfo.name, new ByteArrayUtf8CharSequence(value, 0, value.length), ft));
+      } else {
+        super.stringField(fieldInfo, value);
+      }
+
+    }
+
     @Override
     public Status needsField(FieldInfo fieldInfo) throws IOException {
       Status status = super.needsField(fieldInfo);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java
index 5db49ca..bbc4985 100644
--- a/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestBinaryResponseWriter.java
@@ -18,19 +18,27 @@ package org.apache.solr.response;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
 
+import org.apache.lucene.document.StoredField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.ByteArrayUtf8CharSequence;
+import org.apache.solr.common.util.ByteUtils;
 import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.request.LocalSolrQueryRequest;
 import org.apache.solr.response.BinaryResponseWriter.Resolver;
 import org.apache.solr.search.SolrReturnFields;
-import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.util.SimplePostTool;
 import org.junit.BeforeClass;
 
 /**
@@ -48,6 +56,41 @@ public class TestBinaryResponseWriter extends SolrTestCaseJ4 {
     initCore("solrconfig.xml", "schema12.xml");
   }
 
+  public void testBytesRefWriting() {
+    compareStringFormat("ThisIsUTF8String");
+    compareStringFormat("Thailand (ประเทศไทย)");
+    compareStringFormat("LIVE: सबरीमाला मंदिर के पास पहुंची दो महिलाएं, जमकर हो रहा विरोध-प्रदर्शन");
+  }
+
+  public void testJavabinCodecWithCharSeq() throws IOException {
+    SolrDocument document = new SolrDocument();
+    document.put("id", "1");
+    String text = "नए लुक में धमाल मचाने आ रहे हैं MS Dhoni, कुछ यूं दिखाया हेलीकॉप्टर शॉट";
+    document.put("desc", new StoredField("desc", new ByteArrayUtf8CharSequence(text) {
+    }, TextField.TYPE_STORED));
+
+    NamedList nl = new NamedList();
+    nl.add("doc1", document);
+    SimplePostTool.BAOS baos = new SimplePostTool.BAOS();
+    new JavaBinCodec(new BinaryResponseWriter.Resolver(null, null)).marshal(nl, baos);
+    ByteBuffer byteBuffer = baos.getByteBuffer();
+    nl = (NamedList) new JavaBinCodec().unmarshal(new ByteArrayInputStream(byteBuffer.array(), 0, byteBuffer.limit()));
+    assertEquals(text, nl._get("doc1/desc", null));
+
+
+  }
+
+  private void compareStringFormat(String input) {
+    byte[] bytes1 = new byte[1024];
+    int len1 = ByteUtils.UTF16toUTF8(input, 0, input.length(), bytes1, 0);
+    BytesRef bytesref = new BytesRef(input);
+    System.out.println();
+    assertEquals(len1, bytesref.length);
+    for (int i = 0; i < len1; i++) {
+      assertEquals(input + " not matching char at :" + i, bytesref.bytes[i], bytes1[i]);
+    }
+  }
+
   /**
    * Tests known types implementation by asserting correct encoding/decoding of UUIDField
    */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
index 1b6b03e..5478568 100644
--- a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -30,11 +30,13 @@ public class ConditionalMapWriter implements MapWriter {
     this.predicate = predicate;
   }
 
-  private class EntryWriterWrapper implements EntryWriter {
+  public static class EntryWriterWrapper implements EntryWriter {
     private final EntryWriter delegate;
+    private final BiPredicate<CharSequence, Object> predicate;
 
-    EntryWriterWrapper(EntryWriter delegate) {
+    public EntryWriterWrapper(EntryWriter delegate, BiPredicate<CharSequence, Object> predicate) {
       this.delegate = delegate;
+      this.predicate = predicate;
     }
 
     @Override
@@ -71,7 +73,7 @@ public class ConditionalMapWriter implements MapWriter {
 
   @Override
   public void writeMap(EntryWriter ew) throws IOException {
-    if(delegate!=null) delegate.writeMap(new EntryWriterWrapper(ew));
+    if (delegate != null) delegate.writeMap(new EntryWriterWrapper(ew, predicate));
   }
 
   public static BiPredicate<CharSequence, Object> dedupeKeyPredicate(Set<CharSequence> keys) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
index fe43400..8d07bab 100644
--- a/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/LinkedHashMapWriter.java
@@ -38,7 +38,7 @@ public class LinkedHashMapWriter<V> extends LinkedHashMap<String, V> implements
 
   @Override
   public void writeMap(EntryWriter ew) throws IOException {
-    forEach((k, v) -> ew.putNoEx(k, v));
+    forEach(ew.getBiConsumer());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index a378e1d..926cf4c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.function.BiPredicate;
 
 import org.apache.solr.common.util.Utils;
@@ -154,5 +155,9 @@ public interface MapWriter extends MapSerializable , NavigableObject {
       return this;
 
     }
+
+    default BiConsumer<CharSequence, Object> getBiConsumer(){
+      return (k, v) -> putNoEx(k,v);
+    }
   }
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
index e0dd0ac..f6b978f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriterMap.java
@@ -30,7 +30,7 @@ public class MapWriterMap implements MapWriter {
 
   @Override
   public void writeMap(EntryWriter ew) throws IOException {
-    delegate.forEach((k, v) -> ew.putNoEx(k == null ? null : k.toString(), v));
+    delegate.forEach(ew.getBiConsumer());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
index 910ec51..0563c9e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocument.java
@@ -16,6 +16,7 @@
  */
 package org.apache.solr.common;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -27,6 +28,8 @@ import java.util.Set;
 
 import org.apache.solr.common.util.NamedList;
 
+import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;
+
 
 /**
  * A concrete representation of a document within a Solr index.  Unlike a lucene
@@ -41,7 +44,7 @@ import org.apache.solr.common.util.NamedList;
  */
 public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> implements Iterable<Map.Entry<String, Object>>
 {
-  private final Map<String,Object> _fields;
+  protected final Map<String,Object> _fields;
   
   private List<SolrDocument> _childDocuments;
   
@@ -50,6 +53,11 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
     _fields = new LinkedHashMap<>();
   }
 
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    _fields.forEach(ew.getBiConsumer());
+  }
+
   public SolrDocument(Map<String, Object> fields) {
     this._fields = fields;
   }
@@ -281,14 +289,14 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
       /** Get the field Value */
       @Override
       public Object get(Object key) { 
-        return getFirstValue( (String)key ); 
+        return convertCharSeq(getFirstValue( (String)key));
       }
       
       // Easily Supported methods
       @Override
       public boolean containsKey(Object key) { return _fields.containsKey( key ); }
       @Override
-      public Set<String>  keySet()           { return _fields.keySet();  }
+      public Set<String>  keySet()           { return (Set<String>) convertCharSeq(_fields.keySet());  }
       @Override
       public int          size()             { return _fields.size();    }
       @Override
@@ -360,7 +368,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
 
   @Override
   public Object remove(Object key) {
-    return _fields.remove(key);
+    return convertCharSeq(_fields.remove(key));
   }
 
   @Override
@@ -370,7 +378,7 @@ public class SolrDocument extends SolrDocumentBase<Object, SolrDocument> impleme
 
   @Override
   public Collection<Object> values() {
-    return _fields.values();
+    return convertCharSeq(_fields.values());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/SolrDocumentBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrDocumentBase.java b/solr/solrj/src/java/org/apache/solr/common/SolrDocumentBase.java
index 4429fbe..21d79a9 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrDocumentBase.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrDocumentBase.java
@@ -21,7 +21,7 @@ import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 
-public abstract class SolrDocumentBase<T, K> implements Map<String, T>, Serializable {
+public abstract class SolrDocumentBase<T, K> implements Map<String, T>, Serializable, MapWriter {
 
   /** Get all field names.
   */

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
index 172606f..e1fa311 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrInputDocument.java
@@ -16,6 +16,7 @@
  */
 package org.apache.solr.common;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -44,7 +45,12 @@ public class SolrInputDocument extends SolrDocumentBase<SolrInputField, SolrInpu
       addField(fields[i], fields[i + 1]);
     }
   }
-  
+
+  @Override
+  public void writeMap(EntryWriter ew) throws IOException {
+    _fields.forEach(ew.getBiConsumer());
+  }
+
   public SolrInputDocument(Map<String,SolrInputField> fields) {
     _fields = fields;
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java b/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java
index aecb50d..4ac48f2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/ByteArrayUtf8CharSequence.java
@@ -20,11 +20,14 @@ package org.apache.solr.common.util;
 import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.noggit.CharArr;
 
 /**A mutable byte[] backed Utf8CharSequence. This is quite similar to the BytesRef of Lucene
+ * Do not alter the contents of the byte[] . it may be inconsistent with the cached String
  * This is designed for single-threaded use
  *
  */
@@ -60,6 +63,7 @@ public class ByteArrayUtf8CharSequence implements Utf8CharSequence {
   @Override
   public int write(int start, byte[] buffer, int pos) {
     if (start == -1 || start >= length) return -1;
+    if (length == 0) return 0;
     int writableBytes = Math.min(length - start, buffer.length - pos);
     System.arraycopy(buf, offset + start, buffer, pos, writableBytes);
     return writableBytes;
@@ -139,11 +143,11 @@ public class ByteArrayUtf8CharSequence implements Utf8CharSequence {
     return new ByteArrayUtf8CharSequence(bytes, 0, length, utf16, hashCode);
   }
 
-  public static Map.Entry convertCharSeq(Map.Entry result) {
-    if (result.getKey() instanceof Utf8CharSequence || result.getValue() instanceof Utf8CharSequence) {
-      return new AbstractMap.SimpleEntry(convertCharSeq(result.getKey()), convertCharSeq(result.getValue()));
+  public static Map.Entry convertCharSeq(Map.Entry e) {
+    if (e.getKey() instanceof Utf8CharSequence || e.getValue() instanceof Utf8CharSequence) {
+      return new AbstractMap.SimpleEntry(convertCharSeq(e.getKey()), convertCharSeq(e.getValue()));
     }
-    return result;
+    return e;
 
   }
 
@@ -157,7 +161,12 @@ public class ByteArrayUtf8CharSequence implements Utf8CharSequence {
       }
     }
     if (needsCopy) {
-      ArrayList copy = new ArrayList(vals.size());
+      Collection copy =  null;
+      if (vals instanceof Set){
+        copy = new HashSet(vals.size());
+      } else {
+        copy = new ArrayList(vals.size());
+      }
       for (Object o : vals) copy.add(convertCharSeq(o));
       return copy;
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java b/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
index 7a2d00e..442f9ac 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/FastOutputStream.java
@@ -238,11 +238,11 @@ public class FastOutputStream extends OutputStream implements DataOutput {
     int start = 0;
     int totalWritten = 0;
     for (; ; ) {
+      if (totalWritten >= utf8.size()) break;
       if (pos >= buf.length) flushBuffer();
       int sz = utf8.write(start, buf, pos);
       pos += sz;
       totalWritten += sz;
-      if (totalWritten >= utf8.size()) break;
       start += sz;
     }
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
index ca8c80f..586adfb 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/JavaBinCodec.java
@@ -35,7 +35,9 @@ import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
 
+import org.apache.solr.common.ConditionalMapWriter;
 import org.apache.solr.common.EnumFieldValue;
 import org.apache.solr.common.IteratorWriter;
 import org.apache.solr.common.IteratorWriter.ItemWriter;
@@ -471,9 +473,17 @@ public class JavaBinCodec implements PushWriter {
       writeStr(v);
       return this;
     }
+
+    private BiConsumer<CharSequence, Object> biConsumer;
+
+    @Override
+    public BiConsumer<CharSequence, Object> getBiConsumer() {
+      if (biConsumer == null) biConsumer = MapWriter.EntryWriter.super.getBiConsumer();
+      return biConsumer;
+    }
   }
 
-  private final MapWriter.EntryWriter ew = new BinEntryWriter();
+  public final BinEntryWriter ew = new BinEntryWriter();
 
 
   public void writeMap(MapWriter val) throws IOException {
@@ -514,6 +524,7 @@ public class JavaBinCodec implements PushWriter {
   //use this to ignore the writable interface because , child docs will ignore the fl flag
   // is it a good design?
   private boolean ignoreWritable =false;
+  private ConditionalMapWriter.EntryWriterWrapper cew;
 
   public void writeSolrDocument(SolrDocument doc) throws IOException {
     List<SolrDocument> children = doc.getChildDocuments();
@@ -528,14 +539,8 @@ public class JavaBinCodec implements PushWriter {
     int sz = fieldsCount + (children==null ? 0 : children.size());
     writeTag(SOLRDOC);
     writeTag(ORDERED_MAP, sz);
-    for (Map.Entry<String, Object> entry : doc) {
-      String name = entry.getKey();
-      if(toWrite(name)) {
-        writeExternString(name);
-        Object val = entry.getValue();
-        writeVal(val);
-      }
-    }
+    if (cew == null) cew = new ConditionalMapWriter.EntryWriterWrapper(ew, (k, o) -> toWrite(k.toString()));
+    doc.writeMap(cew);
     if (children != null) {
       try {
         ignoreWritable = true;
@@ -683,7 +688,7 @@ public class JavaBinCodec implements PushWriter {
     return m;
   }
 
-  private final ItemWriter itemWriter = new ItemWriter() {
+  public final ItemWriter itemWriter = new ItemWriter() {
     @Override
     public ItemWriter add(Object o) throws IOException {
       writeVal(o);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
index 71cb78b..ec40f97 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/NamedList.java
@@ -79,7 +79,7 @@ public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry
   @Override
   public void writeMap(EntryWriter ew) throws IOException {
     for (int i = 0; i < nvPairs.size(); i+=2) {
-      ew.put((String) nvPairs.get(i), nvPairs.get(i+1));
+      ew.put((CharSequence) nvPairs.get(i), nvPairs.get(i + 1));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/java/org/apache/solr/common/util/Utf8CharSequence.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utf8CharSequence.java b/solr/solrj/src/java/org/apache/solr/common/util/Utf8CharSequence.java
index 5b884ec..bd056f5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/Utf8CharSequence.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utf8CharSequence.java
@@ -52,10 +52,10 @@ public interface Utf8CharSequence extends CharSequence {
     int start = 0;
     int totalWritten = 0;
     for (; ; ) {
+      if (totalWritten >= size()) break;
       int sz = write(start, buf, 0);
       totalWritten += sz;
       if (sz > 0) os.write(buf, 0, sz);
-      if (totalWritten >= size()) break;
       start += sz;
     }
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c651cbfd/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
index 43a3153..5fe8c9b 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleJettyTest.java
@@ -17,8 +17,13 @@
 package org.apache.solr.client.solrj.embedded;
 
 import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.io.IOUtils;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
@@ -26,12 +31,17 @@ import org.apache.http.entity.InputStreamEntity;
 import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
 import org.apache.solr.client.solrj.SolrExampleTests;
 import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
 import org.apache.solr.client.solrj.impl.HttpClientUtil;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
 import org.junit.Assert;
 import org.junit.BeforeClass;
+import org.junit.Ignore;
 import org.junit.Test;
 import org.noggit.ObjectBuilder;
 
@@ -101,4 +111,59 @@ public class SolrExampleJettyTest extends SolrExampleTests {
         baseURL.replace("/collection1", "/____v2/cores/collection1/update") :
         baseURL + "/update/json/docs";
   }
+
+  @Ignore
+  public void testUtf8QueryPerf() throws Exception {
+    HttpSolrClient client = (HttpSolrClient) getSolrClient();
+    client.deleteByQuery("*:*");
+    client.commit();
+    List<SolrInputDocument> docs = new ArrayList<>();
+    for (int i = 0; i < 10; i++) {
+      SolrInputDocument doc2 = new SolrInputDocument();
+      doc2.addField("id", "" + i);
+      doc2.addField("fld1_s", "1 value 1 value 1 value 1 value 1 value 1 value 1 value ");
+      doc2.addField("fld2_s", "2 value 2 value 2 value 2 value 2 value 2 value 2 value 2 value 2 value 2 value ");
+      doc2.addField("fld3_s", "3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value 3 value ");
+      doc2.addField("fld4_s", "4 value 4 value 4 value 4 value 4 value 4 value 4 value 4 value 4 value ");
+      doc2.addField("fld5_s", "5 value 5 value 5 value 5 value 5 value 5 value 5 value 5 value 5 value 5 value 5 value 5 value ");
+      docs.add(doc2);
+    }
+    client.add(docs);
+    client.commit();
+    QueryResponse rsp = client.query(new SolrQuery("*:*"));
+    assertEquals(10, rsp.getResults().getNumFound());
+
+
+    client.setParser(new BinaryResponseParser() {
+      @Override
+      public NamedList<Object> processResponse(InputStream body, String encoding) {
+        try {
+          IOUtils.skip(body, 1024 * 1000);
+        } catch (IOException e) {
+          e.printStackTrace();
+        }
+        return rsp.getResponse();
+      }
+    });
+
+
+    runQueries(client, 1000, true);
+    /*BinaryResponseWriter.useUtf8CharSeq = false;
+    System.out.println("BinaryResponseWriter.useUtf8CharSeq = " + BinaryResponseWriter.useUtf8CharSeq);
+    runQueries(client, 10000, false);
+    BinaryResponseWriter.useUtf8CharSeq = true;
+    System.out.println("BinaryResponseWriter.useUtf8CharSeq = " + BinaryResponseWriter.useUtf8CharSeq);*/
+    runQueries(client, 10000, false);
+  }
+
+
+  private void runQueries(HttpSolrClient client, int count, boolean warmup) throws SolrServerException, IOException {
+    long start = System.nanoTime();
+    for (int i = 0; i < count; i++) {
+      client.query(new SolrQuery("*:*"));
+    }
+    if (warmup) return;
+    System.out.println("time taken : " + ((System.nanoTime() - start)) / (1000 * 1000));
+  }
+
 }


[07/47] lucene-solr:jira/http2: LUCENE-8592: Fix index sorting corruption due to numeric overflow

Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 3857a97..5bfd9d5 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -355,166 +355,206 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingStringFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.STRING);
-    sortField.setMissingValue(SortField.STRING_FIRST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+      sortField.setMissingValue(SortField.STRING_FIRST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    SortedDocValues values = leaf.getSortedDocValues("foo");
-    // docID 0 is missing:
-    assertEquals(1, values.nextDoc());
-    assertEquals("mmm", values.binaryValue().utf8ToString());
-    assertEquals(2, values.nextDoc());
-    assertEquals("zzz", values.binaryValue().utf8ToString());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      SortedDocValues values = leaf.getSortedDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+        assertEquals(1, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+      } else {
+        // docID 0 is missing:
+        assertEquals(1, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+        assertEquals(2, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedStringFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedSetSortField("foo", false);
-    sortField.setMissingValue(SortField.STRING_FIRST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedSetSortField("foo", reverse);
+      sortField.setMissingValue(SortField.STRING_FIRST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1l, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2l, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3l, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1l, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3l, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingStringLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.STRING);
-    sortField.setMissingValue(SortField.STRING_LAST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
+      sortField.setMissingValue(SortField.STRING_LAST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    SortedDocValues values = leaf.getSortedDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals("mmm", values.binaryValue().utf8ToString());
-    assertEquals(1, values.nextDoc());
-    assertEquals("zzz", values.binaryValue().utf8ToString());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      SortedDocValues values = leaf.getSortedDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+        assertEquals(2, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals("mmm", values.binaryValue().utf8ToString());
+        assertEquals(1, values.nextDoc());
+        assertEquals("zzz", values.binaryValue().utf8ToString());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedStringLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedSetSortField("foo", false);
-    sortField.setMissingValue(SortField.STRING_LAST);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedSetSortField("foo", reverse);
+      sortField.setMissingValue(SortField.STRING_LAST);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-    doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+      doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1l, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2l, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3l, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1l, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1l, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2l, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3l, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testBasicLong() throws Exception {
@@ -597,11 +637,213 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingLongFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        // docID 0 has no value
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedLongFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 27));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 24));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingLongLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        // docID 0 is missing
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedLongLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
+      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 65));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 34));
+      doc.add(new SortedNumericDocValuesField("foo", 74));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testBasicInt() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -610,8 +852,9 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
-    w.addDocument(new Document());
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", -1));
+    w.addDocument(doc);
     w.commit();
 
     doc = new Document();
@@ -623,7 +866,8 @@ public class TestIndexSorting extends LuceneTestCase {
     LeafReader leaf = getOnlyLeafReader(r);
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
-    // docID 0 has no value
+    assertEquals(0, values.nextDoc());
+    assertEquals(-1, values.longValue());
     assertEquals(1, values.nextDoc());
     assertEquals(7, values.longValue());
     assertEquals(2, values.nextDoc());
@@ -633,195 +877,32 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
-  public void testMissingMultiValuedLongFirst() throws Exception {
+  public void testBasicMultiValuedInt() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
     doc.add(new NumericDocValuesField("id", 3));
     doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 27));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", -1));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 2));
     doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 24));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingLongLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    w.addDocument(new Document());
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(18, values.longValue());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingMultiValuedLongLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
-    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 65));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    doc.add(new SortedNumericDocValuesField("foo", 74));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicInt() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", -1));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(-1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(18, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicMultiValuedInt() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", -1));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 22));
-    doc.add(new SortedNumericDocValuesField("foo", 27));
+    doc.add(new SortedNumericDocValuesField("foo", 22));
+    doc.add(new SortedNumericDocValuesField("foo", 27));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -841,332 +922,211 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingIntFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(18, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedIntFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 187667));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 187667));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 34));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingIntLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 18));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 18));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", 7));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("foo", 7));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(18, values.longValue());
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        // docID 0 is missing
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(7, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(18, values.longValue());
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedIntLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
-    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 6372));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 8));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicDouble() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 18.0));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", -1.0));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 7.0));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testBasicMultiValuedDouble() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
-    w.addDocument(doc);
-    w.commit();
-
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingDoubleFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 18.0));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
-
-    // missing
-    w.addDocument(new Document());
-    w.commit();
-
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", 7.0));
-    w.addDocument(doc);
-    w.forceMerge(1);
-
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    r.close();
-    w.close();
-    dir.close();
-  }
-
-  public void testMissingMultiValuedDoubleFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
+      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", 18));
+      doc.add(new SortedNumericDocValuesField("foo", 6372));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", 7));
+      doc.add(new SortedNumericDocValuesField("foo", 8));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
-  public void testMissingDoubleLast() throws Exception {
+  public void testBasicDouble() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -1175,8 +1135,9 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
-    w.addDocument(new Document());
+    doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", -1.0));
+    w.addDocument(doc);
     w.commit();
 
     doc = new Document();
@@ -1189,41 +1150,41 @@ public class TestIndexSorting extends LuceneTestCase {
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
     assertEquals(0, values.nextDoc());
-    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
     assertEquals(1, values.nextDoc());
+    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(2, values.nextDoc());
     assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
     r.close();
     w.close();
     dir.close();
   }
 
-  public void testMissingMultiValuedDoubleLast() throws Exception {
+  public void testBasicMultiValuedDouble() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
-    sortField.setMissingValue(Double.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    // missing
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new NumericDocValuesField("id", 2));
     doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -1242,6 +1203,207 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
+  public void testMissingDoubleFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 18.0));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 7.0));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(2, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedDoubleFirst() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingDoubleLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 18.0));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      w.addDocument(new Document());
+      w.commit();
+
+      doc = new Document();
+      doc.add(new DoubleDocValuesField("foo", 7.0));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(2, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
+  public void testMissingMultiValuedDoubleLast() throws Exception {
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
+      sortField.setMissingValue(Double.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
+
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
+
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
+      w.addDocument(doc);
+      w.forceMerge(1);
+
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
+  }
+
   public void testBasicFloat() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1322,164 +1484,204 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingFloatFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 18.0f));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 18.0f));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 7.0f));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 7.0f));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(1, values.nextDoc());
-    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(2, values.nextDoc());
-    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      } else {
+        assertEquals(1, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(2, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedFloatFirst() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingFloatLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 18.0f));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 18.0f));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    w.addDocument(new Document());
-    w.commit();
+      // missing
+      w.addDocument(new Document());
+      w.commit();
 
-    doc = new Document();
-    doc.add(new FloatDocValuesField("foo", 7.0f));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new FloatDocValuesField("foo", 7.0f));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(1, values.nextDoc());
-    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-    assertEquals(NO_MORE_DOCS, values.nextDoc());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("foo");
+      if (reverse) {
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(2, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+        assertEquals(1, values.nextDoc());
+        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+      }
+      assertEquals(NO_MORE_DOCS, values.nextDoc());
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testMissingMultiValuedFloatLast() throws Exception {
-    Directory dir = newDirectory();
-    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
-    sortField.setMissingValue(Float.POSITIVE_INFINITY);
-    Sort indexSort = new Sort(sortField);
-    iwc.setIndexSort(indexSort);
-    IndexWriter w = new IndexWriter(dir, iwc);
-    Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-    w.addDocument(doc);
-    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-    w.commit();
+    for (boolean reverse : new boolean[] {true, false}) {
+      Directory dir = newDirectory();
+      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
+      sortField.setMissingValue(Float.POSITIVE_INFINITY);
+      Sort indexSort = new Sort(sortField);
+      iwc.setIndexSort(indexSort);
+      IndexWriter w = new IndexWriter(dir, iwc);
+      Document doc = new Document();
+      doc.add(new NumericDocValuesField("id", 2));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+      w.addDocument(doc);
+      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+      w.commit();
 
-    // missing
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    w.addDocument(doc);
-    w.commit();
+      // missing
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 3));
+      w.addDocument(doc);
+      w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-    w.addDocument(doc);
-    w.forceMerge(1);
+      doc = new Document();
+      doc.add(new NumericDocValuesField("id", 1));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+      w.addDocument(doc);
+      w.forceMerge(1);
 
-    DirectoryReader r = DirectoryReader.open(w);
-    LeafReader leaf = getOnlyLeafReader(r);
-    assertEquals(3, leaf.maxDoc());
-    NumericDocValues values = leaf.getNumericDocValues("id");
-    assertEquals(0, values.nextDoc());
-    assertEquals(1, values.longValue());
-    assertEquals(1, values.nextDoc());
-    assertEquals(2, values.longValue());
-    assertEquals(2, values.nextDoc());
-    assertEquals(3, values.longValue());
-    r.close();
-    w.close();
-    dir.close();
+      DirectoryReader r = DirectoryReader.open(w);
+      LeafReader leaf = getOnlyLeafReader(r);
+      assertEquals(3, leaf.maxDoc());
+      NumericDocValues values = leaf.getNumericDocValues("id");
+      if (reverse) {
+        assertEquals(0, values.nextDoc());
+        assertEquals(3, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(1, values.longValue());
+      } else {
+        assertEquals(0, values.nextDoc());
+        assertEquals(1, values.longValue());
+        assertEquals(1, values.nextDoc());
+        assertEquals(2, values.longValue());
+        assertEquals(2, values.nextDoc());
+        assertEquals(3, values.longValue());
+      }
+      r.close();
+      w.close();
+      dir.close();
+    }
   }
 
   public void testRandom1() throws IOException {


[32/47] lucene-solr:jira/http2: SOLR-12923: Harden TestSimTriggerIntegration.

Posted by da...@apache.org.
SOLR-12923: Harden TestSimTriggerIntegration.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/18356de8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/18356de8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/18356de8

Branch: refs/heads/jira/http2
Commit: 18356de83738d64e619898016d873993ec474d17
Parents: 5405b2f
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 22:42:40 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 22:55:32 2018 -0600

----------------------------------------------------------------------
 .../cloud/autoscaling/sim/SimDistribStateManager.java   | 11 +++++++----
 .../autoscaling/sim/TestSimTriggerIntegration.java      | 12 ++++++------
 2 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18356de8/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java
index 2b8940a..7b73038 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimDistribStateManager.java
@@ -476,23 +476,26 @@ public class SimDistribStateManager implements DistribStateManager {
   @Override
   public void removeData(String path, int version) throws NoSuchElementException, NotEmptyException, BadVersionException, IOException {
     multiLock.lock();
+    Node parent;
+    Node n;
     try {
-      Node n = traverse(path, false, CreateMode.PERSISTENT);
+      n = traverse(path, false, CreateMode.PERSISTENT);
       if (n == null) {
         throw new NoSuchElementException(path);
       }
-      Node parent = n.parent;
+      parent = n.parent;
       if (parent == null) {
         throw new IOException("Cannot remove root node");
       }
       if (!n.children.isEmpty()) {
         throw new NotEmptyException(path);
       }
-      parent.removeChild(n.name, version);
     } finally {
       multiLock.unlock();
     }
-
+    
+    // outside the lock to avoid deadlock with update lock
+    parent.removeChild(n.name, version);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/18356de8/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 8f37ddb..c4ec122 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -210,7 +210,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
 
     String newNode = cluster.simAddNode();
 
-    if (!triggerFiredLatch.await(45000 / SPEED, TimeUnit.MILLISECONDS)) {
+    if (!triggerFiredLatch.await(220000 / SPEED, TimeUnit.MILLISECONDS)) {
       fail("Both triggers should have fired by now");
     }
 
@@ -350,7 +350,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
       fail("Two TriggerAction instances should have been created by now");
     }
 
-    boolean await = triggerFiredLatch.await(45000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFiredLatch.await(90000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     assertTrue(triggerFired.get());
     NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
@@ -408,7 +408,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
       fail("Two TriggerAction instances should have been created by now");
     }
 
-    boolean await = triggerFiredLatch.await(20000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFiredLatch.await(60000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     assertTrue(triggerFired.get());
     TriggerEvent nodeAddedEvent = events.iterator().next();
@@ -438,7 +438,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
     }
 
     String newNode = cluster.simAddNode();
-    boolean await = triggerFiredLatch.await(45000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFiredLatch.await(60000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     assertTrue(triggerFired.get());
     TriggerEvent nodeAddedEvent = events.iterator().next();
@@ -726,7 +726,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
     events.clear();
 
     String newNode = cluster.simAddNode();
-    boolean await = triggerFiredLatch.await(60000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFiredLatch.await(90000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     assertTrue(triggerFired.get());
     // reset
@@ -924,7 +924,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
     }
 
 
-    if (!triggerFiredLatch.await(30000 / SPEED, TimeUnit.MILLISECONDS)) {
+    if (!triggerFiredLatch.await(60000 / SPEED, TimeUnit.MILLISECONDS)) {
       fail("Trigger should have fired by now");
     }
     assertEquals(1, events.size());


[34/47] lucene-solr:jira/http2: SOLR-12801: Try to harden closing of zk db in TestConfigSetsAPIZkFailure.

Posted by da...@apache.org.
SOLR-12801: Try to harden closing of zk db in TestConfigSetsAPIZkFailure.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/56b95914
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/56b95914
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/56b95914

Branch: refs/heads/jira/http2
Commit: 56b959141fa3e06db2ea6c079adb01bdab87c003
Parents: 1305501
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 23:03:11 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Dec 10 10:02:58 2018 -0600

----------------------------------------------------------------------
 .../src/java/org/apache/solr/cloud/ZkTestServer.java         | 8 ++++++++
 1 file changed, 8 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/56b95914/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
index 1150c48..8aa1e7d 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
@@ -104,6 +104,8 @@ public class ZkTestServer {
   protected volatile SolrZkClient rootClient;
   protected volatile SolrZkClient chRootClient;
 
+  private volatile ZKDatabase zkDb;
+
   static public enum LimitViolationAction {
     IGNORE,
     REPORT,
@@ -514,6 +516,7 @@ public class ZkTestServer {
   }
 
   public void setZKDatabase(ZKDatabase zkDb) {
+    this.zkDb = zkDb;
     zkServer.zooKeeperServer.setZKDatabase(zkDb);
   }
   
@@ -611,6 +614,11 @@ public class ZkTestServer {
       } catch (Exception e) {
         log.error("Exception shutting down ZooKeeper Test Server",e);
       }
+      
+      if (zkDb != null) {
+        zkDb.close();
+      }
+      
       while (true) {
         try {
           zooThread.join();


[28/47] lucene-solr:jira/http2: SOLR-12801: Harden CollectionsAPISolrJTest#testCreateCollWithDefaultClusterPropertiesOldFormat.

Posted by da...@apache.org.
SOLR-12801: Harden CollectionsAPISolrJTest#testCreateCollWithDefaultClusterPropertiesOldFormat.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/665517d3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/665517d3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/665517d3

Branch: refs/heads/jira/http2
Commit: 665517d3bff736e356643f5c7978c0e10b1ce18b
Parents: 1d5ae98
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 18:50:47 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:50:47 2018 -0600

----------------------------------------------------------------------
 .../src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java   | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/665517d3/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 0925eee..166748e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -75,7 +75,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
     
     // clear any persisted auto scaling configuration
     zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
-    cluster.deleteAllCollections();
     
     final ClusterProperties props = new ClusterProperties(zkClient());
     CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, null).process(cluster.getSolrClient());
@@ -130,7 +129,7 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
           .build()
           .process(cluster.getSolrClient());
 
-      for (int i = 0; i < 30; i++) {
+      for (int i = 0; i < 300; i++) {
         Map m = cluster.getSolrClient().getZkStateReader().getClusterProperty(COLLECTION_DEF, null);
         if (m != null) break;
         Thread.sleep(10);


[36/47] lucene-solr:jira/http2: SOLR-12923: Harden TestSimLargeCluster.

Posted by da...@apache.org.
SOLR-12923: Harden TestSimLargeCluster.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/202dad8c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/202dad8c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/202dad8c

Branch: refs/heads/jira/http2
Commit: 202dad8c88ec29c45cabfe1e24f045b858637618
Parents: bf0e9b3
Author: markrmiller <ma...@apache.org>
Authored: Mon Dec 10 09:39:42 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Dec 10 10:02:58 2018 -0600

----------------------------------------------------------------------
 .../cloud/autoscaling/sim/SimCloudManager.java  |  1 +
 .../sim/SimClusterStateProvider.java            |  2 +-
 .../autoscaling/sim/TestSimLargeCluster.java    | 26 +++++++++-----------
 3 files changed, 13 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/202dad8c/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
index a6d76c8..dad9463 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimCloudManager.java
@@ -889,6 +889,7 @@ public class SimCloudManager implements SolrCloudManager {
     IOUtils.closeQuietly(stateManager);
     triggerThread.interrupt();
     IOUtils.closeQuietly(triggerThread);
+    triggerThread.interrupt();
     try {
       triggerThread.join();
     } catch (InterruptedException e) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/202dad8c/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 930b761..d81d92c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -1369,7 +1369,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
           OverseerCollectionMessageHandler.NUM_SLICES, "1",
           CommonAdminParams.WAIT_FOR_FINAL_STATE, "true");
       simCreateCollection(props, new NamedList());
-      CloudTestUtils.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL, 90, TimeUnit.SECONDS,
+      CloudTestUtils.waitForState(cloudManager, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS,
           CloudTestUtils.clusterShape(1, Integer.parseInt(repFactor), false, true));
     } catch (Exception e) {
       throw new IOException(e);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/202dad8c/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
index e7a16e3..bc4c598 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
@@ -53,7 +53,6 @@ import org.apache.solr.cloud.autoscaling.TriggerListenerBase;
 import org.apache.solr.cloud.autoscaling.TriggerValidationException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CollectionParams;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.common.util.Pair;
@@ -78,7 +77,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
   public static final int SPEED = 50;
 
   public static final int NUM_NODES = 100;
-
+ 
   static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
   static AtomicInteger triggerFinishedCount = new AtomicInteger();
   static AtomicInteger triggerStartedCount = new AtomicInteger();
@@ -133,12 +132,6 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
       }
     }
 
-    // do this in advance if missing
-    if (!cluster.getSimClusterStateProvider().simListCollections().contains(CollectionAdminParams.SYSTEM_COLL)) {
-      cluster.getSimClusterStateProvider().createSystemCollection();
-      CloudTestUtils.waitForState(cluster, CollectionAdminParams.SYSTEM_COLL, 120, TimeUnit.SECONDS,
-          CloudTestUtils.clusterShape(1, 3, false, true));
-    }
   }
 
   public static class TestTriggerListener extends TriggerListenerBase {
@@ -310,7 +303,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
     create.setAutoAddReplicas(false);
     create.process(solrClient);
 
-    log.info("Ready after " + CloudTestUtils.waitForState(cluster, collectionName, 20 * NUM_NODES, TimeUnit.SECONDS,
+    log.info("Ready after " + CloudTestUtils.waitForState(cluster, collectionName, 90 * NUM_NODES, TimeUnit.SECONDS,
         CloudTestUtils.clusterShape(NUM_NODES / 10, NUM_NODES / 8 * 3, false, true)) + " ms");
 
     // start adding nodes
@@ -325,7 +318,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
     assertTrue("trigger did not fire", await);
 
     // wait until started == finished
-    TimeOut timeOut = new TimeOut(20 * waitForSeconds * NUM_NODES, TimeUnit.SECONDS, cluster.getTimeSource());
+    TimeOut timeOut = new TimeOut(45 * waitForSeconds * NUM_NODES, TimeUnit.SECONDS, cluster.getTimeSource());
     while (!timeOut.hasTimedOut()) {
       if (triggerStartedCount.get() == triggerFinishedCount.get()) {
         break;
@@ -510,7 +503,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
     create.setAutoAddReplicas(false);
     create.process(solrClient);
 
-    log.info("Ready after " + CloudTestUtils.waitForState(cluster, collectionName, 30 * NUM_NODES, TimeUnit.SECONDS,
+    log.info("Ready after " + CloudTestUtils.waitForState(cluster, collectionName, 60 * NUM_NODES, TimeUnit.SECONDS,
         CloudTestUtils.clusterShape(NUM_NODES / 5, NUM_NODES / 10, false, true)) + " ms");
 
     // start killing nodes
@@ -701,12 +694,15 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
     assertEquals(response.get("result").toString(), "success");
 
 
-    boolean await = triggerFinishedLatch.await(waitForSeconds * 20000 / SPEED, TimeUnit.MILLISECONDS);
+    boolean await = triggerFinishedLatch.await(waitForSeconds * 45000 / SPEED, TimeUnit.MILLISECONDS);
     assertTrue("The trigger did not fire at all", await);
     // wait for listener to capture the SUCCEEDED stage
-    cluster.getTimeSource().sleep(5000);
-    assertNotNull(listenerEvents.toString(), listenerEvents.get("srt"));
-    assertEquals(listenerEvents.toString(), 1, listenerEvents.get("srt").size());
+    cluster.getTimeSource().sleep(15000);
+    
+    assertNotNull(listenerEvents.entrySet().toString(), listenerEvents.get("srt"));
+
+    assertTrue(listenerEvents.toString(), listenerEvents.get("srt").size() >= 1);
+
     CapturedEvent ev = listenerEvents.get("srt").get(0);
     assertEquals(TriggerEventType.SEARCHRATE, ev.event.getEventType());
     Map<String, Number> m = (Map<String, Number>)ev.event.getProperty(SearchRateTrigger.HOT_NODES);


[21/47] lucene-solr:jira/http2: SOLR-12801: use waitForActiveCollection call

Posted by da...@apache.org.
SOLR-12801: use waitForActiveCollection call


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0824f7b0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0824f7b0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0824f7b0

Branch: refs/heads/jira/http2
Commit: 0824f7b045f4f9b1376de7f6707ae37bf7a48d74
Parents: f6063a1
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 15:51:17 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../org/apache/solr/cloud/AssignBackwardCompatibilityTest.java  | 5 +++--
 .../src/test/org/apache/solr/cloud/TestCloudConsistency.java    | 2 ++
 .../test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java | 3 +--
 .../apache/solr/cloud/api/collections/CustomCollectionTest.java | 2 ++
 4 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0824f7b0/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java b/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
index 8700e14..c1ba972 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
@@ -53,6 +53,7 @@ public class AssignBackwardCompatibilityTest extends SolrCloudTestCase {
     CollectionAdminRequest.createCollection(COLLECTION, 1, 4)
         .setMaxShardsPerNode(1000)
         .process(cluster.getSolrClient());
+    cluster.waitForActiveCollection(COLLECTION, 1, 4);
   }
 
   @Test
@@ -75,7 +76,7 @@ public class AssignBackwardCompatibilityTest extends SolrCloudTestCase {
         clearedCounter = true;
       }
       if (deleteReplica) {
-        waitForState("Expected " + numLiveReplicas + " active replicas", COLLECTION, clusterShape(1, numLiveReplicas));
+        cluster.waitForActiveCollection(COLLECTION, 1, numLiveReplicas);
         DocCollection dc = getCollectionState(COLLECTION);
         Replica replica = getRandomReplica(dc.getSlice("shard1"), (r) -> r.getState() == Replica.State.ACTIVE);
         CollectionAdminRequest.deleteReplica(COLLECTION, "shard1", replica.getName()).process(cluster.getSolrClient());
@@ -90,7 +91,7 @@ public class AssignBackwardCompatibilityTest extends SolrCloudTestCase {
         assertFalse("Core name is not unique coreName=" + coreName + " " + coreNames, coreNames.contains(coreName));
         coreNames.add(coreName);
         numLiveReplicas++;
-        waitForState("Expected " + numLiveReplicas + " active replicas", COLLECTION, clusterShape(1, numLiveReplicas));
+        cluster.waitForActiveCollection(COLLECTION, 1, numLiveReplicas);
 
         Replica newReplica = getCollectionState(COLLECTION).getReplicas().stream()
             .filter(r -> r.getCoreName().equals(coreName))

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0824f7b0/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
index db558c5b..9b27522 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
@@ -229,6 +229,8 @@ public class TestCloudConsistency extends SolrCloudTestCase {
       return newLeader != null && newLeader.getName().equals(leader.getName());
     });
     waitForState("Timeout waiting for active collection", collection, clusterShape(1, 3));
+    
+    cluster.waitForActiveCollection(collection, 1, 3);
   }
 
   private void addDocs(String collection, int numDocs, int startId) throws SolrServerException, IOException {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0824f7b0/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
index 37080e1..7cb82c5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
@@ -263,8 +263,7 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
     CollectionAdminRequest.createCollection(collection, CONF_NAME, NUM_SERVERS, 1)
         .withProperty("config", "solrconfig-tlog.xml")
         .process(cloudClient);
-    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
-    AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, zkStateReader, true, true, 330);
+    cluster.waitForActiveCollection(collection, NUM_SERVERS, NUM_SERVERS);
     assertEquals("sanity query", 0, cloudClient.query(collection, params("q","*:*")).getStatus());
   }
   

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0824f7b0/solr/core/src/test/org/apache/solr/cloud/api/collections/CustomCollectionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CustomCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CustomCollectionTest.java
index 5a14a23..d4f3783 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CustomCollectionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CustomCollectionTest.java
@@ -161,6 +161,8 @@ public class CustomCollectionTest extends SolrCloudTestCase {
         .setMaxShardsPerNode(maxShardsPerNode)
         .setRouterField(shard_fld)
         .process(cluster.getSolrClient());
+    
+    cluster.waitForActiveCollection(collectionName, numShards, numShards * replicationFactor);
 
     new UpdateRequest()
         .add("id", "6", shard_fld, "a")


[12/47] lucene-solr:jira/http2: SOLR-12801: boost wait to try and harden

Posted by da...@apache.org.
SOLR-12801: boost wait to try and harden


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f6063a14
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f6063a14
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f6063a14

Branch: refs/heads/jira/http2
Commit: f6063a14611da106aae4fe218cd0943c7d61af29
Parents: 4133ce4
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 15:12:42 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f6063a14/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
index 383b3ef..cc3f6ed 100644
--- a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
@@ -114,7 +114,7 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
         target.waitForActiveCollection("cdcr-target", 1, 2);
         CloudSolrClient targetSolrClient = target.getSolrClient();
         targetSolrClient.setDefaultCollection("cdcr-target");
-        Thread.sleep(1000);
+        Thread.sleep(3000);
 
         CdcrTestsUtil.cdcrStart(targetSolrClient);
         CdcrTestsUtil.cdcrStart(sourceSolrClient);


[38/47] lucene-solr:jira/http2: LUCENE-8598: Improve field updates packed values

Posted by da...@apache.org.
LUCENE-8598: Improve field updates packed values

DocValuesFieldUpdats are using compact settings for packet ints that causes
dramatic slowdowns when the updates are finished and sorted. Moving to the default
accepted overhead ratio yields up to 4x improvements in applying updates. This change
also improves the packing of numeric values since we know the value range in advance and
can choose a different packing scheme in such a case.
Overall this change yields a good performance improvement since 99% of the times of applying
DV field updates are spend in the sort method which essentially makes applying the updates
4x faster.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0650f995
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0650f995
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0650f995

Branch: refs/heads/jira/http2
Commit: 0650f995f9fd70a4ff711a7baaabc2add9ec39ea
Parents: 202dad8
Author: Simon Willnauer <si...@apache.org>
Authored: Sun Dec 9 19:13:20 2018 +0100
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Dec 10 18:06:08 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |  5 +++-
 .../lucene/index/DocValuesFieldUpdates.java     |  2 +-
 .../apache/lucene/index/FieldUpdatesBuffer.java | 21 +++++++++++++
 .../lucene/index/FrozenBufferedUpdates.java     |  3 +-
 .../index/NumericDocValuesFieldUpdates.java     | 31 ++++++++++++++------
 .../util/packed/AbstractPagedMutable.java       |  3 +-
 .../lucene/index/TestFieldUpdatesBuffer.java    | 17 ++++++++++-
 7 files changed, 67 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 41259bd..4eaca5f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -266,7 +266,10 @@ Optimizations
 
 * LUCENE-8590: BufferedUpdates now uses an optimized storage for buffering docvalues updates that
   can safe up to 80% of the heap used compared to the previous implementation and uses non-object
-  based datastructures. (Simon Willnauer, Mike McCandless, Shai Erera, Adrien Grant)
+  based datastructures. (Simon Willnauer, Mike McCandless, Shai Erera, Adrien Grand)
+
+* LUCENE-8598: Moved to the default accepted overhead ratio for packet ints in DocValuesFieldUpdats
+  yields an up-to 4x performance improvement when applying doc values updates. (Simon Willnauer, Adrien Grand)
 
 Other
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
index 7c2c5b1..9ab3b7c 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
@@ -251,7 +251,7 @@ abstract class DocValuesFieldUpdates implements Accountable {
     }
     this.type = type;
     bitsPerValue = PackedInts.bitsRequired(maxDoc - 1) + SHIFT;
-    docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
+    docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.DEFAULT);
   }
 
   final boolean getFinished() {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
index 31a91bb..d9db980 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FieldUpdatesBuffer.java
@@ -55,6 +55,8 @@ final class FieldUpdatesBuffer {
   private int[] docsUpTo;
   private long[] numericValues; // this will be null if we are buffering binaries
   private FixedBitSet hasValues;
+  private long maxNumeric = Long.MIN_VALUE;
+  private long minNumeric = Long.MAX_VALUE;
   private String[] fields;
   private final boolean isNumeric;
 
@@ -82,6 +84,7 @@ final class FieldUpdatesBuffer {
     this(bytesUsed, initialValue, docUpTo, true);
     if (initialValue.hasValue()) {
       numericValues = new long[] {initialValue.getValue()};
+      maxNumeric = minNumeric = initialValue.getValue();
     } else {
       numericValues = new long[] {0};
     }
@@ -95,6 +98,22 @@ final class FieldUpdatesBuffer {
     }
   }
 
+  long getMaxNumeric() {
+    assert isNumeric;
+    if (minNumeric == Long.MAX_VALUE && maxNumeric == Long.MIN_VALUE) {
+      return 0; // we don't have any value;
+    }
+    return maxNumeric;
+  }
+
+  long getMinNumeric() {
+    assert isNumeric;
+    if (minNumeric == Long.MAX_VALUE && maxNumeric == Long.MIN_VALUE) {
+      return 0; // we don't have any value
+    }
+    return minNumeric;
+  }
+
   void add(String field, int docUpTo, int ord, boolean hasValue) {
     if (fields[0].equals(field) == false || fields.length != 1 ) {
       if (fields.length <= ord) {
@@ -144,6 +163,8 @@ final class FieldUpdatesBuffer {
     final int ord = append(term);
     String field = term.field;
     add(field, docUpTo, ord, true);
+    minNumeric = Math.min(minNumeric, value);
+    maxNumeric = Math.max(maxNumeric, value);
     if (numericValues[0] != value || numericValues.length != 1) {
       if (numericValues.length <= ord) {
         long[] array = ArrayUtil.grow(numericValues, ord+1);

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
index 94138c7..2ba2cc5 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
@@ -542,7 +542,8 @@ final class FrozenBufferedUpdates {
                     .SingleValueNumericDocValuesFieldUpdates(delGen, updateField, segState.reader.maxDoc(),
                     value.getNumericValue(0));
               } else {
-                dvUpdates = new NumericDocValuesFieldUpdates(delGen, updateField, segState.reader.maxDoc());
+                dvUpdates = new NumericDocValuesFieldUpdates(delGen, updateField, value.getMinNumeric(),
+                    value.getMaxNumeric(), segState.reader.maxDoc());
               }
             } else {
               dvUpdates = new BinaryDocValuesFieldUpdates(delGen, updateField, segState.reader.maxDoc());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
index ebc196b..550a86a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
+++ b/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.AbstractPagedMutable;
 import org.apache.lucene.util.packed.PackedInts;
 import org.apache.lucene.util.packed.PagedGrowableWriter;
 import org.apache.lucene.util.packed.PagedMutable;
@@ -31,15 +32,16 @@ import org.apache.lucene.util.packed.PagedMutable;
  * @lucene.experimental
  */
 final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
-
   // TODO: can't this just be NumericDocValues now?  avoid boxing the long value...
   final static class Iterator extends DocValuesFieldUpdates.AbstractIterator {
-    private final PagedGrowableWriter values;
+    private final AbstractPagedMutable values;
+    private final long minValue;
     private long value;
 
-    Iterator(int size, PagedGrowableWriter values, PagedMutable docs, long delGen) {
+    Iterator(int size, long minValue, AbstractPagedMutable values, PagedMutable docs, long delGen) {
       super(size, docs, delGen);
       this.values = values;
+      this.minValue = minValue;
     }
     @Override
     long longValue() {
@@ -53,14 +55,25 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
 
     @Override
     protected void set(long idx) {
-      value = values.get(idx);
+      value = values.get(idx) + minValue;
     }
   }
-  private PagedGrowableWriter values;
+  private AbstractPagedMutable values;
+  private final long minValue;
+
+  NumericDocValuesFieldUpdates(long delGen, String field, int maxDoc) {
+    super(maxDoc, delGen, field, DocValuesType.NUMERIC);
+    // we don't know the min/max range so we use the growable writer here to adjust as we go.
+    values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.DEFAULT);
+    minValue = 0;
+  }
 
-  public NumericDocValuesFieldUpdates(long delGen, String field, int maxDoc) {
+  NumericDocValuesFieldUpdates(long delGen, String field, long minValue, long maxValue, int maxDoc) {
     super(maxDoc, delGen, field, DocValuesType.NUMERIC);
-    values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
+    assert minValue <= maxValue : "minValue must be <= maxValue [" + minValue + " > " + maxValue + "]";
+    int bitsPerValue = PackedInts.unsignedBitsRequired(maxValue - minValue);
+    values = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.DEFAULT);
+    this.minValue = minValue;
   }
   @Override
   void add(int doc, BytesRef value) {
@@ -75,7 +88,7 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
   @Override
   synchronized void add(int doc, long value) {
     int add = add(doc);
-    values.set(add, value);
+    values.set(add, value-minValue);
   }
 
   @Override
@@ -101,7 +114,7 @@ final class NumericDocValuesFieldUpdates extends DocValuesFieldUpdates {
   @Override
   Iterator iterator() {
     ensureFinished();
-    return new Iterator(size, values, docs, delGen);
+    return new Iterator(size, minValue, values, docs, delGen);
   }
   
   @Override

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java b/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
index e73863b..536f129 100644
--- a/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
+++ b/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
@@ -29,7 +29,7 @@ import org.apache.lucene.util.RamUsageEstimator;
  * Base implementation for {@link PagedMutable} and {@link PagedGrowableWriter}.
  * @lucene.internal
  */
-abstract class AbstractPagedMutable<T extends AbstractPagedMutable<T>> extends LongValues implements Accountable {
+public abstract class AbstractPagedMutable<T extends AbstractPagedMutable<T>> extends LongValues implements Accountable {
 
   static final int MIN_BLOCK_SIZE = 1 << 6;
   static final int MAX_BLOCK_SIZE = 1 << 30;
@@ -161,5 +161,4 @@ abstract class AbstractPagedMutable<T extends AbstractPagedMutable<T>> extends L
   public final String toString() {
     return getClass().getSimpleName() + "(size=" + size() + ",pageSize=" + pageSize() + ")";
   }
-
 }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0650f995/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
index 832c7cc..d6270ee 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
@@ -30,7 +30,7 @@ import org.apache.lucene.util.TestUtil;
 
 public class TestFieldUpdatesBuffer extends LuceneTestCase {
 
-  public void testBascis() throws IOException {
+  public void testBasics() throws IOException {
     Counter counter = Counter.newCounter();
     DocValuesUpdate.NumericDocValuesUpdate update =
         new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", 6);
@@ -44,6 +44,8 @@ public class TestFieldUpdatesBuffer extends LuceneTestCase {
     buffer.addUpdate(new Term("id", "8"), 12, 16);
     assertFalse(buffer.hasSingleValue());
     assertTrue(buffer.isNumeric());
+    assertEquals(13, buffer.getMaxNumeric());
+    assertEquals(6, buffer.getMinNumeric());
     FieldUpdatesBuffer.BufferedUpdateIterator iterator = buffer.iterator();
     FieldUpdatesBuffer.BufferedUpdate value = iterator.next();
     assertNotNull(value);
@@ -217,8 +219,12 @@ public class TestFieldUpdatesBuffer extends LuceneTestCase {
     FieldUpdatesBuffer.BufferedUpdate value;
 
     int count = 0;
+    long min = Long.MAX_VALUE;
+    long max = Long.MIN_VALUE;
     while ((value = iterator.next()) != null) {
       long v = buffer.getNumericValue(count);
+      min = Math.min(min, v);
+      max = Math.max(max, v);
       randomUpdate = updates.get(count++);
       assertEquals(randomUpdate.term.bytes.utf8ToString(), value.termValue.utf8ToString());
       assertEquals(randomUpdate.term.field, value.termField);
@@ -232,7 +238,16 @@ public class TestFieldUpdatesBuffer extends LuceneTestCase {
       }
       assertEquals(randomUpdate.docIDUpto, value.docUpTo);
     }
+    assertEquals(max, buffer.getMaxNumeric());
+    assertEquals(min, buffer.getMinNumeric());
     assertEquals(count, updates.size());
   }
 
+  public void testNoNumericValue() {
+    DocValuesUpdate.NumericDocValuesUpdate update =
+        new DocValuesUpdate.NumericDocValuesUpdate(new Term("id", "1"), "age", null);
+    FieldUpdatesBuffer buffer = new FieldUpdatesBuffer(Counter.newCounter(), update, update.docIDUpto);
+    assertEquals(0, buffer.getMinNumeric());
+    assertEquals(0, buffer.getMaxNumeric());
+  }
 }


[10/47] lucene-solr:jira/http2: SOLR-12727: Upgrade ZooKeeper dependency to 3.4.13

Posted by da...@apache.org.
SOLR-12727: Upgrade ZooKeeper dependency to 3.4.13


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f690302e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f690302e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f690302e

Branch: refs/heads/jira/http2
Commit: f690302e052f2fcad3117ef287296c7b740c1636
Parents: df84a3c
Author: Erick Erickson <Er...@gmail.com>
Authored: Sun Dec 9 08:50:51 2018 -0800
Committer: Erick Erickson <Er...@gmail.com>
Committed: Sun Dec 9 08:50:51 2018 -0800

----------------------------------------------------------------------
 lucene/ivy-versions.properties                                   | 2 +-
 lucene/tools/junit4/solr-tests.policy                            | 4 ++--
 solr/CHANGES.txt                                                 | 1 +
 .../src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java    | 2 +-
 solr/licenses/zookeeper-3.4.11.jar.sha1                          | 1 -
 solr/licenses/zookeeper-3.4.13.jar.sha1                          | 1 +
 .../src/java/org/apache/solr/cloud/ZkTestServer.java             | 4 ++--
 7 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 2cfb65c..5a38cbb 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -206,7 +206,7 @@ org.apache.tika.version = 1.19.1
 /org.apache.velocity/velocity = 1.7
 /org.apache.velocity/velocity-tools = 2.0
 /org.apache.xmlbeans/xmlbeans = 3.0.1
-/org.apache.zookeeper/zookeeper = 3.4.11
+/org.apache.zookeeper/zookeeper = 3.4.13
 
 # v1.6.0-alpha.5 of asciidoctor-ant includes asciidoctorj-pdf 1.5.0-alpha.16,
 # which is the same as asciidoctor-pdf 1.5.0-alpha.16

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/lucene/tools/junit4/solr-tests.policy
----------------------------------------------------------------------
diff --git a/lucene/tools/junit4/solr-tests.policy b/lucene/tools/junit4/solr-tests.policy
index 3333e95..d58b82c 100644
--- a/lucene/tools/junit4/solr-tests.policy
+++ b/lucene/tools/junit4/solr-tests.policy
@@ -85,8 +85,8 @@ grant {
   permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read";
   
   permission javax.security.auth.kerberos.ServicePermission "krbtgt/EXAMPLE.COM@EXAMPLE.COM", "initiate";
-  permission javax.security.auth.kerberos.ServicePermission "zookeeper/127.0.0.1@EXAMPLE.COM", "initiate";
-  permission javax.security.auth.kerberos.ServicePermission "zookeeper/127.0.0.1@EXAMPLE.COM", "accept";
+  permission javax.security.auth.kerberos.ServicePermission "zookeeper/localhost@EXAMPLE.COM", "initiate";
+  permission javax.security.auth.kerberos.ServicePermission "zookeeper/localhost@EXAMPLE.COM", "accept";
   permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "initiate";
   permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "accept";
   permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/127.0.0.1@EXAMPLE.COM\" \"krbtgt/EXAMPLE.COM@EXAMPLE.COM\"";

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7936e57..dc6fc47 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -178,6 +178,7 @@ Other Changes
 
 * SOLR-13036: Fix retry logic in JettySolrRunner (Gus Heck)
 
+* SOLR-12727: Upgrade ZooKeeper dependency to 3.4.13 (Kevin Risden, Erick Erickson)
 
 ==================  7.6.0 ==================
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
index 38e2ab6..9d1f0a3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
@@ -196,7 +196,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 {
         System.setProperty("zookeeper.kerberos.removeHostFromPrincipal", "true");
         File keytabFile = new File(kdcDir, "keytabs");
         String zkClientPrincipal = "solr";
-        String zkServerPrincipal = "zookeeper/127.0.0.1";
+        String zkServerPrincipal = "zookeeper/localhost";
 
         kerberosTestServices = KerberosTestServices.builder()
             .withKdc(new File(kdcDir))

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/solr/licenses/zookeeper-3.4.11.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/zookeeper-3.4.11.jar.sha1 b/solr/licenses/zookeeper-3.4.11.jar.sha1
deleted file mode 100644
index 075b3f4..0000000
--- a/solr/licenses/zookeeper-3.4.11.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dc767a0b76db57b9ed442ff80418f3c078f670b7

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/solr/licenses/zookeeper-3.4.13.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/zookeeper-3.4.13.jar.sha1 b/solr/licenses/zookeeper-3.4.13.jar.sha1
new file mode 100644
index 0000000..a9df3b1
--- /dev/null
+++ b/solr/licenses/zookeeper-3.4.13.jar.sha1
@@ -0,0 +1 @@
+31e9937541cef95c4585b547eb2dbd34d3a76f1c

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f690302e/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
index 85f6afb..1150c48 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
@@ -454,7 +454,7 @@ public class ZkTestServer {
   }
 
   public String getZkHost() {
-    return "127.0.0.1:" + zkServer.getLocalPort();
+    return "localhost:" + zkServer.getLocalPort();
   }
 
   public String getZkAddress() {
@@ -469,7 +469,7 @@ public class ZkTestServer {
   public String getZkAddress(String chroot) {
     if (!chroot.startsWith("/"))
       chroot = "/" + chroot;
-    return "127.0.0.1:" + zkServer.getLocalPort() + chroot;
+    return "localhost:" + zkServer.getLocalPort() + chroot;
   }
 
   /**


[06/47] lucene-solr:jira/http2: Revert "Add the ability to set the number of hits to track accurately"

Posted by da...@apache.org.
Revert "Add the ability to set the number of hits to track accurately"

This reverts commit 84d5015eee00992e3c0c5185e6146ed48fe3dd25.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/da7919f7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/da7919f7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/da7919f7

Branch: refs/heads/jira/http2
Commit: da7919f7a4b6c0cc951ab078b7d21122f0669c78
Parents: 84d5015
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:15:44 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:15:44 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |    9 +-
 .../index/TestBackwardsCompatibility.java       |   27 -
 .../lucene/index/sorted-invalid.7.5.0.zip       |  Bin 468799 -> 0 bytes
 .../org/apache/lucene/index/MultiSorter.java    |   30 +-
 .../apache/lucene/index/TestIndexSorting.java   | 1744 ++++++++----------
 5 files changed, 787 insertions(+), 1023 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 007d35b..14b4961 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -314,14 +314,7 @@ Bug fixes
 
 * LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
   to the same doc in the same updates package looses an update if the reset comes before
-  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
-
-* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
-  can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
-  values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
-  (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
-  detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
-  (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
+  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
 
 New Features
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 44911db..6f28797 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1717,33 +1717,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       dir.close();
     }
   }
-
-  /**
-   * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
-   * before https://issues.apache.org/jira/browse/LUCENE-8592.
-   */
-  public void testSortedIndexWithInvalidSort() throws Exception {
-    Path path = createTempDir("sorted");
-    String name = "sorted-invalid.7.5.0.zip";
-    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
-    assertNotNull("Sorted index index " + name + " not found", resource);
-    TestUtil.unzip(resource, path);
-
-    Directory dir = FSDirectory.open(path);
-
-    DirectoryReader reader = DirectoryReader.open(dir);
-    assertEquals(1, reader.leaves().size());
-    Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
-    assertNotNull(sort);
-    assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
-    reader.close();
-    CheckIndex.Status status = TestUtil.checkIndex(dir);
-    assertEquals(1, status.segmentInfos.size());
-    assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
-    assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
-        "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
-    dir.close();
-  }
   
   static long getValue(BinaryDocValues bdv) throws IOException {
     BytesRef term = bdv.binaryValue();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
deleted file mode 100644
index 30c68a3..0000000
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip and /dev/null differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index d81ac75..b484228 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,18 +42,17 @@ final class MultiSorter {
 
     SortField fields[] = sort.getSort();
     final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
-    final int[] reverseMuls = new int[fields.length];
     for(int i=0;i<fields.length;i++) {
       comparables[i] = getComparableProviders(readers, fields[i]);
-      reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
     }
+
     int leafCount = readers.size();
 
     PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
         @Override
         public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
           for(int i=0;i<comparables.length;i++) {
-            int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
+            int cmp = a.values[i].compareTo(b.values[i]);
             if (cmp != 0) {
               return cmp < 0;
             }
@@ -147,13 +146,14 @@ final class MultiSorter {
 
   /** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
   private interface ComparableProvider {
-    Comparable getComparable(int docID) throws IOException;
+    public Comparable getComparable(int docID) throws IOException;
   }
 
   /** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
   private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
 
     ComparableProvider[] providers = new ComparableProvider[readers.size()];
+    final int reverseMul = sortField.getReverse() ? -1 : 1;
     final SortField.Type sortType = Sorter.getSortFieldType(sortField);
 
     switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
         OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
         final int missingOrd;
         if (sortField.getMissingValue() == SortField.STRING_LAST) {
-          missingOrd = Integer.MAX_VALUE;
+          missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
         } else {
-          missingOrd = Integer.MIN_VALUE;
+          missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
         }
 
         for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
                 }
                 if (readerDocID == docID) {
                   // translate segment's ord to global ord space:
-                  return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
+                  return reverseMul * (int) globalOrds.get(readerValues.ordValue());
                 } else {
                   return missingOrd;
                 }
@@ -238,9 +238,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return values.longValue();
+                  return reverseMul * values.longValue();
                 } else {
-                  return missingValue;
+                  return reverseMul * missingValue;
                 }
               }
             };
@@ -279,9 +279,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return (int) values.longValue();
+                  return reverseMul * (int) values.longValue();
                 } else {
-                  return missingValue;
+                  return reverseMul * missingValue;
                 }
               }
             };
@@ -320,9 +320,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return Double.longBitsToDouble(values.longValue());
+                  return reverseMul * Double.longBitsToDouble(values.longValue());
                 } else {
-                  return missingValue;
+                  return reverseMul * missingValue;
                 }
               }
             };
@@ -361,9 +361,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return Float.intBitsToFloat((int) values.longValue());
+                  return reverseMul * Float.intBitsToFloat((int) values.longValue());
                 } else {
-                  return missingValue;
+                  return reverseMul * missingValue;
                 }
               }
             };


[23/47] lucene-solr:jira/http2: SOLR-12801: harden DocValuesNotIndexedTest

Posted by da...@apache.org.
SOLR-12801: harden DocValuesNotIndexedTest


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5d529802
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5d529802
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5d529802

Branch: refs/heads/jira/http2
Commit: 5d529802c45cc5651ad1c09aafbff00e35ddecd6
Parents: 0824f7b
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 17:10:16 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../solr/cloud/DocValuesNotIndexedTest.java     | 29 ++++++++------------
 1 file changed, 12 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5d529802/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
index 8e5482e..f396a5d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
@@ -71,10 +71,10 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
   static final String COLLECTION = "dv_coll";
 
 
-  static List<FieldProps> fieldsToTestSingle = null;
-  static List<FieldProps> fieldsToTestMulti = null;
-  static List<FieldProps> fieldsToTestGroupSortFirst = null;
-  static List<FieldProps> fieldsToTestGroupSortLast = null;
+  volatile static List<FieldProps> fieldsToTestSingle = null;
+  volatile static List<FieldProps> fieldsToTestMulti = null;
+  volatile static List<FieldProps> fieldsToTestGroupSortFirst = null;
+  volatile static List<FieldProps> fieldsToTestGroupSortLast = null;
 
   @Before
   public void createCluster() throws Exception {
@@ -168,20 +168,12 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
   public void after() throws Exception {
     shutdownCluster();
     
-    resetFieldBases(fieldsToTestSingle);
-    resetFieldBases(fieldsToTestMulti);
-    resetFieldBases(fieldsToTestGroupSortFirst);
-    resetFieldBases(fieldsToTestGroupSortLast);
+    fieldsToTestSingle = null;
+    fieldsToTestMulti = null;
+    fieldsToTestGroupSortFirst = null;
+    fieldsToTestGroupSortLast = null;
   }
 
-  private void resetFieldBases(List<FieldProps> props) {
-    // OK, it's not bad with the int and string fields, but every time a new test counts on docs being
-    // indexed so they sort in a particular order, then particularly the boolean and string fields need to be
-    // reset to a known state.
-    for (FieldProps prop : props) {
-      prop.resetBase();
-    }
-  }
   @Test
   public void testDistribFaceting() throws IOException, SolrServerException {
     // For this test, I want to insure that there are shards that do _not_ have a doc with any of the DV_only 
@@ -229,7 +221,7 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
 
   // We should be able to sort thing with missing first/last and that are _NOT_ present at all on one server.
   @Test
-  // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
+  @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12028")
   public void testGroupingSorting() throws IOException, SolrServerException {
     CloudSolrClient client = cluster.getSolrClient();
 
@@ -349,6 +341,9 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
       // Special handling until SOLR-9802 is fixed
       if (prop.getName().startsWith("date")) continue;
       // SOLR-9802 to here
+      
+      // TODO: gsf fails this
+      if (prop.getName().endsWith("GSF") ) continue;
 
       final SolrQuery solrQuery = new SolrQuery(
           "q", "*:*",


[20/47] lucene-solr:jira/http2: SOLR-12923: Harden TestSimNodeLostTrigger#testTrigger.

Posted by da...@apache.org.
SOLR-12923: Harden TestSimNodeLostTrigger#testTrigger.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0aef5612
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0aef5612
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0aef5612

Branch: refs/heads/jira/http2
Commit: 0aef56124529f866da56eec2cc3ade2bf702fbaf
Parents: 3f1b75a
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 11:59:41 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0aef5612/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java
index c1c5f4c..0a9bd19 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimNodeLostTrigger.java
@@ -109,7 +109,7 @@ public class TestSimNodeLostTrigger extends SimSolrCloudTestCase {
       do {
         trigger.run();
         timeSource.sleep(1000);
-        if (counter++ > 10) {
+        if (counter++ > 20) {
           fail("Lost node was not discovered by trigger even after 10 seconds");
         }
       } while (!fired.get());


[40/47] lucene-solr:jira/http2: SOLR-12801: Fix TestSolrCloudWithKerberosAlt.

Posted by da...@apache.org.
SOLR-12801: Fix TestSolrCloudWithKerberosAlt.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d96475a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d96475a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d96475a0

Branch: refs/heads/jira/http2
Commit: d96475a0bb9e41b285c273e1b70d984208d6fd3a
Parents: b6cbb84
Author: markrmiller <ma...@apache.org>
Authored: Mon Dec 10 11:46:27 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Dec 10 11:46:34 2018 -0600

----------------------------------------------------------------------
 .../core/src/test/org/apache/solr/cloud/KerberosTestServices.java | 3 +--
 .../test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java  | 3 +++
 2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d96475a0/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java b/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java
index 4ec5640..cccf590 100644
--- a/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java
+++ b/solr/core/src/test/org/apache/solr/cloud/KerberosTestServices.java
@@ -64,7 +64,6 @@ public class KerberosTestServices {
       Locale.setDefault(Locale.US);
     }
 
-    File dir = null;
     // There is time lag between selecting a port and trying to bind with it. It's possible that
     // another service captures the port in between which'll result in BindException.
     boolean bindException;
@@ -76,7 +75,7 @@ public class KerberosTestServices {
         kdc = getKdc(workDir);
         kdc.start();
       } catch (BindException e) {
-        FileUtils.deleteDirectory(dir); // clean directory
+        FileUtils.deleteDirectory(workDir); // clean directory
         numTries++;
         if (numTries == 3) {
           log.error("Failed setting up MiniKDC. Tried " + numTries + " times.");

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d96475a0/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
index 9d56204..0978e57 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
@@ -21,6 +21,8 @@ import java.lang.invoke.MethodHandles;
 import java.nio.charset.StandardCharsets;
 
 import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.lucene.util.Constants;
 import org.apache.lucene.util.LuceneTestCase;
@@ -46,6 +48,7 @@ import org.slf4j.LoggerFactory;
 })
 
 @LuceneTestCase.Slow
+@ThreadLeakLingering(linger = 10000) // minikdc has some lingering threads
 public class TestSolrCloudWithKerberosAlt extends SolrCloudTestCase {
 
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());


[45/47] lucene-solr:jira/http2: SOLR-13050: add workaround for issue to SystemLogListenerTest

Posted by da...@apache.org.
SOLR-13050: add workaround for issue to SystemLogListenerTest

make sure the node we kill isn't the .system collection leader


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a2199c72
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a2199c72
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a2199c72

Branch: refs/heads/jira/http2
Commit: a2199c72d40c8aaf55dd9ca20816c2aa1ee805ea
Parents: f89f109
Author: Chris Hostetter <ho...@apache.org>
Authored: Mon Dec 10 19:06:42 2018 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Mon Dec 10 19:06:42 2018 -0700

----------------------------------------------------------------------
 .../autoscaling/SystemLogListenerTest.java      | 46 +++++++++++++-------
 1 file changed, 30 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a2199c72/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
index 040a26f..6a84a6f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
@@ -38,6 +38,7 @@ import org.apache.solr.client.solrj.response.QueryResponse;
 import org.apache.solr.cloud.SolrCloudTestCase;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
 import org.apache.solr.common.params.ModifiableSolrParams;
@@ -148,25 +149,16 @@ public class SystemLogListenerTest extends SolrCloudTestCase {
     response = solrClient.request(req);
     assertEquals(response.get("result").toString(), "success");
 
-    // stop non-overseer node
-    NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
-    String overseerLeader = (String) overSeerStatus.get("leader");
-    int nonOverseerLeaderIndex = 0;
-    for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
-      JettySolrRunner jetty = cluster.getJettySolrRunner(i);
-      if (!jetty.getNodeName().equals(overseerLeader)) {
-        nonOverseerLeaderIndex = i;
-      }
-    }
-    log.info("Stopping node " + cluster.getJettySolrRunner(nonOverseerLeaderIndex).getNodeName());
-    JettySolrRunner j = cluster.stopJettySolrRunner(nonOverseerLeaderIndex);
-    cluster.waitForJettyToStop(j);
+    // Stop a node (that's safe to stop for the purposes of this test)
+    final JettySolrRunner stoppedJetty = pickNodeToStop();
+    log.info("Stopping node " + stoppedJetty.getNodeName());
+    cluster.stopJettySolrRunner(stoppedJetty);
+    cluster.waitForJettyToStop(stoppedJetty);
+    
     assertTrue("Trigger was not fired ", triggerFiredLatch.await(60, TimeUnit.SECONDS));
     assertTrue(fired.get());
     Map context = actionContextPropsRef.get();
     assertNotNull(context);
-
-    
     
     TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
     
@@ -199,7 +191,8 @@ public class SystemLogListenerTest extends SolrCloudTestCase {
     QueryResponse resp = cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL, query);
     SolrDocumentList docs = resp.getResults();
     assertNotNull(docs);
-    assertEquals("wrong number of events added to .system", 9, docs.size());
+    assertEquals("wrong number of events added to .system: " + docs.toString(),
+                 9, docs.size());
     docs.forEach(doc -> assertCommonFields(doc));
 
     // STARTED
@@ -273,4 +266,25 @@ public class SystemLogListenerTest extends SolrCloudTestCase {
     assertNotNull(doc.getFieldValue("event_str"));
     assertEquals("NODELOST", doc.getFieldValue("event.type_s"));
   }
+
+  /** 
+   * Helper method for picking a node that can safely be stoped
+   * @see <a href="https://issues.apache.org/jira/browse/SOLR-13050">SOLR-13050</a>
+   */
+  private JettySolrRunner pickNodeToStop() throws Exception {
+    // first get the nodeName of the overser.
+    // stopping the overseer is not something we want to hassle with in this test
+    final String overseerNodeName = (String) cluster.getSolrClient().request
+      (CollectionAdminRequest.getOverseerStatus()).get("leader");
+
+    // now find a node that is *NOT* the overseer or the leader of a .system collection shard
+    for (Replica r :  getCollectionState(CollectionAdminParams.SYSTEM_COLL).getReplicas()) {
+      if ( ! (r.getBool("leader", false) || r.getNodeName().equals(overseerNodeName) ) ) {
+        return cluster.getReplicaJetty(r);
+      }
+    }
+    fail("Couldn't find non-leader, non-overseer, replica of .system collection to kill");
+    return null;
+  }
+  
 }


[13/47] lucene-solr:jira/http2: SOLR-12801: harden MultiThreadedOCPTest

Posted by da...@apache.org.
SOLR-12801: harden MultiThreadedOCPTest


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3f1b75ab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3f1b75ab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3f1b75ab

Branch: refs/heads/jira/http2
Commit: 3f1b75ab82b1924ab124f0e19be4603a7975d70b
Parents: a9a5cdc
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 11:17:24 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f1b75ab/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
index d69b47b..a688897 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
@@ -75,7 +75,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
       DistributedQueue distributedQueue = new ZkDistributedQueue(cloudClient.getZkStateReader().getZkClient(),
           "/overseer/collection-queue-work", new Stats());
       //fill the work queue with blocked tasks by adding more than the no:of parallel tasks
-      for (int i = 0; i < MAX_PARALLEL_TASKS+5; i++) {
+      for (int i = 0; i < MAX_PARALLEL_TASKS + 15; i++) {
         distributedQueue.offer(Utils.toJSON(Utils.makeMap(
             "collection", "A_COLL",
             QUEUE_OPERATION, MOCK_COLL_TASK.toLower(),
@@ -87,7 +87,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
         log.info("MOCK task added {}", i);
 
       }
-      Thread.sleep(10);//wait and post the next message
+      Thread.sleep(100);//wait and post the next message
 
       //this is not going to be blocked because it operates on another collection
       distributedQueue.offer(Utils.toJSON(Utils.makeMap(
@@ -99,7 +99,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
 
 
       Long acoll = null, bcoll = null;
-      for (int i = 0; i < 100; i++) {
+      for (int i = 0; i < 500; i++) {
         if (bcoll == null) {
           CollectionAdminResponse statusResponse = getStatusResponse("200", client);
           bcoll = (Long) statusResponse.getResponse().get("MOCK_FINISHED");
@@ -112,7 +112,7 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
         Thread.sleep(100);
       }
       assertTrue(acoll != null && bcoll != null);
-      assertTrue(acoll > bcoll);
+      assertTrue("acoll: " + acoll + " bcoll: " + bcoll, acoll > bcoll);
     }
 
   }


[16/47] lucene-solr:jira/http2: SOLR-12801: raise jetty idle socket timeout

Posted by da...@apache.org.
SOLR-12801: raise jetty idle socket timeout


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a9a5cdc3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a9a5cdc3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a9a5cdc3

Branch: refs/heads/jira/http2
Commit: a9a5cdc34dc2333455401510b9518767534c56b8
Parents: 40432cd
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 10:29:18 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../org/apache/solr/client/solrj/embedded/JettySolrRunner.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a9a5cdc3/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index 24caec8..32a9bcc 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -85,7 +85,7 @@ public class JettySolrRunner {
 
   private static final int THREAD_POOL_MAX_THREADS = 10000;
   // NOTE: needs to be larger than SolrHttpClient.threadPoolSweeperMaxIdleTime
-  private static final int THREAD_POOL_MAX_IDLE_TIME_MS = 160000;
+  private static final int THREAD_POOL_MAX_IDLE_TIME_MS = 260000;
   
   Server server;
 


[33/47] lucene-solr:jira/http2: LUCENE-8597: Add IntervalIterator.gaps() and Intervals.maxgaps()

Posted by da...@apache.org.
LUCENE-8597: Add IntervalIterator.gaps() and Intervals.maxgaps()


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1305501d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1305501d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1305501d

Branch: refs/heads/jira/http2
Commit: 1305501dbd5bf79811eab9da873cf1567605e6a5
Parents: 18356de
Author: Alan Woodward <ro...@apache.org>
Authored: Mon Dec 10 09:47:29 2018 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Mon Dec 10 09:47:29 2018 +0000

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |   4 +
 .../intervals/DifferenceIntervalFunction.java   |  10 ++
 .../intervals/DisjunctionIntervalsSource.java   |  15 +++
 .../intervals/FilteredIntervalsSource.java      | 103 +++++++++++++++++++
 .../lucene/search/intervals/IntervalFilter.java |   5 +
 .../search/intervals/IntervalFunction.java      |  54 +++++++++-
 .../search/intervals/IntervalIterator.java      |  11 ++
 .../search/intervals/IntervalMatches.java       |   9 ++
 .../intervals/IntervalMatchesIterator.java      |  38 +++++++
 .../lucene/search/intervals/Intervals.java      |  23 ++++-
 .../intervals/LowpassIntervalsSource.java       |  90 ----------------
 .../MinimizingConjunctionIntervalsSource.java   |   7 +-
 .../search/intervals/TermIntervalsSource.java   |   5 +
 .../lucene/search/intervals/package-info.java   |   2 +
 .../search/intervals/TestIntervalQuery.java     |  10 ++
 .../lucene/search/intervals/TestIntervals.java  |  60 ++++++++++-
 16 files changed, 349 insertions(+), 97 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 007d35b..41259bd 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -98,6 +98,10 @@ API Changes
   methods.  This decouples normalization from tokenization entirely.
   (Mayya Sharipova, Alan Woodward)
 
+* LUCENE-8597: IntervalIterator now exposes a gaps() method that reports the
+  number of gaps between its component sub-intervals.  This can be used in a 
+  new filter available via Intervals.maxgaps().  (Alan Woodward)
+
 Changes in Runtime Behavior
 
 * LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java
index 18d4d67..def1d03 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DifferenceIntervalFunction.java
@@ -126,6 +126,11 @@ abstract class DifferenceIntervalFunction {
     }
 
     @Override
+    public int gaps() {
+      return a.gaps();
+    }
+
+    @Override
     public float matchCost() {
       return a.matchCost() + b.matchCost();
     }
@@ -233,6 +238,11 @@ abstract class DifferenceIntervalFunction {
         }
 
         @Override
+        public int gaps() {
+          throw new UnsupportedOperationException();
+        }
+
+        @Override
         public int nextInterval() throws IOException {
           if (positioned == false) {
             positioned = true;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DisjunctionIntervalsSource.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DisjunctionIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DisjunctionIntervalsSource.java
index c54f18b..68a9e5d 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DisjunctionIntervalsSource.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/DisjunctionIntervalsSource.java
@@ -135,6 +135,11 @@ class DisjunctionIntervalsSource extends IntervalsSource {
       return current.end();
     }
 
+    @Override
+    public int gaps() {
+      return current.gaps();
+    }
+
     private void reset() throws IOException {
       intervalQueue.clear();
       for (DisiWrapper dw = disiQueue.topList(); dw != null; dw = dw.next) {
@@ -229,6 +234,11 @@ class DisjunctionIntervalsSource extends IntervalsSource {
     }
 
     @Override
+    public int gaps() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
     public int nextInterval() {
       return NO_MORE_INTERVALS;
     }
@@ -272,6 +282,11 @@ class DisjunctionIntervalsSource extends IntervalsSource {
     }
 
     @Override
+    public int gaps() {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
     public int nextInterval() {
       return NO_MORE_INTERVALS;
     }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/FilteredIntervalsSource.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/FilteredIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/FilteredIntervalsSource.java
new file mode 100644
index 0000000..8eac88d
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/FilteredIntervalsSource.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.intervals;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.MatchesIterator;
+
+/**
+ * An IntervalsSource that filters the intervals from another IntervalsSource
+ */
+public abstract class FilteredIntervalsSource extends IntervalsSource {
+
+  private final String name;
+  private final IntervalsSource in;
+
+  /**
+   * Create a new FilteredIntervalsSource
+   * @param name  the name of the filter
+   * @param in    the source to filter
+   */
+  public FilteredIntervalsSource(String name, IntervalsSource in) {
+    this.name = name;
+    this.in = in;
+  }
+
+  /**
+   * @return {@code false} if the current interval should be filtered out
+   */
+  protected abstract boolean accept(IntervalIterator it);
+
+  @Override
+  public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
+    IntervalIterator i = in.intervals(field, ctx);
+    if (i == null) {
+      return null;
+    }
+    return new IntervalFilter(i) {
+      @Override
+      protected boolean accept() {
+        return FilteredIntervalsSource.this.accept(in);
+      }
+    };
+  }
+
+  @Override
+  public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
+    MatchesIterator mi = in.matches(field, ctx, doc);
+    if (mi == null) {
+      return null;
+    }
+    IntervalIterator filtered = new IntervalFilter(IntervalMatches.wrapMatches(mi, doc)) {
+      @Override
+      protected boolean accept() {
+        return FilteredIntervalsSource.this.accept(in);
+      }
+    };
+    return IntervalMatches.asMatches(filtered, mi, doc);
+  }
+
+  @Override
+  public void extractTerms(String field, Set<Term> terms) {
+    in.extractTerms(field, terms);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    FilteredIntervalsSource that = (FilteredIntervalsSource) o;
+    return Objects.equals(name, that.name) &&
+        Objects.equals(in, that.in);
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(name, in);
+  }
+
+  @Override
+  public String toString() {
+    return name + "(" + in + ")";
+  }
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFilter.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFilter.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFilter.java
index a501b49..46c0302 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFilter.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFilter.java
@@ -65,6 +65,11 @@ public abstract class IntervalFilter extends IntervalIterator {
   }
 
   @Override
+  public int gaps() {
+    return in.gaps();
+  }
+
+  @Override
   public float matchCost() {
     return in.matchCost();
   }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java
index 787574e..1a5eab6 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalFunction.java
@@ -18,6 +18,7 @@
 package org.apache.lucene.search.intervals;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.lucene.util.PriorityQueue;
@@ -67,6 +68,11 @@ abstract class IntervalFunction {
     }
 
     @Override
+    public int gaps() {
+      return 0;
+    }
+
+    @Override
     public int nextInterval() throws IOException {
       if (subIterators.get(0).nextInterval() == IntervalIterator.NO_MORE_INTERVALS)
         return IntervalIterator.NO_MORE_INTERVALS;
@@ -109,6 +115,7 @@ abstract class IntervalFunction {
   private static class OrderedIntervalIterator extends ConjunctionIntervalIterator {
 
     int start = -1, end = -1, i;
+    int firstEnd;
 
     private OrderedIntervalIterator(List<IntervalIterator> subIntervals) {
       super(subIntervals);
@@ -143,6 +150,7 @@ abstract class IntervalFunction {
           i++;
         }
         start = subIterators.get(0).start();
+        firstEnd = subIterators.get(0).end();
         end = subIterators.get(subIterators.size() - 1).end();
         b = subIterators.get(subIterators.size() - 1).start();
         i = 1;
@@ -152,10 +160,19 @@ abstract class IntervalFunction {
     }
 
     @Override
+    public int gaps() {
+      int gaps = subIterators.get(1).start() - firstEnd - 1;
+      for (int i = 2; i < subIterators.size(); i++) {
+        gaps += (subIterators.get(i).start() - subIterators.get(i - 1).end() - 1);
+      }
+      return gaps;
+    }
+
+    @Override
     protected void reset() throws IOException {
       subIterators.get(0).nextInterval();
       i = 1;
-      start = end = -1;
+      start = end = firstEnd = -1;
     }
   }
 
@@ -183,9 +200,10 @@ abstract class IntervalFunction {
 
     private final PriorityQueue<IntervalIterator> queue;
     private final IntervalIterator[] subIterators;
+    private final int[] innerPositions;
     private final boolean allowOverlaps;
 
-    int start = -1, end = -1, queueEnd;
+    int start = -1, end = -1, firstEnd, queueEnd;
 
     UnorderedIntervalIterator(List<IntervalIterator> subIterators, boolean allowOverlaps) {
       super(subIterators);
@@ -196,6 +214,7 @@ abstract class IntervalFunction {
         }
       };
       this.subIterators = new IntervalIterator[subIterators.size()];
+      this.innerPositions = new int[subIterators.size() * 2];
       this.allowOverlaps = allowOverlaps;
 
       for (int i = 0; i < subIterators.size(); i++) {
@@ -241,6 +260,7 @@ abstract class IntervalFunction {
       // then, minimize it
       do {
         start = queue.top().start();
+        firstEnd = queue.top().end();
         end = queueEnd;
         if (queue.top().end() == end)
           return start;
@@ -261,6 +281,26 @@ abstract class IntervalFunction {
     }
 
     @Override
+    public int gaps() {
+      for (int i = 0; i < subIterators.length; i++) {
+        if (subIterators[i].end() > end) {
+          innerPositions[i * 2] = start;
+          innerPositions[i * 2 + 1] = firstEnd;
+        }
+        else {
+          innerPositions[i * 2] = subIterators[i].start();
+          innerPositions[i * 2 + 1] = subIterators[i].end();
+        }
+      }
+      Arrays.sort(innerPositions);
+      int gaps = 0;
+      for (int i = 1; i < subIterators.length; i++) {
+        gaps += (innerPositions[i * 2] - innerPositions[i * 2 - 1] - 1);
+      }
+      return gaps;
+    }
+
+    @Override
     protected void reset() throws IOException {
       queueEnd = start = end = -1;
       this.queue.clear();
@@ -325,6 +365,11 @@ abstract class IntervalFunction {
         }
 
         @Override
+        public int gaps() {
+          return a.gaps();
+        }
+
+        @Override
         public int nextInterval() throws IOException {
           if (bpos == false)
             return IntervalIterator.NO_MORE_INTERVALS;
@@ -372,6 +417,11 @@ abstract class IntervalFunction {
         }
 
         @Override
+        public int gaps() {
+          return a.gaps();
+        }
+
+        @Override
         public int nextInterval() throws IOException {
           if (bpos == false)
             return IntervalIterator.NO_MORE_INTERVALS;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java
index 613cb78..f819aab 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalIterator.java
@@ -60,6 +60,17 @@ public abstract class IntervalIterator extends DocIdSetIterator {
   public abstract int end();
 
   /**
+   * The number of gaps within the current interval
+   *
+   * Note that this returns the number of gaps between the immediate sub-intervals
+   * of this interval, and does not include the gaps inside those sub-intervals.
+   *
+   * Should not be called before {@link #nextInterval()}, or after it has returned
+   * {@link #NO_MORE_INTERVALS}
+   */
+  public abstract int gaps();
+
+  /**
    * Advance the iterator to the next interval
    *
    * @return the start of the next interval, or {@link IntervalIterator#NO_MORE_INTERVALS} if

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java
index 89f115d..a28f6e4 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatches.java
@@ -99,6 +99,15 @@ final class IntervalMatches {
       }
 
       @Override
+      public int gaps() {
+        assert state == State.ITERATING;
+        if (mi instanceof IntervalMatchesIterator) {
+          return ((IntervalMatchesIterator)mi).gaps();
+        }
+        return 0;
+      }
+
+      @Override
       public int nextInterval() throws IOException {
         assert state == State.ITERATING;
         if (mi.next()) {

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatchesIterator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatchesIterator.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatchesIterator.java
new file mode 100644
index 0000000..55482e3
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/IntervalMatchesIterator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search.intervals;
+
+import org.apache.lucene.search.MatchesIterator;
+
+/**
+ * An extension of MatchesIterator that allows the gaps from a wrapped
+ * IntervalIterator to be reported.
+ *
+ * This is necessary because {@link MatchesIterator#getSubMatches()} returns
+ * the submatches of all nested matches as a flat iterator, but
+ * {@link IntervalIterator#gaps()} only returns the gaps between its immediate
+ * sub-matches, so we can't calculate the latter using the former.
+ */
+interface IntervalMatchesIterator extends MatchesIterator {
+
+  /**
+   * The number of top-level gaps inside the current match
+   */
+  int gaps();
+
+}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java
index e9f1bd2..b0a4829 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/Intervals.java
@@ -77,11 +77,30 @@ public final class Intervals {
 
   /**
    * Create an {@link IntervalsSource} that filters a sub-source by the width of its intervals
-   * @param width       the maximum width of intervals in the sub-source ot return
+   * @param width       the maximum width of intervals in the sub-source to filter
    * @param subSource   the sub-source to filter
    */
   public static IntervalsSource maxwidth(int width, IntervalsSource subSource) {
-    return new LowpassIntervalsSource(subSource, width);
+    return new FilteredIntervalsSource("MAXWIDTH/" + width, subSource) {
+      @Override
+      protected boolean accept(IntervalIterator it) {
+        return (it.end() - it.start()) + 1 <= width;
+      }
+    };
+  }
+
+  /**
+   * Create an {@link IntervalsSource} that filters a sub-source by its gaps
+   * @param gaps        the maximum number of gaps in the sub-source to filter
+   * @param subSource   the sub-source to filter
+   */
+  public static IntervalsSource maxgaps(int gaps, IntervalsSource subSource) {
+    return new FilteredIntervalsSource("MAXGAPS/" + gaps, subSource) {
+      @Override
+      protected boolean accept(IntervalIterator it) {
+        return it.gaps() <= gaps;
+      }
+    };
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/LowpassIntervalsSource.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/LowpassIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/LowpassIntervalsSource.java
deleted file mode 100644
index 61cb1fc..0000000
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/LowpassIntervalsSource.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.search.intervals;
-
-import java.io.IOException;
-import java.util.Objects;
-import java.util.Set;
-
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.search.MatchesIterator;
-
-class LowpassIntervalsSource extends IntervalsSource {
-
-  final IntervalsSource in;
-  private final int maxWidth;
-
-  LowpassIntervalsSource(IntervalsSource in, int maxWidth) {
-    this.in = in;
-    this.maxWidth = maxWidth;
-  }
-
-  @Override
-  public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
-    LowpassIntervalsSource that = (LowpassIntervalsSource) o;
-    return maxWidth == that.maxWidth &&
-        Objects.equals(in, that.in);
-  }
-
-  @Override
-  public String toString() {
-    return "MAXWIDTH/" + maxWidth + "(" + in + ")";
-  }
-
-  @Override
-  public void extractTerms(String field, Set<Term> terms) {
-    in.extractTerms(field, terms);
-  }
-
-  @Override
-  public IntervalIterator intervals(String field, LeafReaderContext ctx) throws IOException {
-    IntervalIterator i = in.intervals(field, ctx);
-    if (i == null) {
-      return null;
-    }
-    return new IntervalFilter(i) {
-      @Override
-      protected boolean accept() {
-        return (i.end() - i.start()) + 1 <= maxWidth;
-      }
-    };
-  }
-
-  @Override
-  public MatchesIterator matches(String field, LeafReaderContext ctx, int doc) throws IOException {
-    MatchesIterator mi = in.matches(field, ctx, doc);
-    if (mi == null) {
-      return null;
-    }
-    IntervalIterator filtered = new IntervalFilter(IntervalMatches.wrapMatches(mi, doc)) {
-      @Override
-      protected boolean accept() {
-        return (this.in.end() - this.in.start()) + 1 <= maxWidth;
-      }
-    };
-    return IntervalMatches.asMatches(filtered, mi, doc);
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(in, maxWidth);
-  }
-}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/MinimizingConjunctionIntervalsSource.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/MinimizingConjunctionIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/MinimizingConjunctionIntervalsSource.java
index 669acae..c509692 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/MinimizingConjunctionIntervalsSource.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/MinimizingConjunctionIntervalsSource.java
@@ -61,7 +61,7 @@ class MinimizingConjunctionIntervalsSource extends ConjunctionIntervalsSource {
     return new ConjunctionMatchesIterator(it, subs);
   }
 
-  private static class ConjunctionMatchesIterator implements MatchesIterator {
+  private static class ConjunctionMatchesIterator implements IntervalMatchesIterator {
 
     final IntervalIterator iterator;
     final List<CacheingMatchesIterator> subs;
@@ -112,6 +112,11 @@ class MinimizingConjunctionIntervalsSource extends ConjunctionIntervalsSource {
     }
 
     @Override
+    public int gaps() {
+      return iterator.gaps();
+    }
+
+    @Override
     public MatchesIterator getSubMatches() throws IOException {
       List<MatchesIterator> mis = new ArrayList<>();
       int endPos = endPosition();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/TermIntervalsSource.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/TermIntervalsSource.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/TermIntervalsSource.java
index 27e5b79..f58d867 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/TermIntervalsSource.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/TermIntervalsSource.java
@@ -96,6 +96,11 @@ class TermIntervalsSource extends IntervalsSource {
       }
 
       @Override
+      public int gaps() {
+        return 0;
+      }
+
+      @Override
       public int nextInterval() throws IOException {
         if (upto <= 0)
           return pos = NO_MORE_INTERVALS;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/java/org/apache/lucene/search/intervals/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/package-info.java b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/package-info.java
index 88d93ea..cfcd9e7 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/search/intervals/package-info.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/search/intervals/package-info.java
@@ -44,6 +44,8 @@
  * <ul>
  *   <li>{@link org.apache.lucene.search.intervals.Intervals#maxwidth(int, org.apache.lucene.search.intervals.IntervalsSource)}
  *          &mdash; Filters out intervals that are larger than a set width</li>
+ *   <li>{@link org.apache.lucene.search.intervals.Intervals#maxgaps(int, org.apache.lucene.search.intervals.IntervalsSource)}
+ *          &mdash; Filters out intervals that have more than a set number of gaps between their constituent sub-intervals</li>
  *   <li>{@link org.apache.lucene.search.intervals.Intervals#containedBy(org.apache.lucene.search.intervals.IntervalsSource, org.apache.lucene.search.intervals.IntervalsSource)}
  *          &mdash; Returns intervals that are contained by another interval</li>
  *   <li>{@link org.apache.lucene.search.intervals.Intervals#notContainedBy(org.apache.lucene.search.intervals.IntervalsSource, org.apache.lucene.search.intervals.IntervalsSource)}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java
index 8f0623e..18c69a7 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervalQuery.java
@@ -91,6 +91,16 @@ public class TestIntervalQuery extends LuceneTestCase {
         new int[]{0, 1, 2, 3, 5});
   }
 
+  public void testOrderedNearQueryGaps1() throws IOException {
+    checkHits(new IntervalQuery(field, Intervals.maxgaps(1, Intervals.ordered(Intervals.term("w1"), Intervals.term("w2")))),
+        new int[]{0, 1, 2, 5});
+  }
+
+  public void testOrderedNearQueryGaps2() throws IOException {
+    checkHits(new IntervalQuery(field, Intervals.maxgaps(2, Intervals.ordered(Intervals.term("w1"), Intervals.term("w2")))),
+        new int[]{0, 1, 2, 3, 5});
+  }
+
   public void testNestedOrderedNearQuery() throws IOException {
     // onear/1(w1, onear/2(w2, w3))
     Query q = new IntervalQuery(field,

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1305501d/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java
index 6002b3a..0dee62f 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/search/intervals/TestIntervals.java
@@ -64,7 +64,7 @@ public class TestIntervals extends LuceneTestCase {
       "Down to a sunless sea",
       "So thrice five miles of fertile ground",
       "Pease hot porridge porridge",
-      "Pease porridge porridge hot"
+      "w1 w2 w3 w4 w1 w6 w3 w8 w4 w7 w1 w6"
   };
 
   private static Directory directory;
@@ -84,7 +84,7 @@ public class TestIntervals extends LuceneTestCase {
     for (int i = 0; i < field1_docs.length; i++) {
       Document doc = new Document();
       doc.add(new Field("field1", field1_docs[i], FIELD_TYPE));
-      doc.add(new TextField("field2", field2_docs[i], Field.Store.NO));
+      doc.add(new Field("field2", field2_docs[i], FIELD_TYPE));
       doc.add(new StringField("id", Integer.toString(i), Field.Store.NO));
       doc.add(new NumericDocValuesField("id", i));
       writer.addDocument(doc);
@@ -149,6 +149,19 @@ public class TestIntervals extends LuceneTestCase {
     assertEquals(endOffset, mi.endOffset());
   }
 
+  private void assertGaps(IntervalsSource source, int doc, String field, int[] expectedGaps) throws IOException {
+    int ord = ReaderUtil.subIndex(doc, searcher.getIndexReader().leaves());
+    LeafReaderContext ctx = searcher.getIndexReader().leaves().get(ord);
+    IntervalIterator it = source.intervals(field, ctx);
+    assertEquals(doc, it.advance(doc));
+    for (int expectedGap : expectedGaps) {
+      if (it.nextInterval() == IntervalIterator.NO_MORE_INTERVALS) {
+        fail("Unexpected interval " + it);
+      }
+      assertEquals(expectedGap, it.gaps());
+    }
+  }
+
   public void testIntervalsOnFieldWithNoPositions() throws IOException {
     IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
       Intervals.term("wibble").intervals("id", searcher.getIndexReader().leaves().get(0));
@@ -241,6 +254,10 @@ public class TestIntervals extends LuceneTestCase {
     assertMatch(sub, 17, 17, 96, 99);
     assertFalse(sub.next());
     assertFalse(mi.next());
+
+    assertGaps(source, 1, "field1", new int[]{
+        1, 0, 10
+    });
   }
 
   public void testIntervalDisjunction() throws IOException {
@@ -287,6 +304,7 @@ public class TestIntervals extends LuceneTestCase {
         { 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 17 },
         {}
     });
+
     assertNull(getMatches(source, 0, "field1"));
     MatchesIterator mi = getMatches(source, 1, "field1");
     assertMatch(mi, 0, 2, 0, 18);
@@ -302,6 +320,11 @@ public class TestIntervals extends LuceneTestCase {
     assertMatch(mi, 5, 7, 35, 55);
     assertMatch(mi, 6, 17, 41, 99);
     assertFalse(mi.next());
+
+    assertGaps(source, 1, "field1", new int[]{
+        0, 0, 0, 0, 0, 0, 9
+    });
+
   }
 
   public void testNesting2() throws IOException {
@@ -447,4 +470,37 @@ public class TestIntervals extends LuceneTestCase {
     assertFalse(mi.next());
   }
 
+  public void testMaxGaps() throws IOException {
+
+    IntervalsSource source = Intervals.maxgaps(1,
+        Intervals.unordered(Intervals.term("w1"), Intervals.term("w3"), Intervals.term("w4")));
+    checkIntervals(source, "field2", 1, new int[][]{
+        {}, {}, {}, {}, {},
+        { 0, 3, 2, 4, 3, 6 }
+    });
+
+    MatchesIterator mi = getMatches(source, 5, "field2");
+    assertMatch(mi, 0, 3, 0, 11);
+
+  }
+
+  public void testNestedMaxGaps() throws IOException {
+    IntervalsSource source = Intervals.maxgaps(1,
+        Intervals.unordered(
+            Intervals.ordered(Intervals.term("w1"), Intervals.term("w3")),
+            Intervals.term("w4")
+        ));
+    checkIntervals(source, "field2", 1, new int[][]{
+        {}, {}, {}, {}, {},
+        { 0, 3, 3, 6, 4, 8 }
+    });
+
+    assertGaps(source, 5, "field2", new int[]{ 0, 0, 1 });
+
+    MatchesIterator mi = getMatches(source, 5, "field2");
+    assertMatch(mi, 0, 3, 0, 11);
+    assertMatch(mi, 3, 6, 9, 20);
+    assertMatch(mi, 4, 8, 12, 26);
+  }
+
 }


[41/47] lucene-solr:jira/http2: LUCENE-8598: Only take values into account that come from a buffer that has a value

Posted by da...@apache.org.
LUCENE-8598: Only take values into account that come from a buffer that has a value


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fdd9b55d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fdd9b55d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fdd9b55d

Branch: refs/heads/jira/http2
Commit: fdd9b55df0aecc4ad64f85aa18ed9e4c8abc3e86
Parents: d96475a
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Dec 10 19:18:22 2018 +0100
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Dec 10 19:18:51 2018 +0100

----------------------------------------------------------------------
 .../apache/lucene/index/TestFieldUpdatesBuffer.java  | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fdd9b55d/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java b/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
index d6270ee..0ca3af8 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFieldUpdatesBuffer.java
@@ -221,10 +221,9 @@ public class TestFieldUpdatesBuffer extends LuceneTestCase {
     int count = 0;
     long min = Long.MAX_VALUE;
     long max = Long.MIN_VALUE;
+    boolean hasAtLeastOneValue = false;
     while ((value = iterator.next()) != null) {
       long v = buffer.getNumericValue(count);
-      min = Math.min(min, v);
-      max = Math.max(max, v);
       randomUpdate = updates.get(count++);
       assertEquals(randomUpdate.term.bytes.utf8ToString(), value.termValue.utf8ToString());
       assertEquals(randomUpdate.term.field, value.termField);
@@ -232,14 +231,22 @@ public class TestFieldUpdatesBuffer extends LuceneTestCase {
       if (randomUpdate.hasValue) {
         assertEquals(randomUpdate.getValue(), value.numericValue);
         assertEquals(v, value.numericValue);
+        min = Math.min(min, v);
+        max = Math.max(max, v);
+        hasAtLeastOneValue = true;
       } else {
         assertEquals(0, value.numericValue);
         assertEquals(0, v);
       }
       assertEquals(randomUpdate.docIDUpto, value.docUpTo);
     }
-    assertEquals(max, buffer.getMaxNumeric());
-    assertEquals(min, buffer.getMinNumeric());
+    if (hasAtLeastOneValue) {
+      assertEquals(max, buffer.getMaxNumeric());
+      assertEquals(min, buffer.getMinNumeric());
+    } else {
+      assertEquals(0, buffer.getMaxNumeric());
+      assertEquals(0, buffer.getMinNumeric());
+    }
     assertEquals(count, updates.size());
   }
 


[35/47] lucene-solr:jira/http2: SOLR-12801: Relax specific test permissions to help harden tests.

Posted by da...@apache.org.
SOLR-12801: Relax specific test permissions to help harden tests.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bf0e9b36
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bf0e9b36
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bf0e9b36

Branch: refs/heads/jira/http2
Commit: bf0e9b367aaa50f2c13159d7a2b79ab97bf5291b
Parents: a165761
Author: markrmiller <ma...@apache.org>
Authored: Mon Dec 10 09:38:01 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Mon Dec 10 10:02:58 2018 -0600

----------------------------------------------------------------------
 lucene/tools/junit4/solr-tests.policy | 26 ++++++--------------------
 1 file changed, 6 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bf0e9b36/lucene/tools/junit4/solr-tests.policy
----------------------------------------------------------------------
diff --git a/lucene/tools/junit4/solr-tests.policy b/lucene/tools/junit4/solr-tests.policy
index d58b82c..69013eb 100644
--- a/lucene/tools/junit4/solr-tests.policy
+++ b/lucene/tools/junit4/solr-tests.policy
@@ -61,21 +61,11 @@ grant {
   permission javax.management.MBeanTrustPermission "*";
   permission javax.security.auth.AuthPermission "*";
   permission javax.security.auth.PrivateCredentialPermission "org.apache.hadoop.security.Credentials * \"*\"", "read";
-  permission java.security.SecurityPermission "putProviderProperty.SaslPlainServer";
-  permission java.security.SecurityPermission "insertProvider.SaslPlainServer";
+  permission java.security.SecurityPermission "*";
   permission javax.xml.bind.JAXBPermission "setDatatypeConverter";
-  
-  // TIKA uses BouncyCastle and that registers new provider for PDF parsing + MSOffice parsing. Maybe report as bug!
-  permission java.security.SecurityPermission "putProviderProperty.BC";
-  permission java.security.SecurityPermission "insertProvider.BC";
-
-  // Needed for some things in DNS caching in the JVM
-  permission java.security.SecurityPermission "getProperty.networkaddress.cache.ttl";
-  permission java.security.SecurityPermission "getProperty.networkaddress.cache.negative.ttl";
 
   // SSL related properties for Solr tests
-  permission java.security.SecurityPermission "getProperty.ssl.*";
-  permission javax.net.ssl.SSLPermission "setDefaultSSLContext";
+  permission javax.net.ssl.SSLPermission "*";
 
   // SASL/Kerberos related properties for Solr tests
   permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KerberosTicket * \"*\"", "read";
@@ -84,15 +74,11 @@ grant {
   permission javax.security.auth.PrivateCredentialPermission "javax.security.auth.kerberos.KeyTab * \"*\"", "read";
   permission javax.security.auth.PrivateCredentialPermission "sun.security.jgss.krb5.Krb5Util$KeysFromKeyTab * \"*\"", "read";
   
-  permission javax.security.auth.kerberos.ServicePermission "krbtgt/EXAMPLE.COM@EXAMPLE.COM", "initiate";
-  permission javax.security.auth.kerberos.ServicePermission "zookeeper/localhost@EXAMPLE.COM", "initiate";
-  permission javax.security.auth.kerberos.ServicePermission "zookeeper/localhost@EXAMPLE.COM", "accept";
-  permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "initiate";
-  permission javax.security.auth.kerberos.ServicePermission "HTTP/127.0.0.1@EXAMPLE.COM", "accept";
-  permission javax.security.auth.kerberos.DelegationPermission "\"HTTP/127.0.0.1@EXAMPLE.COM\" \"krbtgt/EXAMPLE.COM@EXAMPLE.COM\"";
+  permission javax.security.auth.kerberos.ServicePermission "*", "initiate";
+  permission javax.security.auth.kerberos.ServicePermission "*", "accept";
+  permission javax.security.auth.kerberos.DelegationPermission "\"*\" \"krbtgt/EXAMPLE.COM@EXAMPLE.COM\"";
   
   // java 8 accessibility requires this perm - should not after 8 I believe (rrd4j is the root reason we hit an accessibility code path)
-  permission java.awt.AWTPermission "listenToAllAWTEvents";
-  permission java.awt.AWTPermission "accessEventQueue";
+  permission java.awt.AWTPermission "*";
 
 };


[43/47] lucene-solr:jira/http2: SOLR-13053: NodeAddedTrigger and NodeLostTrigger do not reserve added/removed time populated by restoreState

Posted by da...@apache.org.
SOLR-13053: NodeAddedTrigger and NodeLostTrigger do not reserve added/removed time populated by restoreState


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af6e15e8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af6e15e8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af6e15e8

Branch: refs/heads/jira/http2
Commit: af6e15e838db2059e8ecb7d6c8089e2b5e042f66
Parents: ef61b54
Author: Cao Manh Dat <da...@apache.org>
Authored: Mon Dec 10 20:08:07 2018 +0000
Committer: Cao Manh Dat <da...@apache.org>
Committed: Mon Dec 10 20:08:07 2018 +0000

----------------------------------------------------------------------
 .../java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java | 3 ++-
 .../java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java  | 3 ++-
 .../solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java    | 4 +---
 3 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af6e15e8/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index 6202944..fe49891 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -69,7 +69,8 @@ public class NodeAddedTrigger extends TriggerBase {
       List<String> added = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
       added.forEach(n -> {
         // don't add nodes that have since gone away
-        if (lastLiveNodes.contains(n)) {
+        if (lastLiveNodes.contains(n) && !nodeNameVsTimeAdded.containsKey(n)) {
+          // since {@code #restoreState(AutoScaling.Trigger)} is called first, the timeAdded for a node may also be restored
           log.debug("Adding node from marker path: {}", n);
           nodeNameVsTimeAdded.put(n, cloudManager.getTimeSource().getTimeNs());
         }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af6e15e8/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 97e855c..422eac3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -69,7 +69,8 @@ public class NodeLostTrigger extends TriggerBase {
       List<String> lost = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
       lost.forEach(n -> {
         // don't add nodes that have since came back
-        if (!lastLiveNodes.contains(n)) {
+        if (!lastLiveNodes.contains(n) && !nodeNameVsTimeRemoved.containsKey(n)) {
+          // since {@code #restoreState(AutoScaling.Trigger)} is called first, the timeRemoved for a node may also be restored
           log.debug("Adding lost node from marker path: {}", n);
           nodeNameVsTimeRemoved.put(n, cloudManager.getTimeSource().getTimeNs());
         }

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af6e15e8/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 0227cda..8314b8d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -297,9 +297,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
   }
 
   @Test
-  // commented 20-July-2018  @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
-  // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
-  @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
+  // commentted 190-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
   public void testNodeLostTriggerRestoreState() throws Exception {
     // for this test we want to update the trigger so we must assert that the actions were created twice
     actionInitCalled = new CountDownLatch(2);


[31/47] lucene-solr:jira/http2: SOLR-12801: Try to harden TestReplicatinHandler#doTestIndexFetchOnMasterRestart.

Posted by da...@apache.org.
SOLR-12801: Try to harden TestReplicatinHandler#doTestIndexFetchOnMasterRestart.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5405b2f1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5405b2f1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5405b2f1

Branch: refs/heads/jira/http2
Commit: 5405b2f1c3bfca7762b16fc07179f27543904be6
Parents: f47976a
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 22:18:37 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 22:55:25 2018 -0600

----------------------------------------------------------------------
 .../solr/handler/TestReplicationHandler.java    | 27 ++++++++++++++------
 1 file changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5405b2f1/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index f1a9a86..f2b0b39 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -647,14 +647,23 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
       String cmp = BaseDistributedSearchTestCase.compare(masterQueryResult, slaveQueryResult, 0, null);
       assertEquals(null, cmp);
 
-      int timesReplicated = Integer.parseInt(getSlaveDetails("timesIndexReplicated"));
-      String timesFailed = getSlaveDetails("timesFailed");
-      if (null == timesFailed) {
+      String timesReplicatedString = getSlaveDetails("timesIndexReplicated");
+      String timesFailed;
+      Integer previousTimesFailed = null;
+      if (timesReplicatedString == null) {
         timesFailed = "0";
+      } else {
+        int timesReplicated = Integer.parseInt(timesReplicatedString);
+        timesFailed = getSlaveDetails("timesFailed");
+        if (null == timesFailed) {
+          timesFailed = "0";
+        }
+
+        previousTimesFailed = Integer.parseInt(timesFailed);
+        // Sometimes replication will fail because master's core is still loading; make sure there was one success
+        assertEquals(1, timesReplicated - previousTimesFailed);
+
       }
-      int previousTimesFailed = Integer.parseInt(timesFailed);
-      // Sometimes replication will fail because master's core is still loading; make sure there was one success
-      assertEquals(1, timesReplicated - previousTimesFailed);
 
       masterJetty.stop();
 
@@ -672,7 +681,9 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
       assertEquals(nDocs, numFound(slaveQueryRsp));
 
       int failed = Integer.parseInt(getSlaveDetails("timesFailed"));
-      assertTrue(failed > previousTimesFailed);
+      if (previousTimesFailed != null) {
+        assertTrue(failed > previousTimesFailed);
+      }
       assertEquals(1, Integer.parseInt(getSlaveDetails("timesIndexReplicated")) - failed);
     } finally {
       resetFactory();
@@ -684,7 +695,7 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
     params.set(CommonParams.QT, "/replication");
     params.set("command", "details");
     QueryResponse response = slaveClient.query(params);
-    System.out.println("SHALIN: " + response.getResponse());
+
     // details/slave/timesIndexReplicated
     NamedList<Object> details = (NamedList<Object>) response.getResponse().get("details");
     NamedList<Object> slave = (NamedList<Object>) details.get("slave");


[14/47] lucene-solr:jira/http2: SOLR-12801: tweak shard count specificiation

Posted by da...@apache.org.
SOLR-12801: tweak shard count specificiation


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/40432cdf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/40432cdf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/40432cdf

Branch: refs/heads/jira/http2
Commit: 40432cdfbcfba7e94a7de1eaf28e30e42bd6d4be
Parents: eac3f58
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 10:29:05 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40432cdf/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
index d43e1dc..d69b47b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MultiThreadedOCPTest.java
@@ -55,12 +55,13 @@ public class MultiThreadedOCPTest extends AbstractFullDistribZkTestBase {
 
   public MultiThreadedOCPTest() {
     sliceCount = 2;
+    
+    fixShardCount(3);
   }
 
   @Test
 // commented 20-July-2018   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
 //commented 20-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
-  @ShardsFixed(num = 3)
   public void test() throws Exception {
     testParallelCollectionAPICalls();
     testTaskExclusivity();


[08/47] lucene-solr:jira/http2: LUCENE-8592: Fix index sorting corruption due to numeric overflow

Posted by da...@apache.org.
LUCENE-8592: Fix index sorting corruption due to numeric overflow

The merge sort of sorted segments can produce an invalid
sort if the sort field is an Integer/Long that uses reverse order and contains values equal to
Integer/Long#MIN_VALUE. These values are always sorted first during a merge
(instead of last because of the reverse order) due to this bug.
Indices affected by the bug can be detected by running the CheckIndex command on a
distribution that contains the fix (7.6+).


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f29ed07
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f29ed07
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f29ed07

Branch: refs/heads/jira/http2
Commit: 9f29ed0757eae12d8311ffd6891f7032370ea39a
Parents: da7919f
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:16:42 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:16:42 2018 +0100

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |    9 +-
 .../index/TestBackwardsCompatibility.java       |   27 +
 .../lucene/index/sorted-invalid.7.5.0.zip       |  Bin 0 -> 468799 bytes
 .../org/apache/lucene/index/MultiSorter.java    |   30 +-
 .../apache/lucene/index/TestIndexSorting.java   | 1738 ++++++++++--------
 5 files changed, 1020 insertions(+), 784 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 14b4961..007d35b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -314,7 +314,14 @@ Bug fixes
 
 * LUCENE-8595: Fix interleaved DV update and reset. Interleaved update and reset value
   to the same doc in the same updates package looses an update if the reset comes before
-  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grant)
+  the update as well as loosing the reset if the update comes frist. (Simon Willnauer, Adrien Grand)
+
+* LUCENE-8592: Fix index sorting corruption due to numeric overflow. The merge of sorted segments
+  can produce an invalid sort if the sort field is an Integer/Long that uses reverse order and contains
+  values equal to Integer/Long#MIN_VALUE. These values are always sorted first during a merge
+  (instead of last because of the reverse order) due to this bug. Indices affected by the bug can be
+  detected by running the CheckIndex command on a distribution that contains the fix (7.6+).
+  (Jim Ferenczi, Adrien Grand, Mike McCandless, Simon Willnauer)
 
 New Features
 

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
index 6f28797..44911db 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
@@ -1717,6 +1717,33 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
       dir.close();
     }
   }
+
+  /**
+   * Tests that {@link CheckIndex} can detect invalid sort on sorted indices created
+   * before https://issues.apache.org/jira/browse/LUCENE-8592.
+   */
+  public void testSortedIndexWithInvalidSort() throws Exception {
+    Path path = createTempDir("sorted");
+    String name = "sorted-invalid.7.5.0.zip";
+    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name);
+    assertNotNull("Sorted index index " + name + " not found", resource);
+    TestUtil.unzip(resource, path);
+
+    Directory dir = FSDirectory.open(path);
+
+    DirectoryReader reader = DirectoryReader.open(dir);
+    assertEquals(1, reader.leaves().size());
+    Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
+    assertNotNull(sort);
+    assertEquals("<long: \"dateDV\">! missingValue=-9223372036854775808", sort.toString());
+    reader.close();
+    CheckIndex.Status status = TestUtil.checkIndex(dir);
+    assertEquals(1, status.segmentInfos.size());
+    assertNotNull(status.segmentInfos.get(0).indexSortStatus.error);
+    assertEquals(status.segmentInfos.get(0).indexSortStatus.error.getMessage(),
+        "segment has indexSort=<long: \"dateDV\">! missingValue=-9223372036854775808 but docID=4 sorts after docID=5");
+    dir.close();
+  }
   
   static long getValue(BinaryDocValues bdv) throws IOException {
     BytesRef term = bdv.binaryValue();

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
new file mode 100644
index 0000000..30c68a3
Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f29ed07/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
index b484228..d81ac75 100644
--- a/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/MultiSorter.java
@@ -42,17 +42,18 @@ final class MultiSorter {
 
     SortField fields[] = sort.getSort();
     final ComparableProvider[][] comparables = new ComparableProvider[fields.length][];
+    final int[] reverseMuls = new int[fields.length];
     for(int i=0;i<fields.length;i++) {
       comparables[i] = getComparableProviders(readers, fields[i]);
+      reverseMuls[i] = fields[i].getReverse() ? -1 : 1;
     }
-
     int leafCount = readers.size();
 
     PriorityQueue<LeafAndDocID> queue = new PriorityQueue<LeafAndDocID>(leafCount) {
         @Override
         public boolean lessThan(LeafAndDocID a, LeafAndDocID b) {
           for(int i=0;i<comparables.length;i++) {
-            int cmp = a.values[i].compareTo(b.values[i]);
+            int cmp = reverseMuls[i] * a.values[i].compareTo(b.values[i]);
             if (cmp != 0) {
               return cmp < 0;
             }
@@ -146,14 +147,13 @@ final class MultiSorter {
 
   /** Returns an object for this docID whose .compareTo represents the requested {@link SortField} sort order. */
   private interface ComparableProvider {
-    public Comparable getComparable(int docID) throws IOException;
+    Comparable getComparable(int docID) throws IOException;
   }
 
   /** Returns {@code ComparableProvider}s for the provided readers to represent the requested {@link SortField} sort order. */
   private static ComparableProvider[] getComparableProviders(List<CodecReader> readers, SortField sortField) throws IOException {
 
     ComparableProvider[] providers = new ComparableProvider[readers.size()];
-    final int reverseMul = sortField.getReverse() ? -1 : 1;
     final SortField.Type sortType = Sorter.getSortFieldType(sortField);
 
     switch(sortType) {
@@ -169,9 +169,9 @@ final class MultiSorter {
         OrdinalMap ordinalMap = OrdinalMap.build(null, values, PackedInts.DEFAULT);
         final int missingOrd;
         if (sortField.getMissingValue() == SortField.STRING_LAST) {
-          missingOrd = sortField.getReverse() ? Integer.MIN_VALUE : Integer.MAX_VALUE;
+          missingOrd = Integer.MAX_VALUE;
         } else {
-          missingOrd = sortField.getReverse() ? Integer.MAX_VALUE : Integer.MIN_VALUE;
+          missingOrd = Integer.MIN_VALUE;
         }
 
         for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
@@ -197,7 +197,7 @@ final class MultiSorter {
                 }
                 if (readerDocID == docID) {
                   // translate segment's ord to global ord space:
-                  return reverseMul * (int) globalOrds.get(readerValues.ordValue());
+                  return Math.toIntExact(globalOrds.get(readerValues.ordValue()));
                 } else {
                   return missingOrd;
                 }
@@ -238,9 +238,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * values.longValue();
+                  return values.longValue();
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -279,9 +279,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * (int) values.longValue();
+                  return (int) values.longValue();
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -320,9 +320,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * Double.longBitsToDouble(values.longValue());
+                  return Double.longBitsToDouble(values.longValue());
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };
@@ -361,9 +361,9 @@ final class MultiSorter {
                   readerDocID = values.advance(docID);
                 }
                 if (readerDocID == docID) {
-                  return reverseMul * Float.intBitsToFloat((int) values.longValue());
+                  return Float.intBitsToFloat((int) values.longValue());
                 } else {
-                  return reverseMul * missingValue;
+                  return missingValue;
                 }
               }
             };


[02/47] lucene-solr:jira/http2: SOLR-13046: Suppress SSL for StreamingTest

Posted by da...@apache.org.
SOLR-13046: Suppress SSL for StreamingTest


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/17fca051
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/17fca051
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/17fca051

Branch: refs/heads/jira/http2
Commit: 17fca051c56977a0fcc9ab79506dbdb89fce2722
Parents: b24af10
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Dec 7 09:19:40 2018 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Dec 7 09:19:40 2018 -0500

----------------------------------------------------------------------
 .../test/org/apache/solr/client/solrj/io/stream/StreamingTest.java | 2 ++
 1 file changed, 2 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/17fca051/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
index b43dc13..003f01b 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
@@ -28,6 +28,7 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.JettySolrRunner;
 import org.apache.solr.client.solrj.io.SolrClientCache;
@@ -62,6 +63,7 @@ import org.junit.Test;
 *
 **/
 
+@SolrTestCaseJ4.SuppressSSL
 @LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
 public class StreamingTest extends SolrCloudTestCase {
 


[15/47] lucene-solr:jira/http2: SOLR-12801: wait 60 seconds instead of 30 for cores to close

Posted by da...@apache.org.
SOLR-12801: wait 60 seconds instead of 30 for cores to close


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eac3f58c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eac3f58c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eac3f58c

Branch: refs/heads/jira/http2
Commit: eac3f58c4395534dd739fe645850e9f120f51323
Parents: 906e32a
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 01:08:33 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eac3f58c/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 13d7f22..9a7fef1 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -313,7 +313,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
 
       if (suiteFailureMarker.wasSuccessful()) {
         // if the tests passed, make sure everything was closed / released
-        String orr = clearObjectTrackerAndCheckEmpty(30, false);
+        String orr = clearObjectTrackerAndCheckEmpty(60, false);
         assertNull(orr, orr);
       } else {
         ObjectReleaseTracker.tryClose();


[25/47] lucene-solr:jira/http2: SOLR-13037: Add @AwaitsFix annotation to TestSimGenericDistributedQueue.

Posted by da...@apache.org.
SOLR-13037: Add @AwaitsFix annotation to TestSimGenericDistributedQueue.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e25d6850
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e25d6850
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e25d6850

Branch: refs/heads/jira/http2
Commit: e25d6850c4a90a053c2c34ccd7478afb508a479a
Parents: 54ca26a
Author: markrmiller <ma...@apache.org>
Authored: Sat Dec 8 11:34:02 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../cloud/autoscaling/sim/TestSimGenericDistributedQueue.java    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e25d6850/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
index 89f3366..68c37a4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
@@ -16,13 +16,15 @@
  */
 package org.apache.solr.cloud.autoscaling.sim;
 
+import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
+
 import org.apache.solr.client.solrj.cloud.DistributedQueue;
 import org.apache.solr.client.solrj.cloud.DistribStateManager;
 
 /**
  *
  */
-// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-02-26
+@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-13037")
 public class TestSimGenericDistributedQueue extends TestSimDistributedQueue {
   DistribStateManager stateManager = new SimDistribStateManager();
 


[11/47] lucene-solr:jira/http2: SOLR-12727: CHANGES.txt, updated ZooKeeper dependency

Posted by da...@apache.org.
SOLR-12727: CHANGES.txt, updated ZooKeeper dependency


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/54ca26ab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/54ca26ab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/54ca26ab

Branch: refs/heads/jira/http2
Commit: 54ca26abe92e17a1d56dbc53c9e98844f1994579
Parents: f690302
Author: Erick Erickson <Er...@gmail.com>
Authored: Sun Dec 9 09:15:42 2018 -0800
Committer: Erick Erickson <Er...@gmail.com>
Committed: Sun Dec 9 09:15:42 2018 -0800

----------------------------------------------------------------------
 solr/CHANGES.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/54ca26ab/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index dc6fc47..68e17a0 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -25,7 +25,7 @@ Versions of Major Components
 Apache Tika 1.19.1
 Carrot2 3.16.0
 Velocity 1.7 and Velocity Tools 2.0
-Apache ZooKeeper 3.4.11
+Apache ZooKeeper 3.4.13
 Jetty 9.4.14.v20181114
 
 Upgrade Notes
@@ -124,7 +124,7 @@ Versions of Major Components
 Apache Tika 1.19.1
 Carrot2 3.16.0
 Velocity 1.7 and Velocity Tools 2.0
-Apache ZooKeeper 3.4.11
+Apache ZooKeeper 3.4.13
 Jetty 9.4.14.v20181114
 
 


[47/47] lucene-solr:jira/http2: Merge with master

Posted by da...@apache.org.
Merge with master


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/925fb154
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/925fb154
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/925fb154

Branch: refs/heads/jira/http2
Commit: 925fb154f134598e4c7f48c7218ab687d6047901
Parents: 0ecb811 c651cbf
Author: Cao Manh Dat <da...@apache.org>
Authored: Tue Dec 11 09:27:19 2018 +0000
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue Dec 11 09:27:19 2018 +0000

----------------------------------------------------------------------
 lucene/CHANGES.txt                              |   21 +-
 .../index/TestBackwardsCompatibility.java       |   27 +
 .../lucene/index/sorted-invalid.7.5.0.zip       |  Bin 0 -> 908443 bytes
 .../java/org/apache/lucene/document/Field.java  |   14 +-
 .../org/apache/lucene/document/StoredField.java |   13 +
 .../lucene/index/DocValuesFieldUpdates.java     |   75 +-
 .../apache/lucene/index/FieldUpdatesBuffer.java |   21 +
 .../lucene/index/FrozenBufferedUpdates.java     |    3 +-
 .../org/apache/lucene/index/IndexableField.java |    7 +
 .../org/apache/lucene/index/MultiSorter.java    |   30 +-
 .../index/NumericDocValuesFieldUpdates.java     |   31 +-
 .../java/org/apache/lucene/util/Constants.java  |    1 +
 .../util/packed/AbstractPagedMutable.java       |    3 +-
 .../lucene/index/TestDocValuesFieldUpdates.java |   59 +-
 .../lucene/index/TestFieldUpdatesBuffer.java    |   24 +-
 .../apache/lucene/index/TestIndexSorting.java   | 1738 ++++++++++--------
 lucene/ivy-versions.properties                  |    2 +-
 .../search/similarity/LegacyBM25Similarity.java |    8 +
 .../intervals/DifferenceIntervalFunction.java   |   10 +
 .../intervals/DisjunctionIntervalsSource.java   |   15 +
 .../intervals/FilteredIntervalsSource.java      |  103 ++
 .../lucene/search/intervals/IntervalFilter.java |    5 +
 .../search/intervals/IntervalFunction.java      |   54 +-
 .../search/intervals/IntervalIterator.java      |   11 +
 .../search/intervals/IntervalMatches.java       |    9 +
 .../intervals/IntervalMatchesIterator.java      |   38 +
 .../lucene/search/intervals/Intervals.java      |   23 +-
 .../intervals/LowpassIntervalsSource.java       |   90 -
 .../MinimizingConjunctionIntervalsSource.java   |    7 +-
 .../search/intervals/TermIntervalsSource.java   |    5 +
 .../lucene/search/intervals/package-info.java   |    2 +
 .../search/intervals/TestIntervalQuery.java     |   10 +
 .../lucene/search/intervals/TestIntervals.java  |   60 +-
 lucene/tools/junit4/solr-tests.policy           |   26 +-
 solr/CHANGES.txt                                |    9 +-
 .../client/solrj/embedded/JettySolrRunner.java  |    2 +-
 .../cloud/autoscaling/NodeAddedTrigger.java     |    3 +-
 .../solr/cloud/autoscaling/NodeLostTrigger.java |    3 +-
 .../solr/response/BinaryResponseWriter.java     |  121 +-
 .../org/apache/solr/response/DocsStreamer.java  |    6 +-
 .../org/apache/solr/response/ResultContext.java |    2 +
 .../java/org/apache/solr/schema/FieldType.java  |    7 +
 .../org/apache/solr/schema/IndexSchema.java     |   10 +-
 .../java/org/apache/solr/schema/StrField.java   |    6 +
 .../java/org/apache/solr/schema/TextField.java  |    5 +
 .../apache/solr/search/SolrDocumentFetcher.java |   24 +-
 .../cloud/AssignBackwardCompatibilityTest.java  |    5 +-
 .../solr/cloud/CollectionsAPISolrJTest.java     |    5 +-
 .../solr/cloud/DocValuesNotIndexedTest.java     |   29 +-
 .../apache/solr/cloud/KerberosTestServices.java |    3 +-
 .../apache/solr/cloud/MultiThreadedOCPTest.java |   11 +-
 .../solr/cloud/SaslZkACLProviderTest.java       |    2 +-
 .../apache/solr/cloud/TestCloudConsistency.java |    2 +
 .../solr/cloud/TestMiniSolrCloudClusterSSL.java |    5 +-
 .../apache/solr/cloud/TestSSLRandomization.java |    2 +
 .../cloud/TestSolrCloudWithKerberosAlt.java     |    3 +
 .../CollectionsAPIAsyncDistributedZkTest.java   |    2 +-
 .../CollectionsAPIDistributedZkTest.java        |   16 +-
 .../api/collections/CustomCollectionTest.java   |    2 +
 .../autoscaling/AutoScalingHandlerTest.java     |    1 +
 .../autoscaling/SystemLogListenerTest.java      |   46 +-
 .../cloud/autoscaling/sim/SimCloudManager.java  |    3 +-
 .../sim/SimClusterStateProvider.java            |  141 +-
 .../autoscaling/sim/SimDistribStateManager.java |   11 +-
 .../autoscaling/sim/TestSimExtremeIndexing.java |    2 -
 .../sim/TestSimGenericDistributedQueue.java     |    4 +-
 .../autoscaling/sim/TestSimLargeCluster.java    |   26 +-
 .../autoscaling/sim/TestSimNodeLostTrigger.java |    2 +-
 .../autoscaling/sim/TestSimPolicyCloud.java     |    6 +-
 .../sim/TestSimTriggerIntegration.java          |   30 +-
 .../solr/cloud/cdcr/CdcrBootstrapTest.java      |    2 +-
 .../solr/handler/TestReplicationHandler.java    |   33 +-
 .../metrics/rrd/SolrRrdBackendFactoryTest.java  |   14 +-
 .../solr/response/TestBinaryResponseWriter.java |   45 +-
 solr/licenses/zookeeper-3.4.11.jar.sha1         |    1 -
 solr/licenses/zookeeper-3.4.13.jar.sha1         |    1 +
 .../solr/client/solrj/embedded/SSLConfig.java   |    8 +-
 .../client/solrj/io/stream/FacetStream.java     |    3 +-
 .../client/solrj/io/stream/RandomStream.java    |    3 +-
 .../client/solrj/io/stream/SearchStream.java    |    3 +-
 .../solrj/io/stream/SignificantTermsStream.java |    3 +-
 .../client/solrj/io/stream/StatsStream.java     |    5 +-
 .../solrj/io/stream/TimeSeriesStream.java       |    3 +-
 .../solr/client/solrj/util/Constants.java       |    1 +
 .../solr/common/ConditionalMapWriter.java       |    8 +-
 .../apache/solr/common/LinkedHashMapWriter.java |    2 +-
 .../java/org/apache/solr/common/MapWriter.java  |    5 +
 .../org/apache/solr/common/MapWriterMap.java    |    2 +-
 .../org/apache/solr/common/SolrDocument.java    |   18 +-
 .../apache/solr/common/SolrDocumentBase.java    |    2 +-
 .../apache/solr/common/SolrInputDocument.java   |    8 +-
 .../common/util/ByteArrayUtf8CharSequence.java  |   19 +-
 .../solr/common/util/FastOutputStream.java      |    2 +-
 .../apache/solr/common/util/JavaBinCodec.java   |   25 +-
 .../org/apache/solr/common/util/NamedList.java  |    2 +-
 .../solr/common/util/Utf8CharSequence.java      |    2 +-
 .../solrj/embedded/SolrExampleJettyTest.java    |   65 +
 .../impl/HttpSolrClientSSLAuthConPoolTest.java  |    4 +
 .../solrj/io/stream/StreamExpressionTest.java   |   66 +
 .../client/solrj/io/stream/StreamingTest.java   |    2 +
 .../java/org/apache/solr/SolrTestCaseJ4.java    |    2 +-
 .../org/apache/solr/cloud/ZkTestServer.java     |   12 +-
 .../org/apache/solr/util/SSLTestConfig.java     |    2 +-
 103 files changed, 2310 insertions(+), 1165 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/lucene/ivy-versions.properties
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/CHANGES.txt
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
index 7d34861,0978e57..19b99c0
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithKerberosAlt.java
@@@ -44,7 -46,9 +46,8 @@@ import org.slf4j.LoggerFactory
  @ThreadLeakFilters(defaultFilters = true, filters = {
      BadZookeeperThreadsFilter.class // Zookeeper login leaks TGT renewal threads
  })
 -
  @LuceneTestCase.Slow
+ @ThreadLeakLingering(linger = 10000) // minikdc has some lingering threads
  public class TestSolrCloudWithKerberosAlt extends SolrCloudTestCase {
  
    private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
index 4091bf7,0000000..4ea9f70
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
@@@ -1,166 -1,0 +1,172 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +package org.apache.solr.client.solrj.embedded;
 +
++import org.apache.solr.client.solrj.util.Constants;
 +import org.eclipse.jetty.util.ssl.SslContextFactory;
 +
 +/** 
 + * Encapsulates settings related to SSL Configuration.
 + * NOTE: all other settings are ignored if {@link #isSSLMode} is false.
 + * @see #setUseSSL
 + */
 +public class SSLConfig {
 +  
 +  private boolean useSsl;
 +  private boolean clientAuth;
 +  private String keyStore;
 +  private String keyStorePassword;
 +  private String trustStore;
 +  private String trustStorePassword;
 +
 +  /** NOTE: all other settings are ignored if useSSL is false; trustStore settings are ignored if clientAuth is false */
 +  public SSLConfig(boolean useSSL, boolean clientAuth, String keyStore, String keyStorePassword, String trustStore, String trustStorePassword) {
-     this.useSsl = useSSL;
++    // @AwaitsFix: SOLR-12988 - ssl issues on Java 11/12
++    if (Constants.JRE_IS_MINIMUM_JAVA11) {
++      this.useSsl = false;
++    } else {
++      this.useSsl = useSSL;
++    }
 +    this.clientAuth = clientAuth;
 +    this.keyStore = keyStore;
 +    this.keyStorePassword = keyStorePassword;
 +    this.trustStore = trustStore;
 +    this.trustStorePassword = trustStorePassword;
 +  }
 +  
 +  public void setUseSSL(boolean useSSL) {
 +    this.useSsl = useSSL;
 +  }
 +  
 +  public void setClientAuth(boolean clientAuth) {
 +    this.clientAuth = clientAuth;
 +  }
 +  
 +  /** All other settings on this object are ignored unless this is true */
 +  public boolean isSSLMode() {
 +    return useSsl;
 +  }
 +  
 +  public boolean isClientAuthMode() {
 +    return clientAuth;
 +  }
 +
 +  public String getKeyStore() {
 +    return keyStore;
 +  }
 +
 +  public String getKeyStorePassword() {
 +    return keyStorePassword;
 +  }
 +
 +  public String getTrustStore() {
 +    return trustStore;
 +  }
 +
 +  public String getTrustStorePassword() {
 +    return trustStorePassword;
 +  }
 +
 +  /**
 +   * Returns an SslContextFactory that should be used by a jetty server based on the specified 
 +   * SSLConfig param which may be null.
 +   *
 +   * if the SSLConfig param is non-null, then this method will return the results of 
 +   * {@link #createContextFactory()}.
 +   * 
 +   * If the SSLConfig param is null, then this method will return null unless the 
 +   * <code>tests.jettySsl</code> system property is true, in which case standard "javax.net.ssl.*" 
 +   * system properties will be used instead, along with "tests.jettySsl.clientAuth".
 +   * 
 +   * @see #createContextFactory()
 +   */
 +  public static SslContextFactory createContextFactory(SSLConfig sslConfig) {
 +
 +    if (sslConfig != null) {
 +      return sslConfig.createContextFactory();
 +    }
 +    // else...
 +    if (Boolean.getBoolean("tests.jettySsl")) {
 +      return configureSslFromSysProps();
 +    }
 +    // else...
 +    return null;
 +  }
 +  
 +  /**
 +   * Returns an SslContextFactory that should be used by a jetty server based on this SSLConfig instance, 
 +   * or null if SSL should not be used.
 +   *
 +   * The default implementation generates a simple factory according to the keystore, truststore, 
 +   * and clientAuth properties of this object.
 +   *
 +   * @see #getKeyStore
 +   * @see #getKeyStorePassword
 +   * @see #isClientAuthMode
 +   * @see #getTrustStore
 +   * @see #getTrustStorePassword
 +   */
 +  public SslContextFactory createContextFactory() {
 +
 +    if (! isSSLMode()) {
 +      return null;
 +    }
 +    // else...
 +    
 +    SslContextFactory factory = new SslContextFactory(false);
 +    if (getKeyStore() != null)
 +      factory.setKeyStorePath(getKeyStore());
 +    if (getKeyStorePassword() != null)
 +      factory.setKeyStorePassword(getKeyStorePassword());
 +    
 +    factory.setNeedClientAuth(isClientAuthMode());
 +    
 +    if (isClientAuthMode()) {
 +      if (getTrustStore() != null)
 +        factory.setTrustStorePath(getTrustStore());
 +      if (getTrustStorePassword() != null)
 +        factory.setTrustStorePassword(getTrustStorePassword());
 +    }
 +    return factory;
 +
 +  }
 +
 +  private static SslContextFactory configureSslFromSysProps() {
 +
 +    SslContextFactory sslcontext = new SslContextFactory(false);
 +
 +    if (null != System.getProperty("javax.net.ssl.keyStore")) {
 +      sslcontext.setKeyStorePath
 +          (System.getProperty("javax.net.ssl.keyStore"));
 +    }
 +    if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
 +      sslcontext.setKeyStorePassword
 +          (System.getProperty("javax.net.ssl.keyStorePassword"));
 +    }
 +    if (null != System.getProperty("javax.net.ssl.trustStore")) {
 +      sslcontext.setTrustStorePath
 +          (System.getProperty("javax.net.ssl.trustStore"));
 +    }
 +    if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
 +      sslcontext.setTrustStorePassword
 +          (System.getProperty("javax.net.ssl.trustStorePassword"));
 +    }
 +    sslcontext.setNeedClientAuth(Boolean.getBoolean("tests.jettySsl.clientAuth"));
 +
 +    return sslcontext;
 +  }
 +}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/solrj/src/java/org/apache/solr/client/solrj/util/Constants.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/util/Constants.java
index 55ca618,0000000..50633ff
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/Constants.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/Constants.java
@@@ -1,40 -1,0 +1,41 @@@
 +/*
 + * Licensed to the Apache Software Foundation (ASF) under one or more
 + * contributor license agreements.  See the NOTICE file distributed with
 + * this work for additional information regarding copyright ownership.
 + * The ASF licenses this file to You under the Apache License, Version 2.0
 + * (the "License"); you may not use this file except in compliance with
 + * the License.  You may obtain a copy of the License at
 + *
 + *     http://www.apache.org/licenses/LICENSE-2.0
 + *
 + * Unless required by applicable law or agreed to in writing, software
 + * distributed under the License is distributed on an "AS IS" BASIS,
 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 + * See the License for the specific language governing permissions and
 + * limitations under the License.
 + */
 +
 +package org.apache.solr.client.solrj.util;
 +
 +import java.util.StringTokenizer;
 +
 +// Clone of org.apache.lucene.util.Constants, so SolrJ can use it
 +public class Constants {
 +  public static final String JVM_SPEC_VERSION = System.getProperty("java.specification.version");
 +  private static final int JVM_MAJOR_VERSION;
 +  private static final int JVM_MINOR_VERSION;
 +
 +  static {
 +    final StringTokenizer st = new StringTokenizer(JVM_SPEC_VERSION, ".");
 +    JVM_MAJOR_VERSION = Integer.parseInt(st.nextToken());
 +    if (st.hasMoreTokens()) {
 +      JVM_MINOR_VERSION = Integer.parseInt(st.nextToken());
 +    } else {
 +      JVM_MINOR_VERSION = 0;
 +    }
 +  }
 +
 +  public static final boolean JRE_IS_MINIMUM_JAVA9 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9);
++  public static final boolean JRE_IS_MINIMUM_JAVA11 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 11);
 +
 +}

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/925fb154/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
----------------------------------------------------------------------
diff --cc solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
index 003463a,307f547..f6281e1
--- a/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
@@@ -100,10 -97,10 +100,10 @@@ public class SSLTestConfig 
     * @see HttpClientUtil#SYS_PROP_CHECK_PEER_NAME
     */
    public SSLTestConfig(boolean useSSL, boolean clientAuth, boolean checkPeerName) {
 -    super(useSSL, clientAuth, null, TEST_KEYSTORE_PASSWORD, null, TEST_KEYSTORE_PASSWORD);
 -
 +    this.useSsl = useSSL;
 +    this.clientAuth = clientAuth;
      this.checkPeerName = checkPeerName;
-     
+ 
      final String resourceName = checkPeerName
        ? TEST_KEYSTORE_LOCALHOST_RESOURCE : TEST_KEYSTORE_BOGUSHOST_RESOURCE;
      trustStore = keyStore = Resource.newClassPathResource(resourceName);


[27/47] lucene-solr:jira/http2: SOLR-12923: Harden AutoScalingHandlerTest.

Posted by da...@apache.org.
SOLR-12923: Harden AutoScalingHandlerTest.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1184fbed
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1184fbed
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1184fbed

Branch: refs/heads/jira/http2
Commit: 1184fbed63bbb4053f00a77d51bce45221c216aa
Parents: b94911b
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 00:15:44 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java   | 1 +
 1 file changed, 1 insertion(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1184fbed/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index d8219ab..fd3f563 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -792,6 +792,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
     CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6)
         .setMaxShardsPerNode(3);
     CollectionAdminResponse adminResponse = create.process(solrClient);
+    cluster.waitForActiveCollection("readApiTestViolations", 1, 6);
     assertTrue(adminResponse.isSuccess());
 
     // reset the original cluster policy


[05/47] lucene-solr:jira/http2: Revert "Add the ability to set the number of hits to track accurately"

Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/da7919f7/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
index 5bfd9d5..3857a97 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexSorting.java
@@ -355,206 +355,166 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingStringFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
-      sortField.setMissingValue(SortField.STRING_FIRST);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.STRING);
+    sortField.setMissingValue(SortField.STRING_FIRST);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      w.addDocument(new Document());
-      w.commit();
+    // missing
+    w.addDocument(new Document());
+    w.commit();
 
-      doc = new Document();
-      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      SortedDocValues values = leaf.getSortedDocValues("foo");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals("zzz", values.binaryValue().utf8ToString());
-        assertEquals(1, values.nextDoc());
-        assertEquals("mmm", values.binaryValue().utf8ToString());
-      } else {
-        // docID 0 is missing:
-        assertEquals(1, values.nextDoc());
-        assertEquals("mmm", values.binaryValue().utf8ToString());
-        assertEquals(2, values.nextDoc());
-        assertEquals("zzz", values.binaryValue().utf8ToString());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    SortedDocValues values = leaf.getSortedDocValues("foo");
+    // docID 0 is missing:
+    assertEquals(1, values.nextDoc());
+    assertEquals("mmm", values.binaryValue().utf8ToString());
+    assertEquals(2, values.nextDoc());
+    assertEquals("zzz", values.binaryValue().utf8ToString());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedStringFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedSetSortField("foo", reverse);
-      sortField.setMissingValue(SortField.STRING_FIRST);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedSetSortField("foo", false);
+    sortField.setMissingValue(SortField.STRING_FIRST);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzza")));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("nnnn")));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3l, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2l, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1l, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1l, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2l, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3l, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1l, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2l, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3l, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingStringLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.STRING, reverse);
-      sortField.setMissingValue(SortField.STRING_LAST);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.STRING);
+    sortField.setMissingValue(SortField.STRING_LAST);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new SortedDocValuesField("foo", new BytesRef("zzz")));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      w.addDocument(new Document());
-      w.commit();
+    // missing
+    w.addDocument(new Document());
+    w.commit();
 
-      doc = new Document();
-      doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new SortedDocValuesField("foo", new BytesRef("mmm")));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      SortedDocValues values = leaf.getSortedDocValues("foo");
-      if (reverse) {
-        assertEquals(1, values.nextDoc());
-        assertEquals("zzz", values.binaryValue().utf8ToString());
-        assertEquals(2, values.nextDoc());
-        assertEquals("mmm", values.binaryValue().utf8ToString());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals("mmm", values.binaryValue().utf8ToString());
-        assertEquals(1, values.nextDoc());
-        assertEquals("zzz", values.binaryValue().utf8ToString());
-      }
-      assertEquals(NO_MORE_DOCS, values.nextDoc());
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    SortedDocValues values = leaf.getSortedDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals("mmm", values.binaryValue().utf8ToString());
+    assertEquals(1, values.nextDoc());
+    assertEquals("zzz", values.binaryValue().utf8ToString());
+    assertEquals(NO_MORE_DOCS, values.nextDoc());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedStringLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedSetSortField("foo", reverse);
-      sortField.setMissingValue(SortField.STRING_LAST);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedSetSortField("foo", false);
+    sortField.setMissingValue(SortField.STRING_LAST);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzz")));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("zzzd")));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
-      doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("mmm")));
+    doc.add(new SortedSetDocValuesField("foo", new BytesRef("ppp")));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3l, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2l, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1l, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1l, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2l, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3l, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1l, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2l, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3l, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testBasicLong() throws Exception {
@@ -637,213 +597,11 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingLongFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
-      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 18));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      w.addDocument(new Document());
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 7));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(18, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(7, values.longValue());
-      } else {
-        // docID 0 has no value
-        assertEquals(1, values.nextDoc());
-        assertEquals(7, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(18, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingMultiValuedLongFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
-      sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      doc.add(new SortedNumericDocValuesField("foo", 18));
-      doc.add(new SortedNumericDocValuesField("foo", 27));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      w.addDocument(doc);
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", 7));
-      doc.add(new SortedNumericDocValuesField("foo", 24));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingLongLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.LONG, reverse);
-      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 18));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      w.addDocument(new Document());
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 7));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        // docID 0 is missing
-        assertEquals(1, values.nextDoc());
-        assertEquals(18, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(7, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(7, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(18, values.longValue());
-      }
-      assertEquals(NO_MORE_DOCS, values.nextDoc());
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingMultiValuedLongLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG, reverse);
-      sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", 18));
-      doc.add(new SortedNumericDocValuesField("foo", 65));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      w.addDocument(doc);
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      doc.add(new SortedNumericDocValuesField("foo", 7));
-      doc.add(new SortedNumericDocValuesField("foo", 34));
-      doc.add(new SortedNumericDocValuesField("foo", 74));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testBasicInt() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+    SortField sortField = new SortField("foo", SortField.Type.LONG);
+    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+    Sort indexSort = new Sort(sortField);
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -852,9 +610,8 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    doc = new Document();
-    doc.add(new NumericDocValuesField("foo", -1));
-    w.addDocument(doc);
+    // missing
+    w.addDocument(new Document());
     w.commit();
 
     doc = new Document();
@@ -866,8 +623,7 @@ public class TestIndexSorting extends LuceneTestCase {
     LeafReader leaf = getOnlyLeafReader(r);
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
-    assertEquals(0, values.nextDoc());
-    assertEquals(-1, values.longValue());
+    // docID 0 has no value
     assertEquals(1, values.nextDoc());
     assertEquals(7, values.longValue());
     assertEquals(2, values.nextDoc());
@@ -877,32 +633,32 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
-  public void testBasicMultiValuedInt() throws Exception {
+  public void testMissingMultiValuedLongFirst() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+    sortField.setMissingValue(Long.valueOf(Long.MIN_VALUE));
+    Sort indexSort = new Sort(sortField);
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
     doc.add(new NumericDocValuesField("id", 3));
     doc.add(new SortedNumericDocValuesField("foo", 18));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
+    doc.add(new SortedNumericDocValuesField("foo", 27));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
+    // missing
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", -1));
-    doc.add(new SortedNumericDocValuesField("foo", 34));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
     doc.add(new NumericDocValuesField("id", 2));
     doc.add(new SortedNumericDocValuesField("foo", 7));
-    doc.add(new SortedNumericDocValuesField("foo", 22));
-    doc.add(new SortedNumericDocValuesField("foo", 27));
+    doc.add(new SortedNumericDocValuesField("foo", 24));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -921,212 +677,496 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
-  public void testMissingIntFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
-      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 18));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      w.addDocument(new Document());
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 7));
-      w.addDocument(doc);
-      w.forceMerge(1);
+  public void testMissingLongLast() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.LONG);
+    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 18));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(18, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(7, values.longValue());
-      } else {
-        assertEquals(1, values.nextDoc());
-        assertEquals(7, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(18, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    // missing
+    w.addDocument(new Document());
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 7));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals(7, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(18, values.longValue());
+    assertEquals(NO_MORE_DOCS, values.nextDoc());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testMissingMultiValuedLongLast() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.LONG);
+    sortField.setMissingValue(Long.valueOf(Long.MAX_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", 18));
+    doc.add(new SortedNumericDocValuesField("foo", 65));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", 7));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
+    doc.add(new SortedNumericDocValuesField("foo", 74));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testBasicInt() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.INT));
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 18));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", -1));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 7));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals(-1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(7, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(18, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testBasicMultiValuedInt() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.INT));
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", 18));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", -1));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", 7));
+    doc.add(new SortedNumericDocValuesField("foo", 22));
+    doc.add(new SortedNumericDocValuesField("foo", 27));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testMissingIntFirst() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.INT);
+    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 18));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    // missing
+    w.addDocument(new Document());
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 7));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(1, values.nextDoc());
+    assertEquals(7, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(18, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedIntFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
-      sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      doc.add(new SortedNumericDocValuesField("foo", 18));
-      doc.add(new SortedNumericDocValuesField("foo", 187667));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+    sortField.setMissingValue(Integer.valueOf(Integer.MIN_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", 18));
+    doc.add(new SortedNumericDocValuesField("foo", 187667));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", 7));
-      doc.add(new SortedNumericDocValuesField("foo", 34));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", 7));
+    doc.add(new SortedNumericDocValuesField("foo", 34));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingIntLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.INT, reverse);
-      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 18));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.INT);
+    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 18));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      w.addDocument(new Document());
-      w.commit();
+    // missing
+    w.addDocument(new Document());
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("foo", 7));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("foo", 7));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        // docID 0 is missing
-        assertEquals(1, values.nextDoc());
-        assertEquals(18, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(7, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(7, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(18, values.longValue());
-      }
-      assertEquals(NO_MORE_DOCS, values.nextDoc());
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals(7, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(18, values.longValue());
+    assertEquals(NO_MORE_DOCS, values.nextDoc());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedIntLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT, reverse);
-      sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", 18));
-      doc.add(new SortedNumericDocValuesField("foo", 6372));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.INT);
+    sortField.setMissingValue(Integer.valueOf(Integer.MAX_VALUE));
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", 18));
+    doc.add(new SortedNumericDocValuesField("foo", 6372));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      doc.add(new SortedNumericDocValuesField("foo", 7));
-      doc.add(new SortedNumericDocValuesField("foo", 8));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", 7));
+    doc.add(new SortedNumericDocValuesField("foo", 8));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testBasicDouble() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", 18.0));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", -1.0));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", 7.0));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(1, values.nextDoc());
+    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(2, values.nextDoc());
+    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testBasicMultiValuedDouble() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testMissingDoubleFirst() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", 18.0));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    // missing
+    w.addDocument(new Document());
+    w.commit();
+
+    doc = new Document();
+    doc.add(new DoubleDocValuesField("foo", 7.0));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(1, values.nextDoc());
+    assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(2, values.nextDoc());
+    assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    r.close();
+    w.close();
+    dir.close();
+  }
+
+  public void testMissingMultiValuedDoubleFirst() throws Exception {
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+    sortField.setMissingValue(Double.NEGATIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
+
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    w.addDocument(doc);
+    w.commit();
+
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
+    w.addDocument(doc);
+    w.forceMerge(1);
+
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
-  public void testBasicDouble() throws Exception {
+  public void testMissingDoubleLast() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortField("foo", SortField.Type.DOUBLE));
+    SortField sortField = new SortField("foo", SortField.Type.DOUBLE);
+    sortField.setMissingValue(Double.POSITIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
@@ -1135,9 +1175,8 @@ public class TestIndexSorting extends LuceneTestCase {
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
-    doc = new Document();
-    doc.add(new DoubleDocValuesField("foo", -1.0));
-    w.addDocument(doc);
+    // missing
+    w.addDocument(new Document());
     w.commit();
 
     doc = new Document();
@@ -1150,41 +1189,41 @@ public class TestIndexSorting extends LuceneTestCase {
     assertEquals(3, leaf.maxDoc());
     NumericDocValues values = leaf.getNumericDocValues("foo");
     assertEquals(0, values.nextDoc());
-    assertEquals(-1.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(1, values.nextDoc());
     assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-    assertEquals(2, values.nextDoc());
+    assertEquals(1, values.nextDoc());
     assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
+    assertEquals(NO_MORE_DOCS, values.nextDoc());
     r.close();
     w.close();
     dir.close();
   }
 
-  public void testBasicMultiValuedDouble() throws Exception {
+  public void testMissingMultiValuedDoubleLast() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-    Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.DOUBLE));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE);
+    sortField.setMissingValue(Double.POSITIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
     iwc.setIndexSort(indexSort);
     IndexWriter w = new IndexWriter(dir, iwc);
     Document doc = new Document();
-    doc.add(new NumericDocValuesField("id", 3));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.54)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(27.0)));
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
     w.addDocument(doc);
     // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
     w.commit();
 
+    // missing
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 1));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(-1.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(0.0)));
+    doc.add(new NumericDocValuesField("id", 3));
     w.addDocument(doc);
     w.commit();
 
     doc = new Document();
-    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new NumericDocValuesField("id", 1));
     doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.67)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
     w.addDocument(doc);
     w.forceMerge(1);
 
@@ -1203,207 +1242,6 @@ public class TestIndexSorting extends LuceneTestCase {
     dir.close();
   }
 
-  public void testMissingDoubleFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
-      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new DoubleDocValuesField("foo", 18.0));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      w.addDocument(new Document());
-      w.commit();
-
-      doc = new Document();
-      doc.add(new DoubleDocValuesField("foo", 7.0));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-        assertEquals(1, values.nextDoc());
-        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-      } else {
-        assertEquals(1, values.nextDoc());
-        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-        assertEquals(2, values.nextDoc());
-        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingMultiValuedDoubleFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
-      sortField.setMissingValue(Double.NEGATIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.76)));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      w.addDocument(doc);
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(70.0)));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingDoubleLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.DOUBLE, reverse);
-      sortField.setMissingValue(Double.POSITIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new DoubleDocValuesField("foo", 18.0));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      w.addDocument(new Document());
-      w.commit();
-
-      doc = new Document();
-      doc.add(new DoubleDocValuesField("foo", 7.0));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(1, values.nextDoc());
-        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-        assertEquals(2, values.nextDoc());
-        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(7.0, Double.longBitsToDouble(values.longValue()), 0.0);
-        assertEquals(1, values.nextDoc());
-        assertEquals(18.0, Double.longBitsToDouble(values.longValue()), 0.0);
-      }
-      assertEquals(NO_MORE_DOCS, values.nextDoc());
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
-  public void testMissingMultiValuedDoubleLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.DOUBLE, reverse);
-      sortField.setMissingValue(Double.POSITIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(18.0)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(8262.0)));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
-
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      w.addDocument(doc);
-      w.commit();
-
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.0)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(7.87)));
-      w.addDocument(doc);
-      w.forceMerge(1);
-
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
-  }
-
   public void testBasicFloat() throws Exception {
     Directory dir = newDirectory();
     IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
@@ -1484,204 +1322,164 @@ public class TestIndexSorting extends LuceneTestCase {
   }
 
   public void testMissingFloatFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
-      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new FloatDocValuesField("foo", 18.0f));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new FloatDocValuesField("foo", 18.0f));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      w.addDocument(new Document());
-      w.commit();
+    // missing
+    w.addDocument(new Document());
+    w.commit();
 
-      doc = new Document();
-      doc.add(new FloatDocValuesField("foo", 7.0f));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new FloatDocValuesField("foo", 7.0f));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-        assertEquals(1, values.nextDoc());
-        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-      } else {
-        assertEquals(1, values.nextDoc());
-        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-        assertEquals(2, values.nextDoc());
-        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(1, values.nextDoc());
+    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+    assertEquals(2, values.nextDoc());
+    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedFloatFirst() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
-      sortField.setMissingValue(Float.NEGATIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
+    sortField.setMissingValue(Float.NEGATIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingFloatLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortField("foo", SortField.Type.FLOAT, reverse);
-      sortField.setMissingValue(Float.POSITIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new FloatDocValuesField("foo", 18.0f));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortField("foo", SortField.Type.FLOAT);
+    sortField.setMissingValue(Float.POSITIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new FloatDocValuesField("foo", 18.0f));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      w.addDocument(new Document());
-      w.commit();
+    // missing
+    w.addDocument(new Document());
+    w.commit();
 
-      doc = new Document();
-      doc.add(new FloatDocValuesField("foo", 7.0f));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new FloatDocValuesField("foo", 7.0f));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("foo");
-      if (reverse) {
-        assertEquals(1, values.nextDoc());
-        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-        assertEquals(2, values.nextDoc());
-        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-        assertEquals(1, values.nextDoc());
-        assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
-      }
-      assertEquals(NO_MORE_DOCS, values.nextDoc());
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("foo");
+    assertEquals(0, values.nextDoc());
+    assertEquals(7.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+    assertEquals(1, values.nextDoc());
+    assertEquals(18.0f, Float.intBitsToFloat((int) values.longValue()), 0.0f);
+    assertEquals(NO_MORE_DOCS, values.nextDoc());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testMissingMultiValuedFloatLast() throws Exception {
-    for (boolean reverse : new boolean[] {true, false}) {
-      Directory dir = newDirectory();
-      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
-      SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT, reverse);
-      sortField.setMissingValue(Float.POSITIVE_INFINITY);
-      Sort indexSort = new Sort(sortField);
-      iwc.setIndexSort(indexSort);
-      IndexWriter w = new IndexWriter(dir, iwc);
-      Document doc = new Document();
-      doc.add(new NumericDocValuesField("id", 2));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
-      w.addDocument(doc);
-      // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
-      w.commit();
+    Directory dir = newDirectory();
+    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+    SortField sortField = new SortedNumericSortField("foo", SortField.Type.FLOAT);
+    sortField.setMissingValue(Float.POSITIVE_INFINITY);
+    Sort indexSort = new Sort(sortField);
+    iwc.setIndexSort(indexSort);
+    IndexWriter w = new IndexWriter(dir, iwc);
+    Document doc = new Document();
+    doc.add(new NumericDocValuesField("id", 2));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(726.0f)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(18.0f)));
+    w.addDocument(doc);
+    // so we get more than one segment, so that forceMerge actually does merge, since we only get a sorted segment by merging:
+    w.commit();
 
-      // missing
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 3));
-      w.addDocument(doc);
-      w.commit();
+    // missing
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 3));
+    w.addDocument(doc);
+    w.commit();
 
-      doc = new Document();
-      doc.add(new NumericDocValuesField("id", 1));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
-      doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
-      w.addDocument(doc);
-      w.forceMerge(1);
+    doc = new Document();
+    doc.add(new NumericDocValuesField("id", 1));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(12.67f)));
+    doc.add(new SortedNumericDocValuesField("foo", NumericUtils.floatToSortableInt(7.0f)));
+    w.addDocument(doc);
+    w.forceMerge(1);
 
-      DirectoryReader r = DirectoryReader.open(w);
-      LeafReader leaf = getOnlyLeafReader(r);
-      assertEquals(3, leaf.maxDoc());
-      NumericDocValues values = leaf.getNumericDocValues("id");
-      if (reverse) {
-        assertEquals(0, values.nextDoc());
-        assertEquals(3, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(1, values.longValue());
-      } else {
-        assertEquals(0, values.nextDoc());
-        assertEquals(1, values.longValue());
-        assertEquals(1, values.nextDoc());
-        assertEquals(2, values.longValue());
-        assertEquals(2, values.nextDoc());
-        assertEquals(3, values.longValue());
-      }
-      r.close();
-      w.close();
-      dir.close();
-    }
+    DirectoryReader r = DirectoryReader.open(w);
+    LeafReader leaf = getOnlyLeafReader(r);
+    assertEquals(3, leaf.maxDoc());
+    NumericDocValues values = leaf.getNumericDocValues("id");
+    assertEquals(0, values.nextDoc());
+    assertEquals(1, values.longValue());
+    assertEquals(1, values.nextDoc());
+    assertEquals(2, values.longValue());
+    assertEquals(2, values.nextDoc());
+    assertEquals(3, values.longValue());
+    r.close();
+    w.close();
+    dir.close();
   }
 
   public void testRandom1() throws IOException {


[22/47] lucene-solr:jira/http2: SOLR-12801: Raise timeout in CollectionsAPIAsyncDistributedZkTest.

Posted by da...@apache.org.
SOLR-12801: Raise timeout in CollectionsAPIAsyncDistributedZkTest.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1d5ae985
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1d5ae985
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1d5ae985

Branch: refs/heads/jira/http2
Commit: 1d5ae9857fc3bb267dcbfee7d7de1d2f3f1c117e
Parents: 5d52980
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 18:05:23 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 .../api/collections/CollectionsAPIAsyncDistributedZkTest.java      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1d5ae985/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
index b81b956..953fad0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
@@ -55,7 +55,7 @@ import org.slf4j.LoggerFactory;
 @Slow
 public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
 
-  private static final int MAX_TIMEOUT_SECONDS = 60;
+  private static final int MAX_TIMEOUT_SECONDS = 90;
   
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 


[09/47] lucene-solr:jira/http2: LUCENE-8592: switch the corrupted sorted index to a 7.6 version (instead of 7x).

Posted by da...@apache.org.
LUCENE-8592: switch the corrupted sorted index to a 7.6 version (instead of 7x).


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df84a3c9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df84a3c9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df84a3c9

Branch: refs/heads/jira/http2
Commit: df84a3c9815a85aa140e5013b44488f45de9a203
Parents: 9f29ed0
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Dec 7 19:29:40 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Dec 7 19:29:40 2018 +0100

----------------------------------------------------------------------
 .../lucene/index/sorted-invalid.7.5.0.zip       | Bin 468799 -> 908443 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df84a3c9/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
----------------------------------------------------------------------
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip
index 30c68a3..615afdf 100644
Binary files a/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip and b/lucene/backward-codecs/src/test/org/apache/lucene/index/sorted-invalid.7.5.0.zip differ


[19/47] lucene-solr:jira/http2: SOLR-12988: Disable SSL on Java 11/12 for now.

Posted by da...@apache.org.
SOLR-12988: Disable SSL on Java 11/12 for now.


Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3eb2612e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3eb2612e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3eb2612e

Branch: refs/heads/jira/http2
Commit: 3eb2612e675c5b85452e78123745e68475f7dd40
Parents: 0aef561
Author: markrmiller <ma...@apache.org>
Authored: Sun Dec 9 14:45:35 2018 -0600
Committer: markrmiller <ma...@apache.org>
Committed: Sun Dec 9 18:31:28 2018 -0600

----------------------------------------------------------------------
 lucene/core/src/java/org/apache/lucene/util/Constants.java  | 1 +
 .../org/apache/solr/client/solrj/embedded/SSLConfig.java    | 9 ++++++++-
 .../org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java  | 2 ++
 .../test/org/apache/solr/cloud/TestSSLRandomization.java    | 2 ++
 .../client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java | 4 ++++
 .../src/java/org/apache/solr/util/SSLTestConfig.java        | 3 ++-
 6 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/lucene/core/src/java/org/apache/lucene/util/Constants.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/Constants.java b/lucene/core/src/java/org/apache/lucene/util/Constants.java
index 007ef85..653c1e3 100644
--- a/lucene/core/src/java/org/apache/lucene/util/Constants.java
+++ b/lucene/core/src/java/org/apache/lucene/util/Constants.java
@@ -87,6 +87,7 @@ public final class Constants {
 
   public static final boolean JRE_IS_MINIMUM_JAVA8 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 8);
   public static final boolean JRE_IS_MINIMUM_JAVA9 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9);
+  public static final boolean JRE_IS_MINIMUM_JAVA11 = JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 11);
 
   /**
    * This is the internal Lucene version, including bugfix versions, recorded into each segment.

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
index 62c9024..e641816 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/SSLConfig.java
@@ -16,6 +16,7 @@
  */
 package org.apache.solr.client.solrj.embedded;
 
+import org.apache.lucene.util.Constants;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 
 /** 
@@ -34,7 +35,13 @@ public class SSLConfig {
 
   /** NOTE: all other settings are ignored if useSSL is false; trustStore settings are ignored if clientAuth is false */
   public SSLConfig(boolean useSSL, boolean clientAuth, String keyStore, String keyStorePassword, String trustStore, String trustStorePassword) {
-    this.useSsl = useSSL;
+    // @AwaitsFix: SOLR-12988 - ssl issues on Java 11/12
+    if (Constants.JRE_IS_MINIMUM_JAVA11) {
+      this.useSsl = false;
+    } else {
+      this.useSsl = useSSL;
+    }
+
     this.clientAuth = clientAuth;
     this.keyStore = keyStore;
     this.keyStorePassword = keyStorePassword;

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
index 97a2de0..37080e1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudClusterSSL.java
@@ -85,6 +85,8 @@ public class TestMiniSolrCloudClusterSSL extends SolrTestCaseJ4 {
   
   @Before
   public void before() {
+    assumeFalse("@AwaitsFix: SOLR-12988 - ssl issues on Java 11/12", Constants.JRE_IS_MINIMUM_JAVA11);
+    
     // undo the randomization of our super class
     log.info("NOTE: This Test ignores the randomized SSL & clientAuth settings selected by base class");
     HttpClientUtil.resetHttpClientBuilder(); // also resets SchemaRegistryProvider

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java b/solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
index e846f73..1241189 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSSLRandomization.java
@@ -19,6 +19,7 @@ package org.apache.solr.cloud;
 import java.lang.invoke.MethodHandles;
 import java.util.Arrays;
 
+import org.apache.lucene.util.Constants;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.util.SSLTestConfig;
 import org.apache.solr.util.RandomizeSSL;
@@ -43,6 +44,7 @@ public class TestSSLRandomization extends SolrCloudTestCase {
 
   @BeforeClass
   public static void createMiniSolrCloudCluster() throws Exception {
+    assumeFalse("@AwaitsFix: SOLR-12988 - ssl issues on Java 11/12", Constants.JRE_IS_MINIMUM_JAVA11);
     configureCluster(TestMiniSolrCloudClusterSSL.NUM_SERVERS).configure();
   }
   

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
index 3a39c67..3b59049 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpSolrClientSSLAuthConPoolTest.java
@@ -20,14 +20,18 @@ package org.apache.solr.client.solrj.impl;
 import java.net.URL;
 import java.util.Arrays;
 
+import org.apache.lucene.util.Constants;
 import org.apache.solr.util.RandomizeSSL;
 import org.junit.BeforeClass;
 
 @RandomizeSSL(1.0)
+
 public class HttpSolrClientSSLAuthConPoolTest extends HttpSolrClientConPoolTest {
 
     @BeforeClass
     public static void checkUrls() throws Exception {
+      assumeFalse("@AwaitsFix: SOLR-12988 - ssl issues on Java 11/12", Constants.JRE_IS_MINIMUM_JAVA11);
+      
       URL[] urls = new URL[] {
           jetty.getBaseUrl(), yetty.getBaseUrl() 
       };

http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3eb2612e/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java b/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
index 3b03f6e..307f547 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/SSLTestConfig.java
@@ -98,8 +98,9 @@ public class SSLTestConfig extends SSLConfig {
    */
   public SSLTestConfig(boolean useSSL, boolean clientAuth, boolean checkPeerName) {
     super(useSSL, clientAuth, null, TEST_KEYSTORE_PASSWORD, null, TEST_KEYSTORE_PASSWORD);
+
     this.checkPeerName = checkPeerName;
-    
+
     final String resourceName = checkPeerName
       ? TEST_KEYSTORE_LOCALHOST_RESOURCE : TEST_KEYSTORE_BOGUSHOST_RESOURCE;
     trustStore = keyStore = Resource.newClassPathResource(resourceName);