You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2013/11/28 22:00:29 UTC

svn commit: r1546457 - in /lucene/dev/trunk/solr: ./ contrib/analysis-extras/src/java/org/apache/solr/schema/ core/src/java/org/apache/solr/handler/component/ core/src/java/org/apache/solr/schema/ core/src/test-files/solr/collection1/conf/ core/src/tes...

Author: sarowe
Date: Thu Nov 28 21:00:28 2013
New Revision: 1546457

URL: http://svn.apache.org/r1546457
Log:
SOLR-5354: Distributed sort is broken with CUSTOM FieldType

Added:
    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml   (with props)
    lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml   (with props)
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java   (with props)
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java   (with props)
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java   (with props)
    lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java   (with props)
Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java
    lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
    lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Nov 28 21:00:28 2013
@@ -159,6 +159,9 @@ Bug Fixes
 
 * SOLR-5494: CoreContainer#remove throws NPE rather than returning null when
   a SolrCore does not exist in core discovery mode. (Mark Miller)
+  
+* SOLR-5354: Distributed sort is broken with CUSTOM FieldType. 
+  (Steve Rowe, hossman, Jessica Cheng)  
 
 Optimizations
 ----------------------

Modified: lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (original)
+++ lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java Thu Nov 28 21:00:28 2013
@@ -43,6 +43,7 @@ import org.apache.lucene.util.Version;
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -300,4 +301,23 @@ public class ICUCollationField extends F
       return Collections.singletonList(createField(field, value, boost));
     }
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final BytesRef val = (BytesRef)value;
+    return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final String val = (String)value;
+    final byte[] bytes = Base64.base64ToByteArray(val);
+    return new BytesRef(bytes);
+  }
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java Thu Nov 28 21:00:28 2013
@@ -34,7 +34,6 @@ import org.apache.lucene.search.grouping
 import org.apache.lucene.search.grouping.TopGroups;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.common.SolrDocument;
 import org.apache.solr.common.SolrDocumentList;
@@ -47,6 +46,7 @@ import org.apache.solr.request.SolrQuery
 import org.apache.solr.response.ResultContext;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.DocIterator;
 import org.apache.solr.search.DocList;
@@ -449,7 +449,6 @@ public class QueryComponent extends Sear
   {
     SolrQueryRequest req = rb.req;
     SolrQueryResponse rsp = rb.rsp;
-    final CharsRef spare = new CharsRef();
     // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
     // currently have an option to return sort field values.  Because of this, we
     // take the documents given and re-derive the sort values.
@@ -458,7 +457,6 @@ public class QueryComponent extends Sear
       Sort sort = searcher.weightSort(rb.getSortSpec().getSort());
       SortField[] sortFields = sort==null ? new SortField[]{SortField.FIELD_SCORE} : sort.getSort();
       NamedList<Object[]> sortVals = new NamedList<Object[]>(); // order is important for the sort fields
-      Field field = new StringField("dummy", "", Field.Store.NO); // a dummy Field
       IndexReaderContext topReaderContext = searcher.getTopReaderContext();
       List<AtomicReaderContext> leaves = topReaderContext.leaves();
       AtomicReaderContext currentLeaf = null;
@@ -516,27 +514,7 @@ public class QueryComponent extends Sear
           doc -= currentLeaf.docBase;  // adjust for what segment this is in
           comparator.copy(0, doc);
           Object val = comparator.value(0);
-
-          // Sortable float, double, int, long types all just use a string
-          // comparator. For these, we need to put the type into a readable
-          // format.  One reason for this is that XML can't represent all
-          // string values (or even all unicode code points).
-          // indexedToReadable() should be a no-op and should
-          // thus be harmless anyway (for all current ways anyway)
-          if (val instanceof String) {
-            field.setStringValue((String)val);
-            val = ft.toObject(field);
-          }
-
-          // Must do the same conversion when sorting by a
-          // String field in Lucene, which returns the terms
-          // data as BytesRef:
-          if (val instanceof BytesRef) {
-            UnicodeUtil.UTF8toUTF16((BytesRef)val, spare);
-            field.setStringValue(spare.toString());
-            val = ft.toObject(field);
-          }
-
+          if (null != ft) val = ft.marshalSortValue(val); 
           vals[position] = val;
         }
 
@@ -778,7 +756,8 @@ public class QueryComponent extends Sear
         sortFields = new SortField[]{SortField.FIELD_SCORE};
       }
  
-      SchemaField uniqueKeyField = rb.req.getSchema().getUniqueKeyField();
+      IndexSchema schema = rb.req.getSchema();
+      SchemaField uniqueKeyField = schema.getUniqueKeyField();
 
 
       // id to shard mapping, to eliminate any accidental dups
@@ -787,7 +766,7 @@ public class QueryComponent extends Sear
       // Merge the docs via a priority queue so we don't have to sort *all* of the
       // documents... we only need to order the top (rows+start)
       ShardFieldSortedHitQueue queue;
-      queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount());
+      queue = new ShardFieldSortedHitQueue(sortFields, ss.getOffset() + ss.getCount(), rb.req.getSearcher());
 
       NamedList<Object> shardInfo = null;
       if(rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
@@ -886,7 +865,7 @@ public class QueryComponent extends Sear
             }
           }
 
-          shardDoc.sortFieldValues = sortFieldValues;
+          shardDoc.sortFieldValues = unmarshalSortValues(sortFieldValues, schema);
 
           queue.insertWithOverflow(shardDoc);
         } // end for-each-doc-in-response
@@ -928,6 +907,26 @@ public class QueryComponent extends Sear
       }
   }
 
+  private NamedList unmarshalSortValues(NamedList sortFieldValues, IndexSchema schema) {
+    NamedList unmarshalledSortValsPerField = new NamedList();
+    for (int fieldNum = 0 ; fieldNum < sortFieldValues.size() ; ++fieldNum) {
+      String fieldName = sortFieldValues.getName(fieldNum);
+      SchemaField field = schema.getFieldOrNull(fieldName);
+      List sortVals = (List)sortFieldValues.getVal(fieldNum);
+      if (null == field) {
+        unmarshalledSortValsPerField.add(fieldName, sortVals);
+      } else {
+        FieldType fieldType = field.getType();
+        List unmarshalledSortVals = new ArrayList();
+        for (Object sortVal : sortVals) {
+          unmarshalledSortVals.add(fieldType.unmarshalSortValue(sortVal));
+        }
+        unmarshalledSortValsPerField.add(fieldName, unmarshalledSortVals);
+      }
+    }
+    return unmarshalledSortValsPerField;
+  }
+
   private void createRetrieveDocs(ResponseBuilder rb) {
 
     // TODO: in a system with nTiers > 2, we could be passed "ids" here

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/ShardDoc.java Thu Nov 28 21:00:28 2013
@@ -16,18 +16,21 @@
  */
 package org.apache.solr.handler.component;
 
-import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.common.SolrException;
 import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.MissingStringLastComparatorSource;
+import org.apache.solr.search.SolrIndexSearcher;
 
-import java.text.Collator;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
-import java.util.Locale;
+
+import static org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR;
 
 public class ShardDoc extends FieldDoc {
   public String shard;
@@ -101,7 +104,7 @@ public class ShardDoc extends FieldDoc {
 class ShardFieldSortedHitQueue extends PriorityQueue<ShardDoc> {
 
   /** Stores a comparator corresponding to each field being sorted by */
-  protected Comparator[] comparators;
+  protected Comparator<ShardDoc>[] comparators;
 
   /** Stores the sort criteria being used. */
   protected SortField[] fields;
@@ -109,9 +112,10 @@ class ShardFieldSortedHitQueue extends P
   /** The order of these fieldNames should correspond to the order of sort field values retrieved from the shard */
   protected List<String> fieldNames = new ArrayList<String>();
 
-  public ShardFieldSortedHitQueue(SortField[] fields, int size) {
+  public ShardFieldSortedHitQueue(SortField[] fields, int size, IndexSearcher searcher) {
     super(size);
     final int n = fields.length;
+    //noinspection unchecked
     comparators = new Comparator[n];
     this.fields = new SortField[n];
     for (int i = 0; i < n; ++i) {
@@ -123,8 +127,7 @@ class ShardFieldSortedHitQueue extends P
       }
 
       String fieldname = fields[i].getField();
-      comparators[i] = getCachedComparator(fieldname, fields[i]
-          .getType(), fields[i].getComparatorSource());
+      comparators[i] = getCachedComparator(fields[i], searcher);
 
      if (fields[i].getType() == SortField.Type.STRING) {
         this.fields[i] = new SortField(fieldname, SortField.Type.STRING,
@@ -169,47 +172,36 @@ class ShardFieldSortedHitQueue extends P
     return c < 0;
   }
 
-  Comparator getCachedComparator(String fieldname, SortField.Type type, FieldComparatorSource factory) {
-    Comparator comparator = null;
-    switch (type) {
-    case SCORE:
-      comparator = comparatorScore(fieldname);
-      break;
-    case STRING:
-      comparator = comparatorNatural(fieldname);
-      break;
-    case CUSTOM:
-      if (factory instanceof MissingStringLastComparatorSource){
-        comparator = comparatorMissingStringLast(fieldname);
-      } else {
-        // TODO: support other types such as random... is there a way to
-        // support generically?  Perhaps just comparing Object
-        comparator = comparatorNatural(fieldname);
-        // throw new RuntimeException("Custom sort not supported factory is "+factory.getClass());
-      }
-      break;
-    case DOC:
-      // TODO: we can support this!
-      throw new RuntimeException("Doc sort not supported");
-    default:
-      comparator = comparatorNatural(fieldname);
-      break;
-    }
-    return comparator;
-  }
-
-  class ShardComparator implements Comparator {
-    String fieldName;
-    int fieldNum;
-    public ShardComparator(String fieldName) {
-      this.fieldName = fieldName;
-      this.fieldNum=0;
+  Comparator<ShardDoc> getCachedComparator(SortField sortField, IndexSearcher searcher) {
+    SortField.Type type = sortField.getType();
+    if (type == SortField.Type.SCORE) {
+      return comparatorScore();
+    } else if (type == SortField.Type.REWRITEABLE) {
+      try {
+        sortField = sortField.rewrite(searcher);
+      } catch (IOException e) {
+        throw new SolrException(SERVER_ERROR, "Exception rewriting sort field " + sortField, e);
+      }
+    }
+    return comparatorFieldComparator(sortField);
+  }
+
+  abstract class ShardComparator implements Comparator<ShardDoc> {
+    final SortField sortField;
+    final String fieldName;
+    final int fieldNum;
+
+    public ShardComparator(SortField sortField) {
+      this.sortField = sortField;
+      this.fieldName = sortField.getField();
+      int fieldNum = 0;
       for (int i=0; i<fieldNames.size(); i++) {
         if (fieldNames.get(i).equals(fieldName)) {
-          this.fieldNum = i;
+          fieldNum = i;
           break;
         }
       }
+      this.fieldNum = fieldNum;
     }
 
     Object sortVal(ShardDoc shardDoc) {
@@ -217,22 +209,14 @@ class ShardFieldSortedHitQueue extends P
       List lst = (List)shardDoc.sortFieldValues.getVal(fieldNum);
       return lst.get(shardDoc.orderInShard);
     }
-
-    @Override
-    public int compare(Object o1, Object o2) {
-      return 0;
-    }
   }
 
-  static Comparator comparatorScore(final String fieldName) {
-    return new Comparator() {
+  static Comparator<ShardDoc> comparatorScore() {
+    return new Comparator<ShardDoc>() {
       @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc e1 = (ShardDoc) o1;
-        ShardDoc e2 = (ShardDoc) o2;
-
-        final float f1 = e1.score;
-        final float f2 = e2.score;
+      public final int compare(final ShardDoc o1, final ShardDoc o2) {
+        final float f1 = o1.score;
+        final float f2 = o2.score;
         if (f1 < f2)
           return -1;
         if (f1 > f2)
@@ -242,71 +226,24 @@ class ShardFieldSortedHitQueue extends P
     };
   }
 
-  // The lucene natural sort ordering corresponds to numeric
-  // and string natural sort orderings (ascending).  Since
-  // the PriorityQueue keeps the biggest elements by default,
-  // we need to reverse the natural compare ordering so that the
-  // smallest elements are kept instead of the largest... hence
-  // the negative sign on the final compareTo().
-  Comparator comparatorNatural(String fieldName) {
-    return new ShardComparator(fieldName) {
-      @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return 1;
-        if(v2==null)
-          return -1;
-        return -v1.compareTo(v2);
-      }
-    };
-  }
-
-
-  Comparator comparatorStringLocale(final String fieldName,
-      Locale locale) {
-    final Collator collator = Collator.getInstance(locale);
-    return new ShardComparator(fieldName) {
-      @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return 1;
-        if(v2==null)
-          return -1;
-        return -collator.compare(v1,v2);
-      }
-    };
-  }
-
+  Comparator<ShardDoc> comparatorFieldComparator(SortField sortField) {
+    final FieldComparator fieldComparator;
+    try {
+      fieldComparator = sortField.getComparator(0, 0);
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to get FieldComparator for sortField " + sortField);
+    }
 
-  Comparator comparatorMissingStringLast(final String fieldName) {
-     return new ShardComparator(fieldName) {
+    return new ShardComparator(sortField) {
+      // Since the PriorityQueue keeps the biggest elements by default,
+      // we need to reverse the field compare ordering so that the
+      // smallest elements are kept instead of the largest... hence
+      // the negative sign.
       @Override
-      public final int compare(final Object o1, final Object o2) {
-        ShardDoc sd1 = (ShardDoc) o1;
-        ShardDoc sd2 = (ShardDoc) o2;
-        Comparable v1 = (Comparable)sortVal(sd1);
-        Comparable v2 = (Comparable)sortVal(sd2);
-        if (v1==v2)
-          return 0;
-        if (v1==null)
-          return -1;
-        if(v2==null)
-          return 1;
-        return -v1.compareTo(v2);
+      public int compare(final ShardDoc o1, final ShardDoc o2) {
+        //noinspection unchecked
+        return -fieldComparator.compareValues(sortVal(o1), sortVal(o2));
       }
     };
   }
-
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/CollationField.java Thu Nov 28 21:00:28 2013
@@ -47,6 +47,7 @@ import org.apache.lucene.util.Version;
 import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.Base64;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -275,4 +276,23 @@ public class CollationField extends Fiel
       return Collections.singletonList(createField(field, value, boost));
     }
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final BytesRef val = (BytesRef)value;
+    return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final String val = (String)value;
+    final byte[] bytes = Base64.base64ToByteArray(val);
+    return new BytesRef(bytes);
+  }
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java Thu Nov 28 21:00:28 2013
@@ -932,4 +932,20 @@ public abstract class FieldType extends 
     }
     return analyzerProps;
   }
+  
+  /** 
+   * Convert a value used by the FieldComparator for this FieldType's SortField
+   * into a marshalable value for distributed sorting.
+   */
+  public Object marshalSortValue(Object value) {
+    return value;
+  }
+  
+  /**
+   * Convert a value marshaled via {@link #marshalSortValue} back 
+   * into a value usable by the FieldComparator for this FieldType's SortField
+   */
+  public Object unmarshalSortValue(Object value) {
+    return value;
+  }
 }

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableDoubleField.java Thu Nov 28 21:00:28 2013
@@ -100,6 +100,27 @@ public class SortableDoubleField extends
     String sval = f.stringValue();
     writer.writeDouble(name, NumberUtils.SortableStr2double(sval));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2double(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.double2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 class SortableDoubleFieldSource extends FieldCacheSource {

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableFloatField.java Thu Nov 28 21:00:28 2013
@@ -101,6 +101,27 @@ public class SortableFloatField extends 
     String sval = f.stringValue();
     writer.writeFloat(name, NumberUtils.SortableStr2float(sval));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2float(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.float2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableIntField.java Thu Nov 28 21:00:28 2013
@@ -104,6 +104,27 @@ public class SortableIntField extends Pr
     String sval = f.stringValue();
     writer.writeInt(name, NumberUtils.SortableStr2int(sval,0,sval.length()));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) { 
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2int(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.int2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SortableLongField.java Thu Nov 28 21:00:28 2013
@@ -100,6 +100,27 @@ public class SortableLongField extends P
     String sval = f.stringValue();
     writer.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef chars = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, chars);
+    return NumberUtils.SortableStr2long(chars.toString());
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    String sortableString = NumberUtils.long2sortableStr(value.toString());
+    BytesRef bytes = new BytesRef();
+    UnicodeUtil.UTF16toUTF8(sortableString, 0, sortableString.length(), bytes);
+    return bytes;
+  }
 }
 
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/StrField.java Thu Nov 28 21:00:28 2013
@@ -29,6 +29,8 @@ import org.apache.lucene.index.StorableF
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
 
@@ -81,6 +83,27 @@ public class StrField extends PrimitiveF
   @Override
   public void checkSchemaField(SchemaField field) {
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef spare = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+    return spare.toString();
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    BytesRef spare = new BytesRef();
+    String stringVal = (String)value;
+    UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+    return spare;
+  }
 }
 
 

Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/TextField.java Thu Nov 28 21:00:28 2013
@@ -23,7 +23,9 @@ import org.apache.lucene.index.StorableF
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.QueryBuilder;
+import org.apache.lucene.util.UnicodeUtil;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.response.TextResponseWriter;
 import org.apache.solr.search.QParser;
@@ -165,4 +167,25 @@ public class TextField extends FieldType
   public boolean isExplicitMultiTermAnalyzer() {
     return isExplicitMultiTermAnalyzer;
   }
+
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    CharsRef spare = new CharsRef();
+    UnicodeUtil.UTF8toUTF16((BytesRef)value, spare);
+    return spare.toString();
+  }
+
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    BytesRef spare = new BytesRef();
+    String stringVal = (String)value;
+    UnicodeUtil.UTF16toUTF8(stringVal, 0, stringVal.length(), spare);
+    return spare;
+  }
 }

Added: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml (added)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml Thu Nov 28 21:00:28 2013
@@ -0,0 +1,42 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="test-custom-field-sort" version="1.5">
+  <types>
+    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldtype name="string" class="solr.StrField" sortMissingLast="true"/>
+    <fieldtype name="text" class="solr.TextField">
+      <analyzer>
+        <tokenizer class="solr.StandardTokenizerFactory"/>
+      </analyzer>
+    </fieldtype>
+    <fieldType class="org.apache.solr.schema.SortableBinaryField" name="sortable_binary"/>
+  </types>
+  <fields>
+    <field name="id" type="int" indexed="true" stored="true" multiValued="false" required="false"/>
+    <field name="_version_" type="long" indexed="true" stored="true" multiValued="false"/>
+    <field name="signatureField" type="string" indexed="true" stored="false"/>
+    <field name="text" type="text" indexed="true" stored="false"/>
+    <field name="payload" type="sortable_binary" indexed="false" 
+           stored="true"  multiValued="false"    docValues="true" required="true"/>
+    <dynamicField name="*_sS" type="string"  indexed="false" stored="true"/>
+  </fields>
+  <defaultSearchField>text</defaultSearchField>
+  <uniqueKey>id</uniqueKey>
+</schema>

Added: lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml (added)
+++ lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml Thu Nov 28 21:00:28 2013
@@ -0,0 +1,83 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="test-distributed-missing-sort" version="1.5">
+  <types>
+    <fieldType name="int"       class="solr.TrieIntField"     precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tint"      class="solr.TrieIntField"     precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldType name="long"      class="solr.TrieLongField"    precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tlong"     class="solr.TrieLongField"    precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldType name="float"     class="solr.TrieFloatField"   precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tfloat"    class="solr.TrieFloatField"   precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldType name="double"    class="solr.TrieDoubleField"  precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+    <fieldType name="tdouble"   class="solr.TrieDoubleField"  precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+
+    <fieldtype name="date"      class="solr.TrieDateField"    precisionStep="0"/>
+    <fieldtype name="tdate"     class="solr.TrieDateField"    precisionStep="6"/>
+
+    <fieldtype name="boolean"   class="solr.BoolField"        sortMissingLast="true"/>
+
+    <fieldtype name="sint_ml"   class="solr.SortableIntField" sortMissingLast="true"/>
+    <fieldtype name="sint_mf"   class="solr.SortableIntField" sortMissingFirst="true"/>
+
+    <fieldType name="long_ml"   class="solr.TrieLongField"    precisionStep="0" omitNorms="true" positionIncrementGap="0" sortMissingLast="true"/>
+    <fieldType name="long_mf"   class="solr.TrieLongField"    precisionStep="0" omitNorms="true" positionIncrementGap="0" sortMissingFirst="true"/>
+
+    <fieldtype name="string_ml" class="solr.StrField"         sortMissingLast="true"/>
+    <fieldtype name="string_mf" class="solr.StrField"         sortMissingFirst="true"/>
+  </types>
+
+  <fields>
+    <field name="id"              type="int"       indexed="true"  stored="true"  multiValued="false" required="false"/>
+    <field name="_version_"       type="long"      indexed="true"  stored="true"  multiValued="false"/>
+    <field name="signatureField"  type="string_ml" indexed="true"  stored="false"/>
+
+    <dynamicField name="*_ti1"    type="tint"      indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_si_ml"  type="sint_ml"   indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_si_mf"  type="sint_mf"   indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_l1_ml"  type="long_ml"   indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_l1_mf"  type="long_mf"   indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_l1"     type="long"      indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_tl1"    type="tlong"     indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_f"      type="float"     indexed="true"  stored="true"/>
+    <dynamicField name="*_f1"     type="float"     indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_tf1"    type="tfloat"    indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_d"      type="double"    indexed="true"  stored="true"/>
+    <dynamicField name="*_d1"     type="double"    indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_td1"    type="tdouble"   indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_dt1"    type="date"      indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_tdt1"   type="tdate"     indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_b"      type="boolean"   indexed="true"  stored="true"/>
+
+    <dynamicField name="*_s1_ml"  type="string_ml" indexed="true"  stored="true"  multiValued="false"/>
+    <dynamicField name="*_s1_mf"  type="string_mf" indexed="true"  stored="true"  multiValued="false"/>
+
+    <dynamicField name="*_mfacet" type="string_ml" indexed="true"  stored="false" multiValued="true" />
+    <dynamicField name="*_sS"     type="string_ml" indexed="false" stored="true"/>
+  </fields>
+  <uniqueKey>id</uniqueKey>
+</schema>

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedMissingSort.java Thu Nov 28 21:00:28 2013
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.client.solrj.response.QueryResponse;
+
+/**
+ * Tests sortMissingFirst and sortMissingLast in distributed sort
+ */
+@Slow
+public class TestDistributedMissingSort extends BaseDistributedSearchTestCase {
+
+  public TestDistributedMissingSort() {
+    schemaString = "schema-distributed-missing-sort.xml";
+  }
+  
+  String sint1_ml = "one_si_ml";    // SortableIntField, sortMissingLast=true, multiValued=false
+  String sint1_mf = "two_si_mf";    // SortableIntField, sortMissingFirst=true, multiValued=false
+  String long1_ml = "three_l1_ml";  // TrieLongField, sortMissingLast=true, multiValued=false
+  String long1_mf = "four_l1_mf";   // TrieLongField, sortMissingFirst=true, multiValued=false
+  String string1_ml = "five_s1_ml"; // StringField, sortMissingLast=true, multiValued=false
+  String string1_mf = "six_s1_mf";  // StringField, sortMissingFirst=true, multiValued=false
+
+  @Override
+  public void doTest() throws Exception {
+    index();
+    testSortMissingLast();
+    testSortMissingFirst();
+  }
+
+  private void index() throws Exception {
+    del("*:*");
+    indexr(id,1, sint1_ml, 100, sint1_mf, 100, long1_ml, 100, long1_mf, 100,
+        "foo_f", 1.414f, "foo_b", "true", "foo_d", 1.414d,
+        string1_ml, "DE", string1_mf, "DE");
+    indexr(id,2, sint1_ml, 50, sint1_mf, 50, long1_ml, 50, long1_mf, 50,
+        string1_ml, "ABC", string1_mf, "ABC");
+    indexr(id,3, sint1_ml, 2, sint1_mf, 2, long1_ml, 2, long1_mf, 2,
+        string1_ml, "HIJK", string1_mf, "HIJK");
+    indexr(id,4, sint1_ml, -100, sint1_mf, -100, long1_ml, -101, long1_mf, -101,
+        string1_ml, "L  M", string1_mf, "L  M");
+    indexr(id,5, sint1_ml, 500, sint1_mf, 500, long1_ml, 500, long1_mf, 500,
+        string1_ml, "YB", string1_mf, "YB");
+    indexr(id,6, sint1_ml, -600, sint1_mf, -600, long1_ml, -600, long1_mf, -600,
+        string1_ml, "WX", string1_mf, "WX");
+    indexr(id,7, sint1_ml, 123, sint1_mf, 123, long1_ml, 123, long1_mf, 123,
+        string1_ml, "N", string1_mf, "N");
+    indexr(id,8, sint1_ml, 876, sint1_mf, 876, long1_ml, 876, long1_mf, 876,
+        string1_ml, "QRS", string1_mf, "QRS");
+    indexr(id,9, sint1_ml, 7, sint1_mf, 7, long1_ml, 7, long1_mf, 7,
+        string1_ml, "P", string1_mf, "P");
+
+    commit();  // try to ensure there's more than one segment
+
+    indexr(id,10, sint1_ml, 4321, sint1_mf, 4321, long1_ml, 4321, long1_mf, 4321,
+        string1_ml, "O", string1_mf, "O");
+    indexr(id,11, sint1_ml, -987, sint1_mf, -987, long1_ml, -987, long1_mf, -987,
+        string1_ml, "YA", string1_mf, "YA");
+    indexr(id,12, sint1_ml, 379, sint1_mf, 379, long1_ml, 379, long1_mf, 379,
+        string1_ml, "TUV", string1_mf, "TUV");
+    indexr(id,13, sint1_ml, 232, sint1_mf, 232, long1_ml, 232, long1_mf, 232,
+        string1_ml, "F G", string1_mf, "F G");
+
+    indexr(id, 14, "SubjectTerms_mfacet", new String[]  {"mathematical models", "mathematical analysis"});
+    indexr(id, 15, "SubjectTerms_mfacet", new String[]  {"test 1", "test 2", "test3"});
+    indexr(id, 16, "SubjectTerms_mfacet", new String[]  {"test 1", "test 2", "test3"});
+    String[] vals = new String[100];
+    for (int i=0; i<100; i++) {
+      vals[i] = "test " + i;
+    }
+    indexr(id, 17, "SubjectTerms_mfacet", vals);
+
+    for (int i=100; i<150; i++) {
+      indexr(id, i);
+    }
+
+    commit();
+
+    handle.clear();
+    handle.put("QTime", SKIPVAL);
+    handle.put("timestamp", SKIPVAL);
+    handle.put("_version_", SKIPVAL); // not a cloud test, but may use updateLog
+  }
+  
+  private void testSortMissingLast() throws Exception {
+    // id field values:         1     2     3     4     5     6     7     8     9    10    11    12    13
+    // sint1_ml field values: 100    50     2  -100   500  -600   123   876     7  4321  -987   379   232
+    // sint1_ml asc sort pos:   7     6     4     3    11     2     8    12     5    13     1    10     9
+    // sint1_ml desc sort pos:  7     8    10    11     3    12     6     2     9     1    13     4     5
+
+    QueryResponse rsp = query("q","*:*", "sort", sint1_ml + " desc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11);
+
+    rsp = query("q","*:*", "sort", sint1_ml + " asc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10);
+
+    rsp = query("q","*:*", "sort", sint1_ml + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11,
+        // missing field sint1_ml="a_si", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149);
+
+    rsp = query("q","*:*", "sort", sint1_ml + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10,
+        // missing field sint1_ml="a_si", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14);
+
+    // id field values:         1     2     3     4     5     6     7     8     9    10    11    12    13
+    // long1_ml field values: 100    50     2  -100   500  -600   123   876     7  4321  -987   379   232
+    // long1_ml asc sort pos:   7     6     4     3    11     2     8    12     5    13     1    10     9
+    // long1_ml desc sort pos:  7     8    10    11     3    12     6     2     9     1    13     4     5
+
+    rsp = query("q","*:*", "sort", long1_ml + " desc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11);
+
+    rsp = query("q","*:*", "sort", long1_ml + " asc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10);
+
+    rsp = query("q","*:*", "sort", long1_ml + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11,
+        // missing field sint1_ml="a_si", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149);
+
+    rsp = query("q","*:*", "sort", long1_ml + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10,
+        // missing field sint1_ml="a_si", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14);
+
+    
+    // id field values:           1     2     3     4     5     6     7     8     9    10    11    12    13
+    // string1_ml field values:  DE   ABC  HIJK  L  M    YB    WX     N   QRS     P     O    YA   TUV   F G
+    // string1_ml asc sort pos:   2     1     4     5    13    11     6     9     8     7    12    10     3
+    // string1_ml desc sort pos: 12    13    10     9     1     3     8     5     6     7     2     4    11
+
+    rsp = query("q","*:*", "sort", string1_ml + " desc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2);
+
+    rsp = query("q","*:*", "sort", string1_ml + " asc", "rows", "13");
+    assertFieldValues(rsp.getResults(), id, 2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5);
+
+    rsp = query("q","*:*", "sort", string1_ml + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2,
+        // missing field string1_ml="a_s1", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149);
+
+    rsp = query("q","*:*", "sort", string1_ml + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5,
+        // missing field string1_ml="a_s1", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14);
+  }
+  
+  private void testSortMissingFirst() throws Exception {
+    // id field values:         1     2     3     4     5     6     7     8     9    10    11    12    13
+    // sint1_mf field values: 100    50     2  -100   500  -600   123   876     7  4321  -987   379   232
+    // sint1_mf asc sort pos:   7     6     4     3    11     2     8    12     5    13     1    10     9
+    // sint1_mf desc sort pos:  7     8    10    11     3    12     6     2     9     1    13     4     5
+
+    QueryResponse rsp = query("q","*:*", "sort", sint1_mf + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field sint1_mf="a_si_mf", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+        10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11);
+
+    rsp = query("q","*:*", "sort", sint1_mf + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field sint1_mf="a_si_mf", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14,
+        11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10);
+
+
+    // id field values:         1     2     3     4     5     6     7     8     9    10    11    12    13
+    // long1_mf field values: 100    50     2  -100   500  -600   123   876     7  4321  -987   379   232
+    // long1_mf asc sort pos:   7     6     4     3    11     2     8    12     5    13     1    10     9
+    // long1_mf desc sort pos:  7     8    10    11     3    12     6     2     9     1    13     4     5
+
+    rsp = query("q","*:*", "sort", long1_mf + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field sint1_mf="a_si_mf", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+        10, 8, 5, 12, 13, 7, 1, 2, 9, 3, 4, 6, 11);
+
+    rsp = query("q","*:*", "sort", long1_mf + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field sint1_mf="a_si_mf", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14,
+        11, 6, 4, 3, 9, 2, 1, 7, 13, 12, 5, 8, 10);
+
+    
+    // id field values:           1     2     3     4     5     6     7     8     9    10    11    12    13
+    // string1_mf field values:  DE   ABC  HIJK  L  M    YB    WX     N   QRS     P     O    YA   TUV   F G
+    // string1_mf asc sort pos:   2     1     4     5    13    11     6     9     8     7    12    10     3
+    // string1_mf desc sort pos: 12    13    10     9     1     3     8     5     6     7     2     4    11
+
+    rsp = query("q","*:*", "sort", string1_mf + " desc," + id + " asc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field string1_mf="a_s1_mf", ascending id sort
+        14, 15, 16, 17,
+        100, 101, 102, 103, 104, 105, 106, 107, 108, 109,
+        110, 111, 112, 113, 114, 115, 116, 117, 118, 119,
+        120, 121, 122, 123, 124, 125, 126, 127, 128, 129,
+        130, 131, 132, 133, 134, 135, 136, 137, 138, 139,
+        140, 141, 142, 143, 144, 145, 146, 147, 148, 149,
+        5, 11, 6, 12, 8, 9, 10, 7, 4, 3, 13, 1, 2);
+
+    rsp = query("q","*:*", "sort", string1_mf + " asc," + id + " desc", "rows", "200");
+    assertFieldValues(rsp.getResults(), id,
+        // missing field string1_mf="a_s1_mf", descending id sort
+        149, 148, 147, 146, 145, 144, 143, 142, 141, 140,
+        139, 138, 137, 136, 135, 134, 133, 132, 131, 130,
+        129, 128, 127, 126, 125, 124, 123, 122, 121, 120,
+        119, 118, 117, 116, 115, 114, 113, 112, 111, 110,
+        109, 108, 107, 106, 105, 104, 103, 102, 101, 100,
+        17, 16, 15, 14,
+        2, 1, 13, 3, 4, 7, 10, 9, 8, 12, 6, 11, 5);
+  }
+}

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/component/DistributedQueryComponentCustomSortTest.java Thu Nov 28 21:00:28 2013
@@ -0,0 +1,110 @@
+package org.apache.solr.handler.component;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.BaseDistributedSearchTestCase;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.junit.BeforeClass;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Test for QueryComponent's distributed querying
+ *
+ * @see org.apache.solr.handler.component.QueryComponent
+ */
+public class DistributedQueryComponentCustomSortTest extends BaseDistributedSearchTestCase {
+
+  public DistributedQueryComponentCustomSortTest() {
+    fixShardCount = true;
+    shardCount = 3;
+    stress = 0;
+  }
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    initCore("solrconfig.xml", "schema-custom-field.xml");
+  }
+
+  @Override
+  public void doTest() throws Exception {
+    del("*:*");
+
+    index(id, "1", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x12, 0x62, 0x15 }));                    //  2 
+    index(id, "2", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x16 }));                    //  5 
+    index(id, "3", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x32, 0x58 }));                    //  8 
+    index(id, "4", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 }));                    //  4 
+    index(id, "5", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x35, 0x10, 0x00 }));              //  9 
+    index(id, "6", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x1a, 0x2b, 0x3c, 0x00, 0x00, 0x03 }));  //  3 
+    index(id, "7", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x00, 0x3c, 0x73 }));                    //  1 
+    index(id, "8", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x59, 0x2d, 0x4d }));                    // 11 
+    index(id, "9", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x39, 0x79, 0x7a }));                    // 10 
+    index(id, "10", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c }));                   //  6 
+    index(id, "11", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0xff, (byte)0xaf, (byte)0x9c })); // 13 
+    index(id, "12", "text", "d", "payload", ByteBuffer.wrap(new byte[] { 0x34, (byte)0xdd, 0x4d }));             //  7 
+    index(id, "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11, 0x33 }));             // 12 
+    commit();
+                                                                                          
+    handle.put("QTime", SKIPVAL);
+
+    QueryResponse rsp;
+    rsp = query("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 7, 1, 6, 4, 2, 10, 12, 3, 5, 9, 8, 13, 11); 
+    rsp = query("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12, 10, 2, 4, 6, 1, 7);
+    
+    rsp = query("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 1, 3, 5, 9);
+    rsp = query("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 9, 5, 3, 1);
+    
+    rsp = query("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 4, 2, 10);
+    rsp = query("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 10, 2, 4);
+    
+    rsp = query("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 7, 6, 8);
+    rsp = query("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 8, 6, 7);
+    
+    rsp = query("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 12, 13, 11);
+    rsp = query("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 11, 13, 12);
+
+
+    // Add two more docs with same payload as in doc #4 
+    index(id, "14", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); 
+    index(id, "15", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 })); 
+
+    // Add three more docs with same payload as in doc #10
+    index(id, "16", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); 
+    index(id, "17", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c })); 
+    index(id, "18", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c }));
+    
+    commit();
+    
+    rsp = query("q", "*:*", "fl", "id", "sort", "payload asc, id desc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 7, 1, 6,   15,14,4,   2,   18,17,16,10,   12, 3, 5, 9, 8, 13, 11);
+    rsp = query("q", "*:*", "fl", "id", "sort", "payload desc, id asc", "rows", "20");
+    assertFieldValues(rsp.getResults(), id, 11, 13, 8, 9, 5, 3, 12,   10,16,17,18,   2,   4,14,15,   6, 1, 7);
+  }
+}

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/SortableBinaryField.java Thu Nov 28 21:00:28 2013
@@ -0,0 +1,103 @@
+package org.apache.solr.schema;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.StorableField;
+import org.apache.lucene.search.FieldComparator;
+import org.apache.lucene.search.FieldComparatorSource;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.util.Base64;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Custom field representing a {@link BinaryField} that's sortable.
+ */
+public class SortableBinaryField extends BinaryField {
+
+  @Override
+  public void checkSchemaField(final SchemaField field) {
+    if (field.hasDocValues() && !field.multiValued() && !(field.isRequired() || field.getDefaultValue() != null)) {
+      throw new IllegalStateException(
+          "Field " + this + " has single-valued doc values enabled, but has no default value and is not required");
+    }
+  }
+
+  @Override
+  public List<StorableField> createFields(SchemaField field, Object value, float boost) {
+    if (field.hasDocValues()) {
+      List<StorableField> fields = new ArrayList<StorableField>();
+      StorableField storedField = createField(field, value, boost);
+      fields.add(storedField);
+      ByteBuffer byteBuffer = toObject(storedField);
+      BytesRef bytes = new BytesRef
+          (byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining());
+      if (field.multiValued()) {
+        fields.add(new SortedSetDocValuesField(field.getName(), bytes));
+      } else {
+        fields.add(new SortedDocValuesField(field.getName(), bytes));
+      }
+      return fields;
+    } else {
+      return Collections.singletonList(createField(field, value, boost));
+    }
+  }
+
+  @Override
+  public SortField getSortField(final SchemaField field, final boolean reverse) {
+    field.checkSortability();
+    return new BinarySortField(field.getName(), reverse);
+  }
+
+  private static class BinarySortField extends SortField {
+    public BinarySortField(final String field, final boolean reverse) {
+      super(field, new FieldComparatorSource() {
+        @Override
+        public FieldComparator.TermOrdValComparator newComparator
+            (final String fieldname, final int numHits, final int sortPos, final boolean reversed) throws IOException {
+          return new FieldComparator.TermOrdValComparator(numHits, fieldname);
+        }}, reverse);
+    }
+  }
+  
+  @Override
+  public Object marshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final BytesRef val = (BytesRef)value;
+    return Base64.byteArrayToBase64(val.bytes, val.offset, val.length);
+  }
+  
+  @Override
+  public Object unmarshalSortValue(Object value) {
+    if (null == value) {
+      return null;
+    }
+    final String val = (String)value;
+    final byte[] bytes = Base64.base64ToByteArray(val);
+    return new BytesRef(bytes);
+  }
+}

Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java?rev=1546457&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/TestCustomSort.java Thu Nov 28 21:00:28 2013
@@ -0,0 +1,126 @@
+package org.apache.solr.search;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.BeforeClass;
+
+import java.nio.ByteBuffer;
+
+
+/**
+ * Test SortField.CUSTOM sorts
+ */
+public class TestCustomSort extends SolrTestCaseJ4 {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    initCore("solrconfig.xml", "schema-custom-field.xml");
+  }
+  
+  public void testSortableBinary() throws Exception {
+    clearIndex();
+    assertU(adoc(sdoc("id", "1", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x12, 0x62, 0x15 }))));                    //  2
+    assertU(adoc(sdoc("id", "2", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x16 }))));                    //  5
+    assertU(adoc(sdoc("id", "3", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x32, 0x58 }))));                    //  8
+    assertU(adoc(sdoc("id", "4", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x25, 0x21, 0x15 }))));                    //  4
+    assertU(adoc(sdoc("id", "5", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x35, 0x35, 0x10, 0x00 }))));              //  9
+    assertU(adoc(sdoc("id", "6", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x1a, 0x2b, 0x3c, 0x00, 0x00, 0x03 }))));  //  3
+    assertU(adoc(sdoc("id", "7", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x00, 0x3c, 0x73 }))));                    //  1
+    assertU(adoc(sdoc("id", "8", "text", "c", "payload", ByteBuffer.wrap(new byte[] { 0x59, 0x2d, 0x4d }))));                    // 11
+    assertU(adoc(sdoc("id", "9", "text", "a", "payload", ByteBuffer.wrap(new byte[] { 0x39, 0x79, 0x7a }))));                    // 10
+    assertU(adoc(sdoc("id", "10", "text", "b", "payload", ByteBuffer.wrap(new byte[] { 0x31, 0x39, 0x7c }))));                   //  6
+    assertU(adoc(sdoc("id", "11", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0xff, (byte)0xaf, (byte)0x9c })))); // 13
+    assertU(adoc(sdoc("id", "12", "text", "d", "payload", ByteBuffer.wrap(new byte[] { 0x34, (byte)0xdd, 0x4d }))));             //  7
+    assertU(adoc(sdoc("id", "13", "text", "d", "payload", ByteBuffer.wrap(new byte[] { (byte)0x80, 0x11, 0x33 }))));             // 12
+    assertU(commit());
+    
+    assertQ(req("q", "*:*", "fl", "id", "sort", "payload asc", "rows", "20")
+        , "//result[@numFound='13']"                   // <result name="response" numFound="13" start="0">
+        , "//result/doc[int='7'  and position()=1]"    //   <doc><int name="id">7</int></doc>   00 3c 73
+        , "//result/doc[int='1'  and position()=2]"    //   <doc><int name="id">1</int></doc>   12 62 15
+        , "//result/doc[int='6'  and position()=3]"    //   <doc><int name="id">6</int></doc>   1a 2b 3c 00 00 03
+        , "//result/doc[int='4'  and position()=4]"    //   <doc><int name="id">4</int></doc>   25 21 15
+        , "//result/doc[int='2'  and position()=5]"    //   <doc><int name="id">2</int></doc>   25 21 16
+        , "//result/doc[int='10' and position()=6]"    //   <doc><int name="id">10</int></doc>  31 39 7c
+        , "//result/doc[int='12' and position()=7]"    //   <doc><int name="id">12</int></doc>  34 dd 4d
+        , "//result/doc[int='3'  and position()=8]"    //   <doc><int name="id">3</int></doc>   35 32 58
+        , "//result/doc[int='5'  and position()=9]"    //   <doc><int name="id">5</int></doc>   35 35 10 00
+        , "//result/doc[int='9'  and position()=10]"   //   <doc><int name="id">9</int></doc>   39 79 7a
+        , "//result/doc[int='8'  and position()=11]"   //   <doc><int name="id">8</int></doc>   59 2d 4d      
+        , "//result/doc[int='13' and position()=12]"   //   <doc><int name="id">13</int></doc>  80 11 33       
+        , "//result/doc[int='11' and position()=13]"); //   <doc><int name="id">11</int></doc>  ff af 9c
+    assertQ(req("q", "*:*", "fl", "id", "sort", "payload desc", "rows", "20")
+        , "//result[@numFound='13']"                   // <result name="response" numFound="13" start="0">
+        , "//result/doc[int='11' and position()=1]"    //   <doc><int name="id">11</int></doc>  ff af 9c            
+        , "//result/doc[int='13' and position()=2]"    //   <doc><int name="id">13</int></doc>  80 11 33                   
+        , "//result/doc[int='8'  and position()=3]"    //   <doc><int name="id">8</int></doc>   59 2d 4d                  
+        , "//result/doc[int='9'  and position()=4]"    //   <doc><int name="id">9</int></doc>   39 79 7a            
+        , "//result/doc[int='5'  and position()=5]"    //   <doc><int name="id">5</int></doc>   35 35 10 00         
+        , "//result/doc[int='3'  and position()=6]"    //   <doc><int name="id">3</int></doc>   35 32 58            
+        , "//result/doc[int='12' and position()=7]"    //   <doc><int name="id">12</int></doc>  34 dd 4d            
+        , "//result/doc[int='10' and position()=8]"    //   <doc><int name="id">10</int></doc>  31 39 7c            
+        , "//result/doc[int='2'  and position()=9]"    //   <doc><int name="id">2</int></doc>   25 21 16            
+        , "//result/doc[int='4'  and position()=10]"   //   <doc><int name="id">4</int></doc>   25 21 15            
+        , "//result/doc[int='6'  and position()=11]"   //   <doc><int name="id">6</int></doc>   1a 2b 3c 00 00 03   
+        , "//result/doc[int='1'  and position()=12]"   //   <doc><int name="id">1</int></doc>   12 62 15            
+        , "//result/doc[int='7'  and position()=13]"); //   <doc><int name="id">7</int></doc>   00 3c 73            
+    assertQ(req("q", "text:a", "fl", "id", "sort", "payload asc", "rows", "20")
+        , "//result[@numFound='4']"                    // <result name="response" numFound="4" start="0">
+        , "//result/doc[int='1'  and position()=1]"    //   <doc><int name="id">1</int></doc>   12 62 15    
+        , "//result/doc[int='3'  and position()=2]"    //   <doc><int name="id">3</int></doc>   35 32 58    
+        , "//result/doc[int='5'  and position()=3]"    //   <doc><int name="id">5</int></doc>   35 35 10 00 
+        , "//result/doc[int='9'  and position()=4]");  //   <doc><int name="id">9</int></doc>   39 79 7a    
+    assertQ(req("q", "text:a", "fl", "id", "sort", "payload desc", "rows", "20")
+        , "//result[@numFound='4']"                    // <result name="response" numFound="4" start="0">
+        , "//result/doc[int='9'  and position()=1]"    //   <doc><int name="id">9</int></doc>   39 79 7a    
+        , "//result/doc[int='5'  and position()=2]"    //   <doc><int name="id">5</int></doc>   35 35 10 00 
+        , "//result/doc[int='3'  and position()=3]"    //   <doc><int name="id">3</int></doc>   35 32 58    
+        , "//result/doc[int='1'  and position()=4]");  //   <doc><int name="id">1</int></doc>   12 62 15    
+    assertQ(req("q", "text:b", "fl", "id", "sort", "payload asc", "rows", "20")
+        , "//result[@numFound='3']"                    // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='4'  and position()=1]"    //   <doc><int name="id">4</int></doc>   25 21 15
+        , "//result/doc[int='2'  and position()=2]"    //   <doc><int name="id">2</int></doc>   25 21 16
+        , "//result/doc[int='10' and position()=3]");  //   <doc><int name="id">10</int></doc>  31 39 7c
+    assertQ(req("q", "text:b", "fl", "id", "sort", "payload desc", "rows", "20")
+        , "//result[@numFound='3']"                    // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='10' and position()=1]"    //   <doc><int name="id">10</int></doc>  31 39 7c
+        , "//result/doc[int='2'  and position()=2]"    //   <doc><int name="id">2</int></doc>   25 21 16
+        , "//result/doc[int='4'  and position()=3]");  //   <doc><int name="id">4</int></doc>   25 21 15
+    assertQ(req("q", "text:c", "fl", "id", "sort", "payload asc", "rows", "20")
+        , "//result[@numFound='3']"                   // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='7'  and position()=1]"   //   <doc><int name="id">7</int></doc>    00 3c 73         
+        , "//result/doc[int='6'  and position()=2]"   //   <doc><int name="id">6</int></doc>    1a 2b 3c 00 00 03
+        , "//result/doc[int='8'  and position()=3]"); //   <doc><int name="id">8</int></doc>    59 2d 4d              
+    assertQ(req("q", "text:c", "fl", "id", "sort", "payload desc", "rows", "20")
+        , "//result[@numFound='3']"                   // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='8'  and position()=1]"   //   <doc><int name="id">8</int></doc>    59 2d 4d              
+        , "//result/doc[int='6'  and position()=2]"   //   <doc><int name="id">6</int></doc>    1a 2b 3c 00 00 03
+        , "//result/doc[int='7'  and position()=3]"); //   <doc><int name="id">7</int></doc>    00 3c 73         
+    assertQ(req("q", "text:d", "fl", "id", "sort", "payload asc", "rows", "20")
+        , "//result[@numFound='3']"                   // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='12' and position()=1]"   //   <doc><int name="id">12</int></doc>   34 dd 4d
+        , "//result/doc[int='13' and position()=2]"   //   <doc><int name="id">13</int></doc>   80 11 33      
+        , "//result/doc[int='11' and position()=3]"); //   <doc><int name="id">11</int></doc>   ff af 9c
+    assertQ(req("q", "text:d", "fl", "id", "sort", "payload desc", "rows", "20")
+        , "//result[@numFound='3']"                   // <result name="response" numFound="3" start="0">
+        , "//result/doc[int='11' and position()=1]"   //   <doc><int name="id">11</int></doc>   ff af 9c
+        , "//result/doc[int='13' and position()=2]"   //   <doc><int name="id">13</int></doc>   80 11 33      
+        , "//result/doc[int='12' and position()=3]"); //   <doc><int name="id">12</int></doc>   34 dd 4d
+  }
+}

Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java Thu Nov 28 21:00:28 2013
@@ -507,11 +507,18 @@ public abstract class BaseDistributedSea
     return rsp;
   }
 
-  protected void query(Object... q) throws Exception {
-    query(true, q);
+  /**
+   * Sets distributed params.
+   * Returns the QueryResponse from {@link #queryServer},
+   */
+  protected QueryResponse query(Object... q) throws Exception {
+    return query(true, q);
   }
-  
-  protected void query(boolean setDistribParams, Object[] q) throws Exception {
+
+  /**
+   * Returns the QueryResponse from {@link #queryServer}  
+   */
+  protected QueryResponse query(boolean setDistribParams, Object[] q) throws Exception {
     
     final ModifiableSolrParams params = new ModifiableSolrParams();
 
@@ -558,6 +565,7 @@ public abstract class BaseDistributedSea
         thread.join();
       }
     }
+    return rsp;
   }
   
   public QueryResponse queryAndCompare(SolrParams params, SolrServer... servers) throws SolrServerException {

Modified: lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java?rev=1546457&r1=1546456&r2=1546457&view=diff
==============================================================================
--- lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (original)
+++ lucene/dev/trunk/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java Thu Nov 28 21:00:28 2013
@@ -47,6 +47,8 @@ import org.apache.lucene.util.LuceneTest
 import org.apache.lucene.util.QuickPatchThreadsFilter;
 import org.apache.lucene.util._TestUtil;
 import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.SolrInputField;
@@ -1627,6 +1629,30 @@ public abstract class SolrTestCaseJ4 ext
       throw new RuntimeException("XPath is invalid", e2);
     }
   }
+                                                         
+  /**
+   * Fails if the number of documents in the given SolrDocumentList differs
+   * from the given number of expected values, or if any of the values in the
+   * given field don't match the expected values in the same order.
+   */
+  public static void assertFieldValues(SolrDocumentList documents, String fieldName, Object... expectedValues) {
+    if (documents.size() != expectedValues.length) {
+      fail("Number of documents (" + documents.size()
+          + ") is different from number of expected values (" + expectedValues.length);
+    }
+    for (int docNum = 1 ; docNum <= documents.size() ; ++docNum) {
+      SolrDocument doc = documents.get(docNum - 1);
+      Object expected = expectedValues[docNum - 1];
+      Object actual = doc.get(fieldName);
+      if (null != expected && null != actual) {
+        if ( ! expected.equals(actual)) {
+          fail( "Unexpected " + fieldName + " field value in document #" + docNum
+              + ": expected=[" + expected + "], actual=[" + actual + "]");
+        }
+      }
+    }
+  }
+
   public static void copyMinConf(File dstRoot) throws IOException {
     copyMinConf(dstRoot, null);
   }