You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jb...@apache.org on 2014/08/15 18:32:42 UTC
svn commit: r1618228 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/CHANGES.txt solr/core/
solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
Author: jbernste
Date: Fri Aug 15 16:32:41 2014
New Revision: 1618228
URL: http://svn.apache.org/r1618228
Log:
SOLR-5244: Exporting Full Sorted Result Sets
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1618228&r1=1618227&r2=1618228&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Fri Aug 15 16:32:41 2014
@@ -110,6 +110,7 @@ New Features
* SOLR-6304 : JsonLoader should be able to flatten an input JSON to multiple docs (Noble Paul)
+* SOLR-5244: Exporting Full Sorted Result Sets (Erik Hatcher, Joel Bernstein)
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java?rev=1618228&r1=1618227&r2=1618228&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java Fri Aug 15 16:32:41 2014
@@ -17,7 +17,6 @@
package org.apache.solr.response;
-import com.carrotsearch.hppc.IntArrayList;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.MultiDocValues;
@@ -48,15 +47,19 @@ import org.apache.solr.schema.StrField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Writer;
import java.io.PrintWriter;
+import java.net.SocketException;
import java.util.List;
public class SortingResponseWriter implements QueryResponseWriter {
+ private final static Logger logger = LoggerFactory.getLogger(SortingResponseWriter.class);
public void init(NamedList args) {
/* NOOP */
@@ -157,16 +160,33 @@ public class SortingResponseWriter imple
count += (outDocsIndex+1);
- for(int i=outDocsIndex; i>=0; --i) {
- SortDoc s = outDocs[i];
- if(commaNeeded){writer.write(',');}
- writer.write('{');
- writeDoc(s, leaves, fieldWriters, sets, writer);
- writer.write('}');
- commaNeeded = true;
- s.reset();
+ try {
+ for(int i=outDocsIndex; i>=0; --i) {
+ SortDoc s = outDocs[i];
+ if(commaNeeded){writer.write(',');}
+ writer.write('{');
+ writeDoc(s, leaves, fieldWriters, sets, writer);
+ writer.write('}');
+ commaNeeded = true;
+ s.reset();
+ }
+ } catch(Throwable e) {
+ Throwable ex = e;
+ while(ex != null) {
+ String m = ex.getMessage();
+ if(m != null && m.contains("Broken pipe")) {
+ logger.info("Early client disconnect during export");
+ return;
+ }
+ ex = ex.getCause();
+ }
+
+ if(e instanceof IOException) {
+ throw ((IOException)e);
+ } else {
+ throw new IOException(e);
+ }
}
- //total+=end-begin;
}
//System.out.println("Sort Time 2:"+Long.toString(total/1000000));
@@ -240,6 +260,8 @@ public class SortingResponseWriter imple
} else {
writers[i] = new StringFieldWriter(field, fieldType);
}
+ } else {
+ throw new IOException("Export fields must either be one of the following types: int,float,long,double,string");
}
}
return writers;
@@ -252,7 +274,13 @@ public class SortingResponseWriter imple
SortField sf = sortFields[i];
String field = sf.getField();
boolean reverse = sf.getReverse();
- FieldType ft = schema.getField(field).getType();
+ SchemaField schemaField = schema.getField(field);
+ FieldType ft = schemaField.getType();
+
+ if(!schemaField.hasDocValues()) {
+ throw new IOException(field+" must have DocValues to use this feature.");
+ }
+
if(ft instanceof TrieIntField) {
if(reverse) {
sortValues[i] = new IntValue(field, new IntDesc());
@@ -285,6 +313,8 @@ public class SortingResponseWriter imple
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
}
+ } else {
+ throw new IOException("Sort fields must be one of the following types: int,float,long,double,string");
}
}
@@ -1060,7 +1090,12 @@ public class SortingResponseWriter imple
}
public void setCurrentValue(int docId) {
- currentOrd = (int)globalOrds.get(currentVals.getOrd(docId));
+ int ord = currentVals.getOrd(docId);
+ if(ord < 0) {
+ currentOrd = -1;
+ } else {
+ currentOrd = (int)globalOrds.get(ord);
+ }
}
public void setCurrentValue(SortValue sv) {
Modified: lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java?rev=1618228&r1=1618227&r2=1618228&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java Fri Aug 15 16:32:41 2014
@@ -48,6 +48,18 @@ public class TestSortingResponseWriter e
"stringdv_m", "liverpool",
"stringdv_m", "Everton"));
+ assertU(adoc("id","7",
+ "floatdv","2.1",
+ "intdv", "7",
+ "longdv", "323223232323",
+ "doubledv","2344.345",
+ "floatdv_m", "123.321",
+ "floatdv_m", "345.123",
+ "doubledv_m", "3444.222",
+ "doubledv_m", "23232.2",
+ "longdv_m", "43434343434",
+ "longdv_m", "343332"));
+
assertU(commit());
assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
assertU(commit());
@@ -80,18 +92,34 @@ public class TestSortingResponseWriter e
String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
+ //Test null value string:
+ s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+ assertEquals(s, "{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"stringdv\":\"\",\"longdv\":323223232323,\"doubledv\":2344.345}]}");
+
//Test multiValue docValues output
s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
- System.out.println(s);
assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester city\"]}]}");
+
+ //Test multiValues docValues output with nulls
+ s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
+ assertEquals(s, "{\"numFound\":1, \"docs\":[{\"intdv_m\":[],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[]}]}");
+
//Test single sort param is working
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
- System.out.println("Output:"+s);
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}");
+ // Test sort on String will null value. Null value should sort last on desc and first on asc.
+
+ s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
+ assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}");
+
+ s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
+ assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}");
+
+
//Test multi-sort params
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
assertEquals(s, "{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}");
@@ -112,7 +140,6 @@ public class TestSortingResponseWriter e
assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}");
s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
- System.out.println("Results:"+s);
assertEquals(s, "{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}");
}