You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2016/02/02 10:53:22 UTC
[10/21] lucene-solr git commit: SOLR-8285: Ensure the /export handler
works with NULL field values
SOLR-8285: Ensure the /export handler works with NULL field values
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e20820a0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e20820a0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e20820a0
Branch: refs/heads/master-solr-8621
Commit: e20820a04d1aff5ccde64b7183308454bda62fef
Parents: a4d1586
Author: jbernste <jb...@apache.org>
Authored: Sun Jan 31 13:18:48 2016 -0500
Committer: jbernste <jb...@apache.org>
Committed: Sun Jan 31 13:21:10 2016 -0500
----------------------------------------------------------------------
.../solr/response/SortingResponseWriter.java | 100 +++++++++++++------
.../response/TestSortingResponseWriter.java | 5 +-
.../solrj/io/stream/StreamExpressionTest.java | 63 ++++++++++++
3 files changed, 135 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
index 4b9f89f..6b6d36c 100644
--- a/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
+++ b/solr/core/src/java/org/apache/solr/response/SortingResponseWriter.java
@@ -22,7 +22,9 @@ import java.io.PrintWriter;
import java.io.Writer;
import java.lang.invoke.MethodHandles;
import java.util.List;
+import java.util.ArrayList;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
@@ -239,13 +241,11 @@ public class SortingResponseWriter implements QueryResponseWriter {
FixedBitSet set = sets[ord];
set.clear(sortDoc.docId);
LeafReaderContext context = leaves.get(ord);
- boolean needsComma = false;
+ int fieldIndex = 0;
for(FieldWriter fieldWriter : fieldWriters) {
- if(needsComma) {
- out.write(',');
+ if(fieldWriter.write(sortDoc.docId, context.reader(), out, fieldIndex)){
+ ++fieldIndex;
}
- fieldWriter.write(sortDoc.docId, context.reader(), out);
- needsComma = true;
}
}
@@ -827,7 +827,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = context.reader().getNumericDocValues(field);
+ this.vals = DocValues.getNumeric(context.reader(), field);
}
public void setCurrentValue(int docId) {
@@ -905,7 +905,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = context.reader().getNumericDocValues(field);
+ this.vals = DocValues.getNumeric(context.reader(), field);
}
public void setCurrentValue(int docId) {
@@ -984,7 +984,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = context.reader().getNumericDocValues(field);
+ this.vals = DocValues.getNumeric(context.reader(), field);
}
public void setCurrentValue(int docId) {
@@ -1061,7 +1061,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = context.reader().getNumericDocValues(field);
+ this.vals = DocValues.getNumeric(context.reader(), field);
}
public void setCurrentValue(int docId) {
@@ -1193,7 +1193,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
}
protected abstract class FieldWriter {
- public abstract void write(int docId, LeafReader reader, Writer out) throws IOException;
+ public abstract boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException;
}
class IntFieldWriter extends FieldWriter {
@@ -1203,14 +1203,18 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- NumericDocValues vals = reader.getNumericDocValues(this.field);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val = (int)vals.get(docId);
- out.write('"');
- out.write(this.field);
- out.write('"');
- out.write(':');
- out.write(Integer.toString(val));
+ if(fieldIndex>0) {
+ out.write(',');
+ }
+ out.write('"');
+ out.write(this.field);
+ out.write('"');
+ out.write(':');
+ out.write(Integer.toString(val));
+ return true;
}
}
@@ -1225,18 +1229,30 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.fieldType = fieldType;
this.numeric = numeric;
}
-
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- SortedSetDocValues vals = reader.getSortedSetDocValues(this.field);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
vals.setDocument(docId);
+ List<Long> ords = new ArrayList();
+ long o = -1;
+ while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ ords.add(o);
+ }
+
+ if(ords.size()== 0) {
+ return false;
+ }
+
+
+ if(fieldIndex>0) {
+ out.write(',');
+ }
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write('[');
int v = 0;
- long ord = -1;
- while((ord = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ for(long ord : ords) {
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
if(v > 0) {
@@ -1255,6 +1271,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
++v;
}
out.write("]");
+ return true;
}
}
@@ -1265,14 +1282,18 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- NumericDocValues vals = reader.getNumericDocValues(this.field);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val = vals.get(docId);
+ if(fieldIndex > 0) {
+ out.write(',');
+ }
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Long.toString(val));
+ return true;
}
}
@@ -1283,14 +1304,18 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- NumericDocValues vals = reader.getNumericDocValues(this.field);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val = (int)vals.get(docId);
+ if(fieldIndex > 0) {
+ out.write(',');
+ }
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Float.toString(Float.intBitsToFloat(val)));
+ return true;
}
}
@@ -1301,14 +1326,18 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.field = field;
}
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- NumericDocValues vals = reader.getNumericDocValues(this.field);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ if(fieldIndex > 0) {
+ out.write(',');
+ }
long val = vals.get(docId);
out.write('"');
out.write(this.field);
out.write('"');
out.write(':');
out.write(Double.toString(Double.longBitsToDouble(val)));
+ return true;
}
}
@@ -1322,10 +1351,18 @@ public class SortingResponseWriter implements QueryResponseWriter {
this.fieldType = fieldType;
}
- public void write(int docId, LeafReader reader, Writer out) throws IOException {
- SortedDocValues vals = reader.getSortedDocValues(this.field);
- BytesRef ref = vals.get(docId);
+ public boolean write(int docId, LeafReader reader, Writer out, int fieldIndex) throws IOException {
+ SortedDocValues vals = DocValues.getSorted(reader, this.field);
+ int ord = vals.getOrd(docId);
+ if(ord == -1) {
+ return false;
+ }
+
+ BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
+ if(fieldIndex > 0) {
+ out.write(',');
+ }
out.write('"');
out.write(this.field);
out.write('"');
@@ -1333,6 +1370,7 @@ public class SortingResponseWriter implements QueryResponseWriter {
out.write('"');
writeStr(cref.toString(), out);
out.write('"');
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
index 0e7f9b1..bb432ec 100644
--- a/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestSortingResponseWriter.java
@@ -114,7 +114,8 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
//Test null value string:
s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
- assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"stringdv\":\"\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
+
+ assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
//Test multiValue docValues output
s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
@@ -122,7 +123,7 @@ public class TestSortingResponseWriter extends SolrTestCaseJ4 {
//Test multiValues docValues output with nulls
s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
- assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"intdv_m\":[],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[]}]}}");
+ assertEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
//Test single sort param is working
s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e20820a0/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 69ef6e8..33ee767 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -136,6 +136,7 @@ public class StreamExpressionTest extends AbstractFullDistribZkTestBase {
testUniqueStream();
testRollupStream();
testStatsStream();
+ testNulls();
testDaemonStream();
testParallelUniqueStream();
testParallelReducerStream();
@@ -303,6 +304,68 @@ public class StreamExpressionTest extends AbstractFullDistribZkTestBase {
del("*:*");
commit();
}
+
+
+ private void testNulls() throws Exception {
+
+ indexr(id, "0", "a_i", "1", "a_f", "0", "s_multi", "aaa", "s_multi", "bbb", "i_multi", "100", "i_multi", "200");
+ indexr(id, "2", "a_s", "hello2", "a_i", "3", "a_f", "0");
+ indexr(id, "3", "a_s", "hello3", "a_i", "4", "a_f", "3");
+ indexr(id, "4", "a_s", "hello4", "a_f", "4");
+ indexr(id, "1", "a_s", "hello1", "a_i", "2", "a_f", "1");
+ commit();
+
+ StreamExpression expression;
+ TupleStream stream;
+ List<Tuple> tuples;
+ Tuple tuple;
+ StreamFactory factory = new StreamFactory()
+ .withCollectionZkHost("collection1", zkServer.getZkAddress())
+ .withFunctionName("search", CloudSolrStream.class);
+ // Basic test
+ expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f, s_multi, i_multi\", qt=\"/export\", sort=\"a_i asc\")");
+ stream = new CloudSolrStream(expression, factory);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 5);
+ assertOrder(tuples, 4, 0, 1, 2, 3);
+
+ tuple = tuples.get(0);
+ assertTrue("hello4".equals(tuple.getString("a_s")));
+ assertNull(tuple.get("s_multi"));
+ assertNull(tuple.get("i_multi"));
+ assertEquals(0L, (long)tuple.getLong("a_i"));
+
+
+ tuple = tuples.get(1);
+ assertNull(tuple.get("a_s"));
+ List<String> strings = tuple.getStrings("s_multi");
+ assertNotNull(strings);
+ assertEquals("aaa", strings.get(0));
+ assertEquals("bbb", strings.get(1));
+ List<Long> longs = tuple.getLongs("i_multi");
+ assertNotNull(longs);
+
+ //test sort (asc) with null string field. Null should sort to the top.
+ expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f, s_multi, i_multi\", qt=\"/export\", sort=\"a_s asc\")");
+ stream = new CloudSolrStream(expression, factory);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 5);
+ assertOrder(tuples, 0, 1, 2, 3, 4);
+
+ //test sort(desc) with null string field. Null should sort to the bottom.
+ expression = StreamExpressionParser.parse("search(collection1, q=*:*, fl=\"id,a_s,a_i,a_f, s_multi, i_multi\", qt=\"/export\", sort=\"a_s desc\")");
+ stream = new CloudSolrStream(expression, factory);
+ tuples = getTuples(stream);
+
+ assert(tuples.size() == 5);
+ assertOrder(tuples, 4, 3, 2, 1, 0);
+
+ del("*:*");
+ commit();
+ }
+
private void testMergeStream() throws Exception {