You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by tf...@apache.org on 2017/06/12 21:22:35 UTC
lucene-solr:master: SOLR-10835: Add support for point fields in
Export Handler
Repository: lucene-solr
Updated Branches:
refs/heads/master 0411504dd -> c51f6fae7
SOLR-10835: Add support for point fields in Export Handler
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c51f6fae
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c51f6fae
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c51f6fae
Branch: refs/heads/master
Commit: c51f6fae75b7b2de1cbe77a13b76d6e08e9fa35c
Parents: 0411504
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Mon Jun 12 14:04:16 2017 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Mon Jun 12 14:21:41 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/handler/ExportWriter.java | 105 +++++++---
.../collection1/conf/schema-sortingresponse.xml | 71 ++++++-
.../apache/solr/response/TestExportWriter.java | 199 ++++++++++++++++++-
4 files changed, 328 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c51f6fae/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e9a22db..9135cfa 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -360,6 +360,8 @@ Bug Fixes
* SOLR-10715: /v2/ should not be an alias for /v2/collections (Cao Manh Dat)
+* SOLR-10835: Add support for point fields in Export Handler (Tomás Fernández Löbbe)
+
Optimizations
----------------------
* SOLR-10634: JSON Facet API: When a field/terms facet will retrieve all buckets (i.e. limit:-1)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c51f6fae/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
index 46ec3a4..61f937c 100644
--- a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
@@ -17,6 +17,10 @@
package org.apache.solr.handler;
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.common.util.Utils.makeMap;
+
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
@@ -26,6 +30,7 @@ import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
+import java.util.function.LongFunction;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexableField;
@@ -34,6 +39,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
@@ -44,6 +50,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
+import org.apache.lucene.util.NumericUtils;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
@@ -60,25 +67,21 @@ import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.DateValueFieldType;
+import org.apache.solr.schema.DoubleValueFieldType;
import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.FloatValueFieldType;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IntValueFieldType;
+import org.apache.solr.schema.LongValueFieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
-import org.apache.solr.schema.TrieDateField;
-import org.apache.solr.schema.TrieDoubleField;
-import org.apache.solr.schema.TrieFloatField;
-import org.apache.solr.schema.TrieIntField;
-import org.apache.solr.schema.TrieLongField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
-import static org.apache.solr.common.util.Utils.makeMap;
-
public class ExportWriter implements SolrCore.RawWriter, Closeable {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private OutputStreamWriter respWriter;
@@ -322,25 +325,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
boolean multiValued = schemaField.multiValued();
FieldType fieldType = schemaField.getType();
- if (fieldType instanceof TrieIntField) {
+ if (fieldType instanceof IntValueFieldType) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new IntFieldWriter(field);
}
- } else if (fieldType instanceof TrieLongField) {
+ } else if (fieldType instanceof LongValueFieldType) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new LongFieldWriter(field);
}
- } else if (fieldType instanceof TrieFloatField) {
+ } else if (fieldType instanceof FloatValueFieldType) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new FloatFieldWriter(field);
}
- } else if (fieldType instanceof TrieDoubleField) {
+ } else if (fieldType instanceof DoubleValueFieldType) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
@@ -352,7 +355,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
} else {
writers[i] = new StringFieldWriter(field, fieldType);
}
- } else if (fieldType instanceof TrieDateField) {
+ } else if (fieldType instanceof DateValueFieldType) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
} else {
@@ -385,25 +388,25 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
throw new IOException(field+" must have DocValues to use this feature.");
}
- if(ft instanceof TrieIntField) {
+ if(ft instanceof IntValueFieldType) {
if(reverse) {
sortValues[i] = new IntValue(field, new IntDesc());
} else {
sortValues[i] = new IntValue(field, new IntAsc());
}
- } else if(ft instanceof TrieFloatField) {
+ } else if(ft instanceof FloatValueFieldType) {
if(reverse) {
sortValues[i] = new FloatValue(field, new FloatDesc());
} else {
sortValues[i] = new FloatValue(field, new FloatAsc());
}
- } else if(ft instanceof TrieDoubleField) {
+ } else if(ft instanceof DoubleValueFieldType) {
if(reverse) {
sortValues[i] = new DoubleValue(field, new DoubleDesc());
} else {
sortValues[i] = new DoubleValue(field, new DoubleAsc());
}
- } else if(ft instanceof TrieLongField) {
+ } else if(ft instanceof LongValueFieldType) {
if(reverse) {
sortValues[i] = new LongValue(field, new LongDesc());
} else {
@@ -417,7 +420,7 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
}
- } else if (ft instanceof TrieDateField) {
+ } else if (ft instanceof DateValueFieldType) {
if (reverse) {
sortValues[i] = new LongValue(field, new LongDesc());
} else {
@@ -1352,6 +1355,23 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
return true;
}
}
+
+ static LongFunction<Object> bitsToValue(FieldType fieldType) {
+ switch (fieldType.getNumberType()) {
+ case LONG:
+ return (bits)-> bits;
+ case DATE:
+ return (bits)-> new Date(bits);
+ case INTEGER:
+ return (bits)-> (int)bits;
+ case FLOAT:
+ return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
+ case DOUBLE:
+ return (bits)-> NumericUtils.sortableLongToDouble(bits);
+ default:
+ throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
+ }
+ }
class MultiFieldWriter extends FieldWriter {
private String field;
@@ -1359,29 +1379,48 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
private SchemaField schemaField;
private boolean numeric;
private CharsRefBuilder cref = new CharsRefBuilder();
+ private final LongFunction<Object> bitsToValue;
public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
this.field = field;
this.fieldType = fieldType;
this.schemaField = schemaField;
this.numeric = numeric;
+ if (this.fieldType.isPointField()) {
+ bitsToValue = bitsToValue(fieldType);
+ } else {
+ bitsToValue = null;
+ }
}
public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
- SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
- if (vals.advance(docId) != docId) return false;
- out.put(this.field,
- (IteratorWriter) w -> {
- long o;
- while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- BytesRef ref = vals.lookupOrd(o);
- fieldType.indexedToReadable(ref, cref);
- IndexableField f = fieldType.createField(schemaField, cref.toString());
- if (f == null) w.add(cref.toString());
- else w.add(fieldType.toObject(f));
- }
- });
- return true;
+ if (this.fieldType.isPointField()) {
+ SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
+ if (!vals.advanceExact(docId)) return false;
+ out.put(this.field,
+ (IteratorWriter) w -> {
+ for (int i = 0; i < vals.docValueCount(); i++) {
+ w.add(bitsToValue.apply(vals.nextValue()));
+ }
+ });
+ return true;
+ } else {
+ SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
+ if (vals.advance(docId) != docId) return false;
+ out.put(this.field,
+ (IteratorWriter) w -> {
+ long o;
+ while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ BytesRef ref = vals.lookupOrd(o);
+ fieldType.indexedToReadable(ref, cref);
+ IndexableField f = fieldType.createField(schemaField, cref.toString());
+ if (f == null) w.add(cref.toString());
+ else w.add(fieldType.toObject(f));
+ }
+ });
+ return true;
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c51f6fae/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
index ecf0daf..6ec93fa 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
@@ -26,6 +26,14 @@
seconds part (.999) is optional.
-->
<fieldType name="date" class="solr.TrieDateField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
+
+ <!-- Point Fields -->
+ <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
+ <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
+ <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
+ <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
+ <fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
+
<fieldType name="boolean" class="solr.BoolField"/>
<fieldType name="string" class="solr.StrField"/>
@@ -33,20 +41,63 @@
<fieldType name="uuid" class="solr.UUIDField"/>
<field name="id" type="string" required="true" indexed="true"/>
- <field name="floatdv_m" type="float" indexed="false" stored="false" docValues="true" multiValued="true"/>
- <field name="intdv_m" type="int" indexed="false" stored="false" docValues="true" multiValued="true"/>
- <field name="doubledv_m" type="double" indexed="false" stored="false" docValues="true" multiValued="true"/>
- <field name="longdv_m" type="long" indexed="false" stored="false" docValues="true" multiValued="true"/>
- <field name="datedv_m" type="date" indexed="false" stored="false" docValues="true" multiValued="true"/>
+ <field name="floatdv_m" type="${solr.tests.floatClass:pfloat}" indexed="false" stored="false" docValues="true" multiValued="true"/>
+ <field name="intdv_m" type="${solr.tests.intClass:pint}" indexed="false" stored="false" docValues="true" multiValued="true"/>
+ <field name="doubledv_m" type="${solr.tests.doubleClass:pdouble}" indexed="false" stored="false" docValues="true" multiValued="true"/>
+ <field name="longdv_m" type="${solr.tests.longClass:plong}" indexed="false" stored="false" docValues="true" multiValued="true"/>
+ <field name="datedv_m" type="${solr.tests.dateClass:pdate}" indexed="false" stored="false" docValues="true" multiValued="true"/>
<field name="stringdv_m" type="string" indexed="false" stored="false" docValues="true" multiValued="true"/>
- <field name="floatdv" type="float" indexed="false" stored="false" docValues="true"/>
- <field name="intdv" type="int" indexed="false" stored="false" docValues="true"/>
- <field name="doubledv" type="double" indexed="false" stored="false" docValues="true"/>
- <field name="longdv" type="long" indexed="false" stored="false" docValues="true"/>
- <field name="datedv" type="date" indexed="false" stored="false" docValues="true"/>
+ <field name="floatdv" type="${solr.tests.floatClass:pfloat}" indexed="false" stored="false" docValues="true"/>
+ <field name="intdv" type="${solr.tests.intClass:pint}" indexed="false" stored="false" docValues="true"/>
+ <field name="doubledv" type="${solr.tests.doubleClass:pdouble}" indexed="false" stored="false" docValues="true"/>
+ <field name="longdv" type="${solr.tests.longClass:plong}" indexed="false" stored="false" docValues="true"/>
+ <field name="datedv" type="${solr.tests.dateClass:pdate}" indexed="false" stored="false" docValues="true"/>
<field name="stringdv" type="string" indexed="false" stored="false" docValues="true"/>
+ <!-- Point fields explicitly -->
+ <dynamicField name="*_i_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_i_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_dt_p" type="pdate" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_dts_p" type="pdate" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_dt_ni_p" type="pdate" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_dts_ni_p" type="pdate" indexed="false" stored="true" docValues="true" multiValued="true"/>
+
+ <!-- Trie fields explicitly -->
+ <dynamicField name="*_i_t" type="int" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_t" type="int" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_i_ni_t" type="int" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_ni_t" type="int" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_t" type="long" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_t" type="long" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_ni_t" type="long" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_ni_t" type="long" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_t" type="float" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_t" type="float" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_ni_t" type="float" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_ni_t" type="float" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_t" type="double" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_t" type="double" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_ni_t" type="double" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_ni_t" type="double" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_dt_t" type="date" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_dts_t" type="date" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_dt_ni_t" type="date" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_dts_ni_t" type="date" indexed="false" stored="true" docValues="true" multiValued="true"/>
<uniqueKey>id</uniqueKey>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c51f6fae/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java b/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
index 5a303e9..c556787 100644
--- a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
+++ b/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
@@ -16,17 +16,40 @@
*/
package org.apache.solr.response;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import org.apache.lucene.index.LeafReader;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.SuppressForbidden;
import org.apache.solr.common.util.Utils;
-import org.junit.*;
-import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.util.RefCounted;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
-@SuppressCodecs({"Lucene3x", "Lucene40","Lucene41","Lucene42","Lucene45"})
public class TestExportWriter extends SolrTestCaseJ4 {
+
@BeforeClass
public static void beforeClass() throws Exception {
System.setProperty("export.test", "true");
initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml");
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ assertU(delQ("*:*"));
+ assertU(commit());
createIndex();
}
@@ -47,7 +70,12 @@ public class TestExportWriter extends SolrTestCaseJ4 {
"longdv_m", "343332",
"stringdv_m", "manchester \"city\"",
"stringdv_m", "liverpool",
- "stringdv_m", "Everton"));
+ "stringdv_m", "Everton",
+ "datedv", "2017-06-16T07:00:00Z",
+ "datedv_m", "2017-06-16T01:00:00Z",
+ "datedv_m", "2017-06-16T02:00:00Z",
+ "datedv_m", "2017-06-16T03:00:00Z",
+ "datedv_m", "2017-06-16T04:00:00Z"));
assertU(adoc("id","7",
"floatdv","2.1",
@@ -80,7 +108,11 @@ public class TestExportWriter extends SolrTestCaseJ4 {
"longdv_m", "343332",
"stringdv_m", "manchester \"city\"",
"stringdv_m", "liverpool",
- "stringdv_m", "everton"));
+ "stringdv_m", "everton",
+ "int_is_t", "1",
+ "int_is_t", "1",
+ "int_is_t", "1",
+ "int_is_t", "1"));
assertU(commit());
assertU(adoc("id","8",
"floatdv","2.1",
@@ -98,7 +130,14 @@ public class TestExportWriter extends SolrTestCaseJ4 {
"longdv_m", "343332",
"stringdv_m", "manchester \"city\"",
"stringdv_m", "liverpool",
- "stringdv_m", "everton"));
+ "stringdv_m", "everton",
+ "datedv", "2017-01-01T00:00:00Z",
+ "datedv_m", "2017-01-01T01:00:00Z",
+ "datedv_m", "2017-01-01T02:00:00Z",
+ "int_is_p", "1",
+ "int_is_p", "1",
+ "int_is_p", "1",
+ "int_is_p", "1"));
assertU(commit());
@@ -192,4 +231,152 @@ public class TestExportWriter extends SolrTestCaseJ4 {
// Interesting you don't even need to specify a "q" parameter.
}
+
+ @Test
+ public void testDates() throws Exception {
+ String s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv", "sort", "datedv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv\":\"2017-06-16T07:00:00Z\"}]}}");
+ s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv_m", "sort", "datedv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv_m\":[\"2017-06-16T01:00:00Z\",\"2017-06-16T02:00:00Z\",\"2017-06-16T03:00:00Z\",\"2017-06-16T04:00:00Z\"]}]}}");
+ }
+
+ @Test
+ public void testDuplicates() throws Exception {
+ RefCounted<SolrIndexSearcher> ref = null;
+ try {
+ ref = h.getCore().getSearcher();
+ LeafReader reader = ref.get().getSlowAtomicReader();
+ // MultiValued Trie fields use SortedSet
+ assertNotNull(reader.getSortedSetDocValues("int_is_t"));
+ assertNull(reader.getSortedNumericDocValues("int_is_t"));
+ // MultiValued Point fields use SortedNumerics
+ assertNull(reader.getSortedSetDocValues("int_is_p"));
+ assertNotNull(reader.getSortedNumericDocValues("int_is_p"));
+ } finally {
+ if (ref != null) ref.decref();
+ }
+ String s = h.query(req("q", "id:3", "qt", "/export", "fl", "int_is_t", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_t\":[1]}]}}");
+ s = h.query(req("q", "id:8", "qt", "/export", "fl", "int_is_p", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_p\":[1,1,1,1]}]}}");
+ }
+
+ /**
+ * This test doesn't validate the correctness of results, it just compares the response of the same request
+ * when asking for Trie fields vs Point fields. Can be removed once Trie fields are no longer supported
+ */
+ @Test
+ @SuppressForbidden(reason="using new Date(time) to create random dates")
+ public void testRandomNumerics() throws Exception {
+ assertU(delQ("*:*"));
+ assertU(commit());
+ List<String> trieFields = new ArrayList<String>();
+ List<String> pointFields = new ArrayList<String>();
+ for (String mv:new String[]{"s", ""}) {
+ for (String indexed:new String[]{"_ni", ""}) {
+ for (String type:new String[]{"i", "l", "f", "d", "dt"}) {
+ String field = "number_" + type + mv + indexed;
+ SchemaField sf = h.getCore().getLatestSchema().getField(field + "_t");
+ assertTrue(sf.hasDocValues());
+ assertTrue(sf.getType().getNumberType() != null);
+ assertFalse(sf.getType().isPointField());
+
+ sf = h.getCore().getLatestSchema().getField(field + "_p");
+ assertTrue(sf.hasDocValues());
+ assertTrue(sf.getType().getNumberType() != null);
+ assertTrue(sf.getType().isPointField());
+
+ trieFields.add(field + "_t");
+ pointFields.add(field + "_p");
+ }
+ }
+ }
+ for (int i = 0; i < atLeast(100); i++) {
+ if (random().nextInt(20) == 0) {
+ //have some empty docs
+ assertU(adoc("id", String.valueOf(i)));
+ continue;
+ }
+
+ if (random().nextInt(20) == 0 && i > 0) {
+ //delete some docs
+ assertU(delI(String.valueOf(i - 1)));
+ }
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", String.valueOf(i));
+ addInt(doc, random().nextInt(), false);
+ addLong(doc, random().nextLong(), false);
+ addFloat(doc, random().nextFloat() * 3000 * (random().nextBoolean()?1:-1), false);
+ addDouble(doc, random().nextDouble() * 3000 * (random().nextBoolean()?1:-1), false);
+ addDate(doc, new Date(), false);
+
+ // MV need to be unique in order to be the same in Trie vs Points
+ Set<Integer> ints = new HashSet<>();
+ Set<Long> longs = new HashSet<>();
+ Set<Float> floats = new HashSet<>();
+ Set<Double> doubles = new HashSet<>();
+ Set<Date> dates = new HashSet<>();
+ for (int j=0; j < random().nextInt(20); j++) {
+ ints.add(random().nextInt());
+ longs.add(random().nextLong());
+ floats.add(random().nextFloat() * 3000 * (random().nextBoolean()?1:-1));
+ doubles.add(random().nextDouble() * 3000 * (random().nextBoolean()?1:-1));
+ dates.add(new Date(System.currentTimeMillis() + random().nextInt()));
+ }
+ ints.stream().forEach((val)->addInt(doc, val, true));
+ longs.stream().forEach((val)->addLong(doc, val, true));
+ floats.stream().forEach((val)->addFloat(doc, val, true));
+ doubles.stream().forEach((val)->addDouble(doc, val, true));
+ dates.stream().forEach((val)->addDate(doc, val, true));
+
+ assertU(adoc(doc));
+ if (random().nextInt(20) == 0) {
+ assertU(commit());
+ }
+ }
+ assertU(commit());
+ doTestQuery("id:1", trieFields, pointFields);
+ doTestQuery("*:*", trieFields, pointFields);
+ doTestQuery("id:[0 TO 2]", trieFields, pointFields);// "id" field is really a string, this is not a numeric range query
+ doTestQuery("id:[0 TO 9]", trieFields, pointFields);
+ doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields);
+ }
+
+ private void doTestQuery(String query, List<String> trieFields, List<String> pointFields) throws Exception {
+ String trieFieldsFl = String.join(",", trieFields);
+ String pointFieldsFl = String.join(",", pointFields);
+ String sort = pickRandom((String)pickRandom(trieFields.toArray()), (String)pickRandom(pointFields.toArray())).replace("s_", "_") + pickRandom(" asc", " desc");
+ String resultPoints = h.query(req("q", query, "qt", "/export", "fl", pointFieldsFl, "sort", sort));
+ String resultTries = h.query(req("q", query, "qt", "/export", "fl", trieFieldsFl, "sort", sort));
+ assertJsonEquals(resultPoints.replaceAll("_p", ""), resultTries.replaceAll("_t", ""));
+ }
+
+ private void addFloat(SolrInputDocument doc, float value, boolean mv) {
+ addField(doc, "f", String.valueOf(value), mv);
+ }
+
+ private void addDouble(SolrInputDocument doc, double value, boolean mv) {
+ addField(doc, "d", String.valueOf(value), mv);
+ }
+
+ private void addLong(SolrInputDocument doc, long value, boolean mv) {
+ addField(doc, "l", String.valueOf(value), mv);
+ }
+
+ private void addInt(SolrInputDocument doc, int value, boolean mv) {
+ addField(doc, "i", String.valueOf(value), mv);
+ }
+
+ private void addDate(SolrInputDocument doc, Date value, boolean mv) {
+ addField(doc, "dt", new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT).format(value), mv);
+ }
+
+ private void addField(SolrInputDocument doc, String type, String value, boolean mv) {
+ doc.addField("number_" + type + (mv?"s":"") + "_t", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_p", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_ni_t", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_ni_p", value);
+ }
+
}