You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2018/07/20 19:23:07 UTC
[1/3] lucene-solr:master: SOLR-11598: Support more than 4 sort fields
in the export writer
Repository: lucene-solr
Updated Branches:
refs/heads/master 509561bf2 -> 9d9c3a0cd
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java b/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
deleted file mode 100644
index 278ef99..0000000
--- a/solr/core/src/test/org/apache/solr/response/TestExportWriter.java
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.response;
-
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Set;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.schema.SchemaField;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestExportWriter extends SolrTestCaseJ4 {
-
- @BeforeClass
- public static void beforeClass() throws Exception {
- System.setProperty("export.test", "true");
- initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml");
- }
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- assertU(delQ("*:*"));
- assertU(commit());
- createIndex();
- }
-
- public static void createIndex() {
- assertU(adoc("id","1",
- "floatdv","2.1",
- "intdv", "1",
- "stringdv", "hello world",
- "longdv", "323223232323",
- "doubledv","2344.345",
- "intdv_m","100",
- "intdv_m","250",
- "floatdv_m", "123.321",
- "floatdv_m", "345.123",
- "doubledv_m", "3444.222",
- "doubledv_m", "23232.2",
- "longdv_m", "43434343434",
- "longdv_m", "343332",
- "stringdv_m", "manchester \"city\"",
- "stringdv_m", "liverpool",
- "stringdv_m", "Everton",
- "datedv", "2017-06-16T07:00:00Z",
- "datedv_m", "2017-06-16T01:00:00Z",
- "datedv_m", "2017-06-16T02:00:00Z",
- "datedv_m", "2017-06-16T03:00:00Z",
- "datedv_m", "2017-06-16T04:00:00Z"));
-
- assertU(adoc("id","7",
- "floatdv","2.1",
- "intdv", "7",
- "longdv", "323223232323",
- "doubledv","2344.345",
- "floatdv_m", "123.321",
- "floatdv_m", "345.123",
- "doubledv_m", "3444.222",
- "doubledv_m", "23232.2",
- "longdv_m", "43434343434",
- "longdv_m", "343332"));
-
- assertU(commit());
- assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
- assertU(commit());
- assertU(adoc("id","3",
- "floatdv","2.1",
- "intdv", "3",
- "stringdv", "chello world",
- "longdv", "323223232323",
- "doubledv","2344.346",
- "intdv_m","100",
- "intdv_m","250",
- "floatdv_m", "123.321",
- "floatdv_m", "345.123",
- "doubledv_m", "3444.222",
- "doubledv_m", "23232.2",
- "longdv_m", "43434343434",
- "longdv_m", "343332",
- "stringdv_m", "manchester \"city\"",
- "stringdv_m", "liverpool",
- "stringdv_m", "everton",
- "int_is_t", "1",
- "int_is_t", "1",
- "int_is_t", "1",
- "int_is_t", "1"));
- assertU(commit());
- assertU(adoc("id","8",
- "floatdv","2.1",
- "intdv", "10000000",
- "stringdv", "chello \"world\"",
- "longdv", "323223232323",
- "doubledv","2344.346",
- "intdv_m","100",
- "intdv_m","250",
- "floatdv_m", "123.321",
- "floatdv_m", "345.123",
- "doubledv_m", "3444.222",
- "doubledv_m", "23232.2",
- "longdv_m", "43434343434",
- "longdv_m", "343332",
- "stringdv_m", "manchester \"city\"",
- "stringdv_m", "liverpool",
- "stringdv_m", "everton",
- "datedv", "2017-01-01T00:00:00Z",
- "datedv_m", "2017-01-01T01:00:00Z",
- "datedv_m", "2017-01-01T02:00:00Z",
- "int_is_p", "1",
- "int_is_p", "1",
- "int_is_p", "1",
- "int_is_p", "1"));
- assertU(commit());
-
-
- }
-
- @Test
- public void testSortingOutput() throws Exception {
-
- //Test single value DocValue output
- String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
-
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
-
- //Test null value string:
- s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
-
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
-
- //Test multiValue docValues output
- s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester \\\"city\\\"\"]}]}}");
-
- //Test multiValues docValues output with nulls
- s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
-
- //Test single sort param is working
- s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
-
- s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
-
- // Test sort on String will null value. Null value should sort last on desc and first on asc.
-
- s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}}");
-
- s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}}");
-
-
- //Test multi-sort params
- s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
-
- s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
-
- //Test three sort fields
- s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv asc,intdv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
-
- //Test three sort fields
- s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv desc,intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
-
- //Test four sort fields
- s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,floatdv desc,floatdv asc,intdv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
-
- s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}}");
-
- s = h.query(req("q", "intdv:[2 TO 1000]", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":7},{\"intdv\":2}]}}");
-
- s = h.query(req("q", "stringdv:blah", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":0, \"docs\":[]}}");
-
- s = h.query(req("q", "id:8", "qt", "/export", "fl", "stringdv", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"stringdv\":\"chello \\\"world\\\"\"}]}}");
- }
-
- private void assertJsonEquals(String actual, String expected) {
- assertEquals(Utils.toJSONString(Utils.fromJSONString(expected)), Utils.toJSONString(Utils.fromJSONString(actual)));
- }
-
- @Test
- public void testExportRequiredParams() throws Exception {
-
- //Test whether missing required parameters returns expected errors.
-
- //String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
- String s;
- s = h.query(req("qt", "/export"));
- assertTrue("Should have had a sort error", s.contains("No sort criteria"));
- s = h.query(req("sort", "intdv asc", "qt", "/export"));
- assertTrue("Should have had fl error", s.contains("export field list (fl) must be specified"));
- s = h.query(req("sort", "intdv asc", "qt", "/export", "fl", "stringdv"));
- // Interesting you don't even need to specify a "q" parameter.
-
- }
-
- @Test
- public void testDates() throws Exception {
- String s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv", "sort", "datedv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv\":\"2017-06-16T07:00:00Z\"}]}}");
- s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv_m", "sort", "datedv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv_m\":[\"2017-06-16T01:00:00Z\",\"2017-06-16T02:00:00Z\",\"2017-06-16T03:00:00Z\",\"2017-06-16T04:00:00Z\"]}]}}");
- }
-
- @Test
- public void testDuplicates() throws Exception {
- // see SOLR-10924
- String expected = h.getCore().getLatestSchema().getField("int_is_t").getType().isPointField()
- ? "1,1,1,1" : "1";
- String s = h.query(req("q", "id:3", "qt", "/export", "fl", "int_is_t", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_t\":["+expected+"]}]}}");
- expected = h.getCore().getLatestSchema().getField("int_is_p").getType().isPointField()
- ? "1,1,1,1" : "1";
- s = h.query(req("q", "id:8", "qt", "/export", "fl", "int_is_p", "sort", "intdv asc"));
- assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_p\":[1,1,1,1]}]}}");
- }
-
- /**
- * This test doesn't validate the correctness of results, it just compares the response of the same request
- * when asking for Trie fields vs Point fields. Can be removed once Trie fields are no longer supported
- */
- @Test
- @SuppressForbidden(reason="using new Date(time) to create random dates")
- public void testRandomNumerics() throws Exception {
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
- assertU(delQ("*:*"));
- assertU(commit());
- List<String> trieFields = new ArrayList<String>();
- List<String> pointFields = new ArrayList<String>();
- for (String mv:new String[]{"s", ""}) {
- for (String indexed:new String[]{"_ni", ""}) {
- for (String type:new String[]{"i", "l", "f", "d", "dt"}) {
- String field = "number_" + type + mv + indexed;
- SchemaField sf = h.getCore().getLatestSchema().getField(field + "_t");
- assertTrue(sf.hasDocValues());
- assertTrue(sf.getType().getNumberType() != null);
-
- sf = h.getCore().getLatestSchema().getField(field + "_p");
- assertTrue(sf.hasDocValues());
- assertTrue(sf.getType().getNumberType() != null);
- assertTrue(sf.getType().isPointField());
-
- trieFields.add(field + "_t");
- pointFields.add(field + "_p");
- }
- }
- }
- for (int i = 0; i < atLeast(100); i++) {
- if (random().nextInt(20) == 0) {
- //have some empty docs
- assertU(adoc("id", String.valueOf(i)));
- continue;
- }
-
- if (random().nextInt(20) == 0 && i > 0) {
- //delete some docs
- assertU(delI(String.valueOf(i - 1)));
- }
-
- SolrInputDocument doc = new SolrInputDocument();
- doc.addField("id", String.valueOf(i));
- addInt(doc, random().nextInt(), false);
- addLong(doc, random().nextLong(), false);
- addFloat(doc, random().nextFloat() * 3000 * (random().nextBoolean()?1:-1), false);
- addDouble(doc, random().nextDouble() * 3000 * (random().nextBoolean()?1:-1), false);
- addDate(doc, dateFormat.format(new Date()), false);
-
- // MV need to be unique in order to be the same in Trie vs Points
- Set<Integer> ints = new HashSet<>();
- Set<Long> longs = new HashSet<>();
- Set<Float> floats = new HashSet<>();
- Set<Double> doubles = new HashSet<>();
- Set<String> dates = new HashSet<>();
- for (int j=0; j < random().nextInt(20); j++) {
- ints.add(random().nextInt());
- longs.add(random().nextLong());
- floats.add(random().nextFloat() * 3000 * (random().nextBoolean()?1:-1));
- doubles.add(random().nextDouble() * 3000 * (random().nextBoolean()?1:-1));
- dates.add(dateFormat.format(new Date(System.currentTimeMillis() + random().nextInt())));
- }
- ints.stream().forEach((val)->addInt(doc, val, true));
- longs.stream().forEach((val)->addLong(doc, val, true));
- floats.stream().forEach((val)->addFloat(doc, val, true));
- doubles.stream().forEach((val)->addDouble(doc, val, true));
- dates.stream().forEach((val)->addDate(doc, val, true));
-
- assertU(adoc(doc));
- if (random().nextInt(20) == 0) {
- assertU(commit());
- }
- }
- assertU(commit());
- doTestQuery("id:1", trieFields, pointFields);
- doTestQuery("*:*", trieFields, pointFields);
- doTestQuery("id:[0 TO 2]", trieFields, pointFields);// "id" field is really a string, this is not a numeric range query
- doTestQuery("id:[0 TO 9]", trieFields, pointFields);
- doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields);
- }
-
- private void doTestQuery(String query, List<String> trieFields, List<String> pointFields) throws Exception {
- String trieFieldsFl = String.join(",", trieFields);
- String pointFieldsFl = String.join(",", pointFields);
- String sort = pickRandom((String)pickRandom(trieFields.toArray()), (String)pickRandom(pointFields.toArray())).replace("s_", "_") + pickRandom(" asc", " desc");
- String resultPoints = h.query(req("q", query, "qt", "/export", "fl", pointFieldsFl, "sort", sort));
- String resultTries = h.query(req("q", query, "qt", "/export", "fl", trieFieldsFl, "sort", sort));
- assertJsonEquals(resultPoints.replaceAll("_p", ""), resultTries.replaceAll("_t", ""));
- }
-
- private void addFloat(SolrInputDocument doc, float value, boolean mv) {
- addField(doc, "f", String.valueOf(value), mv);
- }
-
- private void addDouble(SolrInputDocument doc, double value, boolean mv) {
- addField(doc, "d", String.valueOf(value), mv);
- }
-
- private void addLong(SolrInputDocument doc, long value, boolean mv) {
- addField(doc, "l", String.valueOf(value), mv);
- }
-
- private void addInt(SolrInputDocument doc, int value, boolean mv) {
- addField(doc, "i", String.valueOf(value), mv);
- }
-
- private void addDate(SolrInputDocument doc, String value, boolean mv) {
- addField(doc, "dt", value, mv);
- }
-
- private void addField(SolrInputDocument doc, String type, String value, boolean mv) {
- doc.addField("number_" + type + (mv?"s":"") + "_t", value);
- doc.addField("number_" + type + (mv?"s":"") + "_p", value);
- doc.addField("number_" + type + (mv?"s":"") + "_ni_t", value);
- doc.addField("number_" + type + (mv?"s":"") + "_ni_p", value);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/solr-ref-guide/src/exporting-result-sets.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/exporting-result-sets.adoc b/solr/solr-ref-guide/src/exporting-result-sets.adoc
index be1811f..b0565d8 100644
--- a/solr/solr-ref-guide/src/exporting-result-sets.adoc
+++ b/solr/solr-ref-guide/src/exporting-result-sets.adoc
@@ -52,7 +52,8 @@ http://localhost:8983/solr/core_name/export?q=my-query&sort=severity+desc,timest
The `sort` property defines how documents will be sorted in the exported result set. Results can be sorted by any field that has a field type of int,long, float, double, string. The sort fields must be single valued fields.
-Up to four sort fields can be specified per request, with the 'asc' or 'desc' properties.
+The export performance will get slower as you add more sort fields. If there is enough physical memory available outside of the JVM to load up the sort fields then the performance will be linearly slower with addition of sort fields.
+It can get worse otherwise.
=== Specifying the Field List
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/solr-ref-guide/src/parallel-sql-interface.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adoc
index 9fb3d16..01442ee 100644
--- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc
+++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc
@@ -321,10 +321,6 @@ The Column Identifiers can contain both fields in the Solr index and aggregate f
The non-function fields in the field list determine the fields to calculate the aggregations over.
-*`GROUP BY` Clause*
-
-The `GROUP BY` clause can contain up to 4 fields in the Solr index. These fields should correspond with the non-function fields in the field list.
-
=== HAVING Clause
The `HAVING` clause may contain any function listed in the field list. Complex `HAVING` clauses such as this are supported:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/solrj/src/java/org/apache/solr/common/SolrException.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/SolrException.java b/solr/solrj/src/java/org/apache/solr/common/SolrException.java
index f73f5dd..673bae7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/SolrException.java
+++ b/solr/solrj/src/java/org/apache/solr/common/SolrException.java
@@ -213,7 +213,7 @@ public class SolrException extends RuntimeException {
return null;
}
- public static Throwable getRootCause(Throwable t) {
+ public static Throwable getRootCause(Throwable t) {
while (true) {
Throwable cause = t.getCause();
if (cause!=null) {
[3/3] lucene-solr:master: SOLR-11598: Support more than 4 sort fields
in the export writer
Posted by va...@apache.org.
SOLR-11598: Support more than 4 sort fields in the export writer
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9d9c3a0c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9d9c3a0c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9d9c3a0c
Branch: refs/heads/master
Commit: 9d9c3a0cd87832980a4745ec96fb2cd1216dcb4e
Parents: 509561b
Author: Varun Thacker <va...@apache.org>
Authored: Fri Jul 20 11:38:02 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Fri Jul 20 11:38:07 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +
.../org/apache/solr/handler/ExportHandler.java | 1 +
.../org/apache/solr/handler/ExportWriter.java | 1764 ------------------
.../solr/handler/export/BoolFieldWriter.java | 52 +
.../solr/handler/export/DateFieldWriter.java | 46 +
.../apache/solr/handler/export/DoubeValue.java | 81 +
.../apache/solr/handler/export/DoubleCmp.java | 43 +
.../solr/handler/export/DoubleFieldWriter.java | 45 +
.../solr/handler/export/ExportWriter.java | 448 +++++
.../apache/solr/handler/export/FieldWriter.java | 27 +
.../apache/solr/handler/export/FloatCmp.java | 44 +
.../solr/handler/export/FloatFieldWriter.java | 45 +
.../apache/solr/handler/export/FloatValue.java | 78 +
.../org/apache/solr/handler/export/IntComp.java | 45 +
.../solr/handler/export/IntFieldWriter.java | 45 +
.../apache/solr/handler/export/IntValue.java | 77 +
.../org/apache/solr/handler/export/LongCmp.java | 45 +
.../solr/handler/export/LongFieldWriter.java | 45 +
.../apache/solr/handler/export/LongValue.java | 78 +
.../solr/handler/export/MultiFieldWriter.java | 104 ++
.../solr/handler/export/PriorityQueue.java | 218 +++
.../org/apache/solr/handler/export/SortDoc.java | 114 ++
.../apache/solr/handler/export/SortQueue.java | 52 +
.../apache/solr/handler/export/SortValue.java | 30 +
.../solr/handler/export/StringFieldWriter.java | 52 +
.../apache/solr/handler/export/StringValue.java | 99 +
.../solr/handler/export/package-info.java | 23 +
.../collection1/conf/schema-sortingresponse.xml | 5 +-
.../solr/handler/export/TestExportWriter.java | 749 ++++++++
.../apache/solr/response/TestExportWriter.java | 370 ----
.../src/exporting-result-sets.adoc | 3 +-
.../src/parallel-sql-interface.adoc | 4 -
.../org/apache/solr/common/SolrException.java | 2 +-
33 files changed, 2697 insertions(+), 2141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 449f91d..e6d468c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -169,6 +169,10 @@ Optimizations
* SOLR-11654: Time Routed Alias will now route documents to the ideal shard of a collection, thus avoiding a hop.
Usually documents were already routed well but not always. (Gus Heck, David Smiley)
+* SOLR-11598: The export handler does not limit users to 4 sort fields and is now unlimited. However the speed at
+ which we can export is directly proportional to the number of sort fields specified. This change also allows streaming
+ expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
index 9c75ef0..ea9239d 100644
--- a/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ExportHandler.java
@@ -25,6 +25,7 @@ import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.handler.component.SearchHandler;
+import org.apache.solr.handler.export.ExportWriter;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
deleted file mode 100644
index f618eef..0000000
--- a/solr/core/src/java/org/apache/solr/handler/ExportWriter.java
+++ /dev/null
@@ -1,1764 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.handler;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.Date;
-import java.util.List;
-import java.util.function.LongFunction;
-
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.OrdinalMap;
-import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.SortedNumericDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Sort;
-import org.apache.lucene.search.SortField;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CharsRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.LongValues;
-import org.apache.lucene.util.NumericUtils;
-import org.apache.solr.client.solrj.impl.BinaryResponseParser;
-import org.apache.solr.common.IteratorWriter;
-import org.apache.solr.common.MapWriter.EntryWriter;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.PushWriter;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.JavaBinCodec;
-import org.apache.solr.core.SolrCore;
-import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrRequestInfo;
-import org.apache.solr.response.BinaryResponseWriter;
-import org.apache.solr.response.JSONResponseWriter;
-import org.apache.solr.response.QueryResponseWriter;
-import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.schema.BoolField;
-import org.apache.solr.schema.DateValueFieldType;
-import org.apache.solr.schema.DoubleValueFieldType;
-import org.apache.solr.schema.FieldType;
-import org.apache.solr.schema.FloatValueFieldType;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.IntValueFieldType;
-import org.apache.solr.schema.LongValueFieldType;
-import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.StrField;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.SortSpec;
-import org.apache.solr.search.SyntaxError;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static java.util.Collections.singletonList;
-import static java.util.Collections.singletonMap;
-import static org.apache.solr.common.util.Utils.makeMap;
-
-public class ExportWriter implements SolrCore.RawWriter, Closeable {
- private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private OutputStreamWriter respWriter;
- final SolrQueryRequest req;
- final SolrQueryResponse res;
- FieldWriter[] fieldWriters;
- int totalHits = 0;
- FixedBitSet[] sets = null;
- PushWriter writer;
- private String wt;
-
-
- ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
- this.req = req;
- this.res = res;
- this.wt = wt;
-
- }
-
- @Override
- public String getContentType() {
- if ("javabin".equals(wt)) {
- return BinaryResponseParser.BINARY_CONTENT_TYPE;
- } else return "json";
- }
-
- @Override
- public void close() throws IOException {
- if (writer != null) writer.close();
- if (respWriter != null) {
- respWriter.flush();
- respWriter.close();
- }
-
- }
-
- protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
- w.writeMap(mw -> {
- mw.put("responseHeader", singletonMap("status", 400))
- .put("response", makeMap(
- "numFound", 0,
- "docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
- });
- if (log) {
- SolrException.log(logger, e);
- }
- }
-
- public void write(OutputStream os) throws IOException {
- QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
- if (rw instanceof BinaryResponseWriter) {
- //todo add support for other writers after testing
- writer = new JavaBinCodec(os, null);
- } else {
- respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
- writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
- }
- Exception exception = res.getException();
- if (exception != null) {
- if (!(exception instanceof IgnoreException)) {
- writeException(exception, writer, false);
- }
- return;
- }
-
- SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
- SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
-
- if(sortSpec == null) {
- writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
- return;
- }
-
- SolrIndexSearcher searcher = req.getSearcher();
- Sort sort = searcher.weightSort(sortSpec.getSort());
-
- if(sort == null) {
- writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
- return;
- }
-
- if(sort != null && sort.needsScores()) {
- writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
- return;
- }
-
- // There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
- // if (lastDocRequested <= 0) {
- // That causes the totalHits and export entries in the context to _not_ get set.
- // The only time that really matters is when we search against an _empty_ set. That's too obscure
- // a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
- // ant test -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
- // You'll have to uncomment the if below to hit the null pointer exception.
- // This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
- // This came to light in the very artifical case of indexing a single doc to Cloud.
- if (req.getContext().get("totalHits") != null) {
- totalHits = ((Integer)req.getContext().get("totalHits")).intValue();
- sets = (FixedBitSet[]) req.getContext().get("export");
- if (sets == null) {
- writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
- return;
- }
- }
- SolrParams params = req.getParams();
- String fl = params.get("fl");
-
- String[] fields = null;
-
- if(fl == null) {
- writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
- return;
- } else {
- fields = fl.split(",");
-
- for(int i=0;i<fields.length; i++) {
-
- fields[i] = fields[i].trim();
-
- if(fields[i].equals("score")) {
- writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
- return;
- }
- }
- }
-
- try {
- fieldWriters = getFieldWriters(fields, req.getSearcher());
- } catch (Exception e) {
- writeException(e, writer, true);
- return;
- }
-
- writer.writeMap(m -> {
- m.put("responseHeader", singletonMap("status", 0));
- m.put("response", (MapWriter) mw -> {
- mw.put("numFound", totalHits);
- mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
- });
- });
-
- }
-
- protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
- //Write the data.
- List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
- SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
- int count = 0;
- int queueSize = 30000;
- SortQueue queue = new SortQueue(queueSize, sortDoc);
- SortDoc[] outDocs = new SortDoc[queueSize];
-
- while(count < totalHits) {
- //long begin = System.nanoTime();
- queue.reset();
- SortDoc top = queue.top();
- for(int i=0; i<leaves.size(); i++) {
- sortDoc.setNextReader(leaves.get(i));
- DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
- int docId = -1;
- while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
- sortDoc.setValues(docId);
- if(top.lessThan(sortDoc)) {
- top.setValues(sortDoc);
- top = queue.updateTop();
- }
- }
- }
-
- int outDocsIndex = -1;
-
- for(int i=0; i<queueSize; i++) {
- SortDoc s = queue.pop();
- if(s.docId > -1) {
- outDocs[++outDocsIndex] = s;
- }
- }
-
- //long end = System.nanoTime();
-
- count += (outDocsIndex+1);
-
- try {
- for(int i=outDocsIndex; i>=0; --i) {
- SortDoc s = outDocs[i];
- writer.add((MapWriter) ew -> {
- writeDoc(s, leaves, ew);
- s.reset();
- });
- }
- } catch(Throwable e) {
- Throwable ex = e;
- while(ex != null) {
- String m = ex.getMessage();
- if(m != null && m.contains("Broken pipe")) {
- throw new IgnoreException();
- }
- ex = ex.getCause();
- }
-
- if(e instanceof IOException) {
- throw ((IOException)e);
- } else {
- throw new IOException(e);
- }
- }
- }
- }
-
- protected void writeDoc(SortDoc sortDoc,
- List<LeafReaderContext> leaves,
- EntryWriter ew) throws IOException {
-
- int ord = sortDoc.ord;
- FixedBitSet set = sets[ord];
- set.clear(sortDoc.docId);
- LeafReaderContext context = leaves.get(ord);
- int fieldIndex = 0;
- for (FieldWriter fieldWriter : fieldWriters) {
- if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
- ++fieldIndex;
- }
- }
- }
-
- protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
- IndexSchema schema = searcher.getSchema();
- FieldWriter[] writers = new FieldWriter[fields.length];
- for(int i=0; i<fields.length; i++) {
- String field = fields[i];
- SchemaField schemaField = null;
-
- try {
- schemaField = schema.getField(field);
- } catch (Exception e) {
- throw new IOException(e);
- }
-
- if(!schemaField.hasDocValues()) {
- throw new IOException(field+" must have DocValues to use this feature.");
- }
-
- boolean multiValued = schemaField.multiValued();
- FieldType fieldType = schemaField.getType();
- if (fieldType instanceof IntValueFieldType) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
- } else {
- writers[i] = new IntFieldWriter(field);
- }
- } else if (fieldType instanceof LongValueFieldType) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
- } else {
- writers[i] = new LongFieldWriter(field);
- }
- } else if (fieldType instanceof FloatValueFieldType) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
- } else {
- writers[i] = new FloatFieldWriter(field);
- }
- } else if (fieldType instanceof DoubleValueFieldType) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
- } else {
- writers[i] = new DoubleFieldWriter(field);
- }
- } else if (fieldType instanceof StrField) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
- } else {
- writers[i] = new StringFieldWriter(field, fieldType);
- }
- } else if (fieldType instanceof DateValueFieldType) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
- } else {
- writers[i] = new DateFieldWriter(field);
- }
- } else if (fieldType instanceof BoolField) {
- if (multiValued) {
- writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
- } else {
- writers[i] = new BoolFieldWriter(field, fieldType);
- }
- } else {
- throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
- }
- }
- return writers;
- }
-
- private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
- SortValue[] sortValues = new SortValue[sortFields.length];
- IndexSchema schema = searcher.getSchema();
- for(int i=0; i<sortFields.length; ++i) {
- SortField sf = sortFields[i];
- String field = sf.getField();
- boolean reverse = sf.getReverse();
- SchemaField schemaField = schema.getField(field);
- FieldType ft = schemaField.getType();
-
- if(!schemaField.hasDocValues()) {
- throw new IOException(field+" must have DocValues to use this feature.");
- }
-
- if(ft instanceof IntValueFieldType) {
- if(reverse) {
- sortValues[i] = new IntValue(field, new IntDesc());
- } else {
- sortValues[i] = new IntValue(field, new IntAsc());
- }
- } else if(ft instanceof FloatValueFieldType) {
- if(reverse) {
- sortValues[i] = new FloatValue(field, new FloatDesc());
- } else {
- sortValues[i] = new FloatValue(field, new FloatAsc());
- }
- } else if(ft instanceof DoubleValueFieldType) {
- if(reverse) {
- sortValues[i] = new DoubleValue(field, new DoubleDesc());
- } else {
- sortValues[i] = new DoubleValue(field, new DoubleAsc());
- }
- } else if(ft instanceof LongValueFieldType) {
- if(reverse) {
- sortValues[i] = new LongValue(field, new LongDesc());
- } else {
- sortValues[i] = new LongValue(field, new LongAsc());
- }
- } else if(ft instanceof StrField) {
- LeafReader reader = searcher.getSlowAtomicReader();
- SortedDocValues vals = reader.getSortedDocValues(field);
- if(reverse) {
- sortValues[i] = new StringValue(vals, field, new IntDesc());
- } else {
- sortValues[i] = new StringValue(vals, field, new IntAsc());
- }
- } else if (ft instanceof DateValueFieldType) {
- if (reverse) {
- sortValues[i] = new LongValue(field, new LongDesc());
- } else {
- sortValues[i] = new LongValue(field, new LongAsc());
- }
- } else if (ft instanceof BoolField) {
- // This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
- // _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
- // we can just use the existing StringValue here.
- LeafReader reader = searcher.getSlowAtomicReader();
- SortedDocValues vals = reader.getSortedDocValues(field);
- if (reverse) {
- sortValues[i] = new StringValue(vals, field, new IntDesc());
- } else {
- sortValues[i] = new StringValue(vals, field, new IntAsc());
- }
- } else {
- throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
- }
- }
-
- if(sortValues.length == 1) {
- return new SingleValueSortDoc(sortValues[0]);
- } else if(sortValues.length == 2) {
- return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
- } else if(sortValues.length == 3) {
- return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
- } else if(sortValues.length == 4) {
- return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
- } else {
- throw new IOException("A max of 4 sorts can be specified");
- }
- }
-
- class SortQueue extends PriorityQueue<SortDoc> {
-
- private SortDoc proto;
- private Object[] cache;
-
- public SortQueue(int len, SortDoc proto) {
- super(len);
- this.proto = proto;
- }
-
- protected boolean lessThan(SortDoc t1, SortDoc t2) {
- return t1.lessThan(t2);
- }
-
- private void populate() {
- Object[] heap = getHeapArray();
- cache = new SortDoc[heap.length];
- for (int i = 1; i < heap.length; i++) {
- cache[i] = heap[i] = proto.copy();
- }
- size = maxSize;
- }
-
- private void reset() {
- Object[] heap = getHeapArray();
- if(cache != null) {
- System.arraycopy(cache, 1, heap, 1, heap.length-1);
- size = maxSize;
- } else {
- populate();
- }
- }
- }
-
- class SortDoc {
-
- protected int docId = -1;
- protected int ord = -1;
- protected int docBase = -1;
-
- private SortValue[] sortValues;
-
- public SortDoc() {
-
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.ord = context.ord;
- for (SortValue value : sortValues) {
- value.setNextReader(context);
- }
- }
-
- public void reset() {
- this.docId = -1;
- }
-
- public void setValues(int docId) throws IOException {
- this.docId = docId;
- for(SortValue sortValue : sortValues) {
- sortValue.setCurrentValue(docId);
- }
- }
-
- public void setValues(SortDoc sortDoc) throws IOException {
- this.docId = sortDoc.docId;
- this.ord = sortDoc.ord;
- SortValue[] vals = sortDoc.sortValues;
- for(int i=0; i<vals.length; i++) {
- sortValues[i].setCurrentValue(vals[i]);
- }
- }
-
- public SortDoc(SortValue[] sortValues) {
- this.sortValues = sortValues;
- }
-
- public SortDoc copy() {
- SortValue[] svs = new SortValue[sortValues.length];
- for(int i=0; i<sortValues.length; i++) {
- svs[i] = sortValues[i].copy();
- }
-
- return new SortDoc(svs);
- }
-
- public boolean lessThan(Object o) {
- if(docId == -1) {
- return true;
- }
-
- SortDoc sd = (SortDoc)o;
- SortValue[] sortValues1 = sd.sortValues;
- for(int i=0; i<sortValues.length; i++) {
- int comp = sortValues[i].compareTo(sortValues1[i]);
- if(comp < 0) {
- return true;
- } if(comp > 0) {
- return false;
- }
- }
- return docId+docBase < sd.docId+sd.docBase;
- }
-
- public String toString() {
- return "";
- }
- }
-
- class SingleValueSortDoc extends SortDoc {
-
- protected SortValue value1;
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.ord = context.ord;
- value1.setNextReader(context);
- }
-
- public void reset() {
- this.docId = -1;
- this.value1.reset();
- }
-
- public void setValues(int docId) throws IOException {
- this.docId = docId;
- value1.setCurrentValue(docId);
- }
-
- public void setValues(SortDoc sortDoc) throws IOException {
- this.docId = sortDoc.docId;
- this.ord = sortDoc.ord;
- value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
- }
-
- public SingleValueSortDoc(SortValue value1) {
- super();
- this.value1 = value1;
- }
-
- public SortDoc copy() {
- return new SingleValueSortDoc(value1.copy());
- }
-
- public boolean lessThan(Object o) {
- SingleValueSortDoc sd = (SingleValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- return docId+docBase > sd.docId+sd.docBase;
- }
- }
-
- public int compareTo(Object o) {
- SingleValueSortDoc sd = (SingleValueSortDoc)o;
- return value1.compareTo(sd.value1);
- }
-
- public String toString() {
- return docId+":"+value1.toString();
- }
- }
-
- class DoubleValueSortDoc extends SingleValueSortDoc {
-
- protected SortValue value2;
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.ord = context.ord;
- value1.setNextReader(context);
- value2.setNextReader(context);
- }
-
- public void reset() {
- this.docId = -1;
- value1.reset();
- value2.reset();
- }
-
- public void setValues(int docId) throws IOException {
- this.docId = docId;
- value1.setCurrentValue(docId);
- value2.setCurrentValue(docId);
- }
-
- public void setValues(SortDoc sortDoc) throws IOException {
- this.docId = sortDoc.docId;
- this.ord = sortDoc.ord;
- value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
- value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
- }
-
- public DoubleValueSortDoc(SortValue value1, SortValue value2) {
- super(value1);
- this.value2 = value2;
- }
-
- public SortDoc copy() {
- return new DoubleValueSortDoc(value1.copy(), value2.copy());
- }
-
- public boolean lessThan(Object o) {
- DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value2.compareTo(sd.value2);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- return docId+docBase > sd.docId+sd.docBase;
- }
- }
- }
-
- public int compareTo(Object o) {
- DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == 0) {
- return value2.compareTo(sd.value2);
- } else {
- return comp;
- }
- }
- }
-
- class TripleValueSortDoc extends DoubleValueSortDoc {
-
- protected SortValue value3;
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.ord = context.ord;
- value1.setNextReader(context);
- value2.setNextReader(context);
- value3.setNextReader(context);
- }
-
- public void reset() {
- this.docId = -1;
- value1.reset();
- value2.reset();
- value3.reset();
- }
-
- public void setValues(int docId) throws IOException {
- this.docId = docId;
- value1.setCurrentValue(docId);
- value2.setCurrentValue(docId);
- value3.setCurrentValue(docId);
- }
-
- public void setValues(SortDoc sortDoc) throws IOException {
- this.docId = sortDoc.docId;
- this.ord = sortDoc.ord;
- value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
- value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
- value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
- }
-
- public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
- super(value1, value2);
- this.value3 = value3;
- }
-
- public SortDoc copy() {
- return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
- }
-
- public boolean lessThan(Object o) {
-
- TripleValueSortDoc sd = (TripleValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value2.compareTo(sd.value2);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value3.compareTo(sd.value3);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- return docId+docBase > sd.docId+sd.docBase;
- }
- }
- }
- }
-
- public int compareTo(Object o) {
-
- TripleValueSortDoc sd = (TripleValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == 0) {
- comp = value2.compareTo(sd.value2);
- if(comp == 0) {
- return value3.compareTo(sd.value3);
- } else {
- return comp;
- }
- } else {
- return comp;
- }
- }
- }
-
- class QuadValueSortDoc extends TripleValueSortDoc {
-
- protected SortValue value4;
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.ord = context.ord;
- value1.setNextReader(context);
- value2.setNextReader(context);
- value3.setNextReader(context);
- value4.setNextReader(context);
- }
-
- public void reset() {
- this.docId = -1;
- value1.reset();
- value2.reset();
- value3.reset();
- value4.reset();
- }
-
- public void setValues(int docId) throws IOException {
- this.docId = docId;
- value1.setCurrentValue(docId);
- value2.setCurrentValue(docId);
- value3.setCurrentValue(docId);
- value4.setCurrentValue(docId);
- }
-
- public void setValues(SortDoc sortDoc) throws IOException {
- this.docId = sortDoc.docId;
- this.ord = sortDoc.ord;
- value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
- value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
- value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
- value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
- }
-
- public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
- super(value1, value2, value3);
- this.value4 = value4;
- }
-
- public SortDoc copy() {
- return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
- }
-
- public boolean lessThan(Object o) {
-
- QuadValueSortDoc sd = (QuadValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value2.compareTo(sd.value2);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value3.compareTo(sd.value3);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- comp = value4.compareTo(sd.value4);
- if(comp == -1) {
- return true;
- } else if (comp == 1) {
- return false;
- } else {
- return docId+docBase > sd.docId+sd.docBase;
- }
- }
- }
- }
- }
-
- public int compareTo(Object o) {
- QuadValueSortDoc sd = (QuadValueSortDoc)o;
- int comp = value1.compareTo(sd.value1);
- if(comp == 0) {
- comp = value2.compareTo(sd.value2);
- if(comp == 0) {
- comp = value3.compareTo(sd.value3);
- if(comp == 0) {
- return value4.compareTo(sd.value4);
- } else {
- return comp;
- }
- } else {
- return comp;
- }
- } else {
- return comp;
- }
- }
- }
-
- public interface SortValue extends Comparable<SortValue> {
- public void setCurrentValue(int docId) throws IOException;
- public void setNextReader(LeafReaderContext context) throws IOException;
- public void setCurrentValue(SortValue value);
- public void reset();
- public SortValue copy();
- }
-
- class IntValue implements SortValue {
-
- protected NumericDocValues vals;
- protected String field;
- protected int currentValue;
- protected IntComp comp;
- private int lastDocID;
-
- public IntValue copy() {
- return new IntValue(field, comp);
- }
-
- public IntValue(String field, IntComp comp) {
- this.field = field;
- this.comp = comp;
- this.currentValue = comp.resetValue();
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = DocValues.getNumeric(context.reader(), field);
- lastDocID = 0;
- }
-
- public void setCurrentValue(int docId) throws IOException {
- if (docId < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
- }
- lastDocID = docId;
- int curDocID = vals.docID();
- if (docId > curDocID) {
- curDocID = vals.advance(docId);
- }
- if (docId == curDocID) {
- currentValue = (int) vals.longValue();
- } else {
- currentValue = 0;
- }
- }
-
- public int compareTo(SortValue o) {
- IntValue iv = (IntValue)o;
- return comp.compare(currentValue, iv.currentValue);
- }
-
- public void setCurrentValue (SortValue value) {
- currentValue = ((IntValue)value).currentValue;
- }
-
- public void reset() {
- currentValue = comp.resetValue();
- }
- }
-
- public interface IntComp {
- public int compare(int a, int b);
- public int resetValue();
- }
-
- static class IntDesc implements IntComp {
-
- public int resetValue() {
- return Integer.MIN_VALUE;
- }
-
- public int compare(int a, int b) {
- if(a < b) {
- return -1;
- } else if (a > b) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
- static class IntAsc implements IntComp {
-
- public int resetValue() {
- return Integer.MAX_VALUE;
- }
-
- public int compare(int a, int b) {
- if(a < b) {
- return 1;
- } else if (a > b) {
- return -1;
- } else {
- return 0;
- }
- }
- }
-
- class LongValue implements SortValue {
-
- protected NumericDocValues vals;
- protected String field;
- protected long currentValue;
- protected LongComp comp;
- private int lastDocID;
-
- public LongValue(String field, LongComp comp) {
- this.field = field;
- this.comp = comp;
- this.currentValue = comp.resetValue();
- }
-
- public LongValue copy() {
- return new LongValue(field, comp);
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = DocValues.getNumeric(context.reader(), field);
- lastDocID = 0;
- }
-
- public void setCurrentValue(int docId) throws IOException {
- if (docId < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
- }
- lastDocID = docId;
- int curDocID = vals.docID();
- if (docId > curDocID) {
- curDocID = vals.advance(docId);
- }
- if (docId == curDocID) {
- currentValue = vals.longValue();
- } else {
- currentValue = 0;
- }
- }
-
- public void setCurrentValue(SortValue sv) {
- LongValue lv = (LongValue)sv;
- this.currentValue = lv.currentValue;
- }
-
- public int compareTo(SortValue o) {
- LongValue l = (LongValue)o;
- return comp.compare(currentValue, l.currentValue);
- }
-
- public void reset() {
- this.currentValue = comp.resetValue();
- }
- }
-
- interface LongComp {
- public int compare(long a, long b);
- public long resetValue();
- }
-
- static class LongDesc implements LongComp {
-
- public long resetValue() {
- return Long.MIN_VALUE;
- }
-
- public int compare(long a, long b) {
- if(a < b) {
- return -1;
- } else if (a > b) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
- static class LongAsc implements LongComp {
-
- public long resetValue() {
- return Long.MAX_VALUE;
- }
-
- public int compare(long a, long b) {
- if(a < b) {
- return 1;
- } else if (a > b) {
- return -1;
- } else {
- return 0;
- }
- }
- }
-
- class FloatValue implements SortValue {
-
- protected NumericDocValues vals;
- protected String field;
- protected float currentValue;
- protected FloatComp comp;
- private int lastDocID;
-
- public FloatValue(String field, FloatComp comp) {
- this.field = field;
- this.comp = comp;
- this.currentValue = comp.resetValue();
- }
-
- public FloatValue copy() {
- return new FloatValue(field, comp);
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.vals = DocValues.getNumeric(context.reader(), field);
- lastDocID = 0;
- }
-
- public void setCurrentValue(int docId) throws IOException {
- if (docId < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
- }
- lastDocID = docId;
- int curDocID = vals.docID();
- if (docId > curDocID) {
- curDocID = vals.advance(docId);
- }
- if (docId == curDocID) {
- currentValue = Float.intBitsToFloat((int)vals.longValue());
- } else {
- currentValue = 0f;
- }
- }
-
- public void setCurrentValue(SortValue sv) {
- FloatValue fv = (FloatValue)sv;
- this.currentValue = fv.currentValue;
- }
-
- public void reset() {
- this.currentValue = comp.resetValue();
- }
-
- public int compareTo(SortValue o) {
- FloatValue fv = (FloatValue)o;
- return comp.compare(currentValue, fv.currentValue);
- }
- }
-
- interface FloatComp {
- public int compare(float a, float b);
- public float resetValue();
- }
-
- public static class FloatDesc implements FloatComp {
- public float resetValue() {
- return -Float.MAX_VALUE;
- }
-
- public int compare(float a, float b) {
- if(a < b) {
- return -1;
- } else if (a > b) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
- public static class FloatAsc implements FloatComp {
- public float resetValue() {
- return Float.MAX_VALUE;
- }
-
- public int compare(float a, float b) {
- if(a < b) {
- return 1;
- } else if (a > b) {
- return -1;
- } else {
- return 0;
- }
- }
- }
-
- class DoubleValue implements SortValue {
-
- protected NumericDocValues vals;
- protected String field;
- protected double currentValue;
- protected DoubleComp comp;
- private int lastDocID;
- private LeafReader reader;
-
- public DoubleValue(String field, DoubleComp comp) {
- this.field = field;
- this.comp = comp;
- this.currentValue = comp.resetValue();
- }
-
- public DoubleValue copy() {
- return new DoubleValue(field, comp);
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- this.reader = context.reader();
- this.vals = DocValues.getNumeric(this.reader, this.field);
- lastDocID = 0;
- }
-
- public void setCurrentValue(int docId) throws IOException {
- if (docId < lastDocID) {
- // TODO: can we enforce caller to go in order instead?
- this.vals = DocValues.getNumeric(this.reader, this.field);
- }
- lastDocID = docId;
- int curDocID = vals.docID();
- if (docId > curDocID) {
- curDocID = vals.advance(docId);
- }
- if (docId == curDocID) {
- currentValue = Double.longBitsToDouble(vals.longValue());
- } else {
- currentValue = 0f;
- }
- }
-
- public void setCurrentValue(SortValue sv) {
- DoubleValue dv = (DoubleValue)sv;
- this.currentValue = dv.currentValue;
- }
-
- public void reset() {
- this.currentValue = comp.resetValue();
- }
-
- public int compareTo(SortValue o) {
- DoubleValue dv = (DoubleValue)o;
- return comp.compare(currentValue, dv.currentValue);
- }
- }
-
- interface DoubleComp {
- public int compare(double a, double b);
- public double resetValue();
- }
-
- public static class DoubleDesc implements DoubleComp {
- public double resetValue() {
- return -Double.MAX_VALUE;
- }
-
- public int compare(double a, double b) {
- if(a < b) {
- return -1;
- } else if (a > b) {
- return 1;
- } else {
- return 0;
- }
- }
- }
-
- public static class DoubleAsc implements DoubleComp {
- public double resetValue() {
- return Double.MAX_VALUE;
- }
-
- public int compare(double a, double b) {
- if(a < b) {
- return 1;
- } else if (a > b) {
- return -1;
- } else {
- return 0;
- }
- }
- }
-
- class StringValue implements SortValue {
-
- protected SortedDocValues vals;
-
- protected OrdinalMap ordinalMap;
- protected LongValues globalOrds;
- protected SortedDocValues currentVals;
-
- protected String field;
- protected int segment;
- protected int currentOrd;
- protected IntComp comp;
- protected int lastDocID;
-
- public StringValue(SortedDocValues vals, String field, IntComp comp) {
- this.vals = vals;
- if(vals instanceof MultiDocValues.MultiSortedDocValues) {
- this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) vals).mapping;
- }
- this.field = field;
- this.comp = comp;
- this.currentOrd = comp.resetValue();
- }
-
- public StringValue copy() {
- return new StringValue(vals, field, comp);
- }
-
- public void setCurrentValue(int docId) throws IOException {
-
- if (docId < lastDocID) {
- throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
- }
-
- lastDocID = docId;
-
- if (docId > currentVals.docID()) {
- currentVals.advance(docId);
- }
- if (docId == currentVals.docID()) {
- int ord = currentVals.ordValue();
- if(globalOrds != null) {
- currentOrd = (int)globalOrds.get(ord);
- } else {
- currentOrd = ord;
- }
- } else {
- currentOrd = -1;
- }
- }
-
- public void setCurrentValue(SortValue sv) {
- StringValue v = (StringValue)sv;
- this.currentOrd = v.currentOrd;
- }
-
- public void setNextReader(LeafReaderContext context) throws IOException {
- segment = context.ord;
- if(ordinalMap != null) {
- globalOrds = ordinalMap.getGlobalOrds(segment);
- }
- currentVals = DocValues.getSorted(context.reader(), field);
- lastDocID = 0;
- }
-
- public void reset() {
- this.currentOrd = comp.resetValue();
- }
-
- public int compareTo(SortValue o) {
- StringValue sv = (StringValue)o;
- return comp.compare(currentOrd, sv.currentOrd);
- }
-
- public String toString() {
- return Integer.toString(this.currentOrd);
- }
- }
-
- protected abstract class FieldWriter {
- public abstract boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException;
- }
-
- class IntFieldWriter extends FieldWriter {
- private String field;
-
- public IntFieldWriter(String field) {
- this.field = field;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- int val;
- if (vals.advance(docId) == docId) {
- val = (int) vals.longValue();
- } else {
- return false;
- }
- ew.put(this.field, val);
- return true;
- }
- }
-
- static LongFunction<Object> bitsToValue(FieldType fieldType) {
- switch (fieldType.getNumberType()) {
- case LONG:
- return (bits)-> bits;
- case DATE:
- return (bits)-> new Date(bits);
- case INTEGER:
- return (bits)-> (int)bits;
- case FLOAT:
- return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
- case DOUBLE:
- return (bits)-> NumericUtils.sortableLongToDouble(bits);
- default:
- throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
- }
- }
-
- class MultiFieldWriter extends FieldWriter {
- private String field;
- private FieldType fieldType;
- private SchemaField schemaField;
- private boolean numeric;
- private CharsRefBuilder cref = new CharsRefBuilder();
- private final LongFunction<Object> bitsToValue;
-
- public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
- this.field = field;
- this.fieldType = fieldType;
- this.schemaField = schemaField;
- this.numeric = numeric;
- if (this.fieldType.isPointField()) {
- bitsToValue = bitsToValue(fieldType);
- } else {
- bitsToValue = null;
- }
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
- if (this.fieldType.isPointField()) {
- SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
- if (!vals.advanceExact(docId)) return false;
- out.put(this.field,
- (IteratorWriter) w -> {
- for (int i = 0; i < vals.docValueCount(); i++) {
- w.add(bitsToValue.apply(vals.nextValue()));
- }
- });
- return true;
- } else {
- SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
- if (vals.advance(docId) != docId) return false;
- out.put(this.field,
- (IteratorWriter) w -> {
- long o;
- while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
- BytesRef ref = vals.lookupOrd(o);
- fieldType.indexedToReadable(ref, cref);
- IndexableField f = fieldType.createField(schemaField, cref.toString());
- if (f == null) w.add(cref.toString());
- else w.add(fieldType.toObject(f));
- }
- });
- return true;
- }
-
- }
- }
-
- class LongFieldWriter extends FieldWriter {
- private String field;
-
- public LongFieldWriter(String field) {
- this.field = field;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- long val;
- if (vals.advance(docId) == docId) {
- val = vals.longValue();
- } else {
- return false;
- }
- ew.put(field, val);
- return true;
- }
- }
-
- class DateFieldWriter extends FieldWriter {
- private String field;
-
- public DateFieldWriter(String field) {
- this.field = field;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- long val;
- if (vals.advance(docId) == docId) {
- val = vals.longValue();
- } else {
- return false;
- }
- ew.put(this.field, new Date(val));
- return true;
- }
- }
-
- class BoolFieldWriter extends FieldWriter {
- private String field;
- private FieldType fieldType;
- private CharsRefBuilder cref = new CharsRefBuilder();
-
- public BoolFieldWriter(String field, FieldType fieldType) {
- this.field = field;
- this.fieldType = fieldType;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- SortedDocValues vals = DocValues.getSorted(reader, this.field);
- if (vals.advance(docId) != docId) {
- return false;
- }
- int ord = vals.ordValue();
-
- BytesRef ref = vals.lookupOrd(ord);
- fieldType.indexedToReadable(ref, cref);
- ew.put(this.field, "true".equals(cref.toString()));
- return true;
- }
- }
-
- class FloatFieldWriter extends FieldWriter {
- private String field;
-
- public FloatFieldWriter(String field) {
- this.field = field;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- int val;
- if (vals.advance(docId) == docId) {
- val = (int)vals.longValue();
- } else {
- return false;
- }
- ew.put(this.field, Float.intBitsToFloat(val));
- return true;
- }
- }
-
- class DoubleFieldWriter extends FieldWriter {
- private String field;
-
- public DoubleFieldWriter(String field) {
- this.field = field;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- NumericDocValues vals = DocValues.getNumeric(reader, this.field);
- long val;
- if (vals.advance(docId) == docId) {
- val = vals.longValue();
- } else {
- return false;
- }
- ew.put(this.field, Double.longBitsToDouble(val));
- return true;
- }
- }
-
- class StringFieldWriter extends FieldWriter {
- private String field;
- private FieldType fieldType;
- private CharsRefBuilder cref = new CharsRefBuilder();
-
- public StringFieldWriter(String field, FieldType fieldType) {
- this.field = field;
- this.fieldType = fieldType;
- }
-
- public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
- SortedDocValues vals = DocValues.getSorted(reader, this.field);
- if (vals.advance(docId) != docId) {
- return false;
- }
- int ord = vals.ordValue();
-
- BytesRef ref = vals.lookupOrd(ord);
- fieldType.indexedToReadable(ref, cref);
- ew.put(this.field, cref.toString());
- return true;
- }
- }
-
- public abstract class PriorityQueue<T> {
- protected int size = 0;
- protected final int maxSize;
- private final T[] heap;
-
- public PriorityQueue(int maxSize) {
- this(maxSize, true);
- }
-
- public PriorityQueue(int maxSize, boolean prepopulate) {
- final int heapSize;
- if (0 == maxSize) {
- // We allocate 1 extra to avoid if statement in top()
- heapSize = 2;
- } else {
- if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
- // Don't wrap heapSize to -1, in this case, which
- // causes a confusing NegativeArraySizeException.
- // Note that very likely this will simply then hit
- // an OOME, but at least that's more indicative to
- // caller that this values is too big. We don't +1
- // in this case, but it's very unlikely in practice
- // one will actually insert this many objects into
- // the PQ:
- // Throw exception to prevent confusing OOME:
- throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
- } else {
- // NOTE: we add +1 because all access to heap is
- // 1-based not 0-based. heap[0] is unused.
- heapSize = maxSize + 1;
- }
- }
- // T is unbounded type, so this unchecked cast works always:
- @SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
- this.heap = h;
- this.maxSize = maxSize;
-
- if (prepopulate) {
- // If sentinel objects are supported, populate the queue with them
- T sentinel = getSentinelObject();
- if (sentinel != null) {
- heap[1] = sentinel;
- for (int i = 2; i < heap.length; i++) {
- heap[i] = getSentinelObject();
- }
- size = maxSize;
- }
- }
- }
-
- /** Determines the ordering of objects in this priority queue. Subclasses
- * must define this one method.
- * @return <code>true</code> iff parameter <tt>a</tt> is less than parameter <tt>b</tt>.
- */
- protected abstract boolean lessThan(T a, T b);
-
-
- protected T getSentinelObject() {
- return null;
- }
-
- /**
- * Adds an Object to a PriorityQueue in log(size) time. If one tries to add
- * more objects than maxSize from initialize an
- *
- * @return the new 'top' element in the queue.
- */
- public final T add(T element) {
- size++;
- heap[size] = element;
- upHeap();
- return heap[1];
- }
-
- /**
- * Adds an Object to a PriorityQueue in log(size) time.
- * It returns the object (if any) that was
- * dropped off the heap because it was full. This can be
- * the given parameter (in case it is smaller than the
- * full heap's minimum, and couldn't be added), or another
- * object that was previously the smallest value in the
- * heap and now has been replaced by a larger one, or null
- * if the queue wasn't yet full with maxSize elements.
- */
- public T insertWithOverflow(T element) {
- if (size < maxSize) {
- add(element);
- return null;
- } else if (size > 0 && !lessThan(element, heap[1])) {
- T ret = heap[1];
- heap[1] = element;
- updateTop();
- return ret;
- } else {
- return element;
- }
- }
-
- /** Returns the least element of the PriorityQueue in constant time. */
- public final T top() {
- // We don't need to check size here: if maxSize is 0,
- // then heap is length 2 array with both entries null.
- // If size is 0 then heap[1] is already null.
- return heap[1];
- }
-
- /** Removes and returns the least element of the PriorityQueue in log(size)
- time. */
- public final T pop() {
- if (size > 0) {
- T result = heap[1]; // save first value
- heap[1] = heap[size]; // move last to first
- heap[size] = null; // permit GC of objects
- size--;
- downHeap(); // adjust heap
- return result;
- } else {
- return null;
- }
- }
-
- /**
- * Should be called when the Object at top changes values. Still log(n) worst
- * case, but it's at least twice as fast to
- *
- * <pre class="prettyprint">
- * pq.top().change();
- * pq.updateTop();
- * </pre>
- *
- * instead of
- *
- * <pre class="prettyprint">
- * o = pq.pop();
- * o.change();
- * pq.push(o);
- * </pre>
- *
- * @return the new 'top' element.
- */
- public final T updateTop() {
- downHeap();
- return heap[1];
- }
-
- /** Returns the number of elements currently stored in the PriorityQueue. */
- public final int size() {
- return size;
- }
-
- /** Removes all entries from the PriorityQueue. */
- public final void clear() {
- for (int i = 0; i <= size; i++) {
- heap[i] = null;
- }
- size = 0;
- }
-
- private final void upHeap() {
- int i = size;
- T node = heap[i]; // save bottom node
- int j = i >>> 1;
- while (j > 0 && lessThan(node, heap[j])) {
- heap[i] = heap[j]; // shift parents down
- i = j;
- j = j >>> 1;
- }
- heap[i] = node; // install saved node
- }
-
- private final void downHeap() {
- int i = 1;
- T node = heap[i]; // save top node
- int j = i << 1; // find smaller child
- int k = j + 1;
- if (k <= size && lessThan(heap[k], heap[j])) {
- j = k;
- }
- while (j <= size && lessThan(heap[j], node)) {
- heap[i] = heap[j]; // shift up child
- i = j;
- j = i << 1;
- k = j + 1;
- if (k <= size && lessThan(heap[k], heap[j])) {
- j = k;
- }
- }
- heap[i] = node; // install saved node
- }
-
- /** This method returns the internal heap array as Object[].
- * @lucene.internal
- */
- public final Object[] getHeapArray() {
- return (Object[]) heap;
- }
- }
-
- public static class IgnoreException extends IOException {
- public void printStackTrace(PrintWriter pw) {
- pw.print("Early Client Disconnect");
- }
-
- public String getMessage() {
- return "Early Client Disconnect";
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
new file mode 100644
index 0000000..e67f5ed
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/BoolFieldWriter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.schema.FieldType;
+
+class BoolFieldWriter extends FieldWriter {
+ private String field;
+ private FieldType fieldType;
+ private CharsRefBuilder cref = new CharsRefBuilder();
+
+ public BoolFieldWriter(String field, FieldType fieldType) {
+ this.field = field;
+ this.fieldType = fieldType;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ SortedDocValues vals = DocValues.getSorted(reader, this.field);
+ if (vals.advance(docId) != docId) {
+ return false;
+ }
+ int ord = vals.ordValue();
+
+ BytesRef ref = vals.lookupOrd(ord);
+ fieldType.indexedToReadable(ref, cref);
+ ew.put(this.field, "true".equals(cref.toString()));
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
new file mode 100644
index 0000000..3584317
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DateFieldWriter.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+import java.util.Date;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class DateFieldWriter extends FieldWriter {
+ private String field;
+
+ public DateFieldWriter(String field) {
+ this.field = field;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ return false;
+ }
+ ew.put(this.field, new Date(val));
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java b/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
new file mode 100644
index 0000000..a37e13d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubeValue.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+
+class DoubleValue implements SortValue {
+
+ protected NumericDocValues vals;
+ protected String field;
+ protected double currentValue;
+ protected DoubleComp comp;
+ private int lastDocID;
+ private LeafReader reader;
+
+ public DoubleValue(String field, DoubleComp comp) {
+ this.field = field;
+ this.comp = comp;
+ this.currentValue = comp.resetValue();
+ }
+
+ public DoubleValue copy() {
+ return new DoubleValue(field, comp);
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.reader = context.reader();
+ this.vals = DocValues.getNumeric(this.reader, this.field);
+ lastDocID = 0;
+ }
+
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = Double.longBitsToDouble(vals.longValue());
+ } else {
+ currentValue = 0f;
+ }
+ }
+
+ public void setCurrentValue(SortValue sv) {
+ DoubleValue dv = (DoubleValue)sv;
+ this.currentValue = dv.currentValue;
+ }
+
+ public void reset() {
+ this.currentValue = comp.resetValue();
+ }
+
+ public int compareTo(SortValue o) {
+ DoubleValue dv = (DoubleValue)o;
+ return comp.compare(currentValue, dv.currentValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
new file mode 100644
index 0000000..50341fd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleCmp.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+interface DoubleComp {
+ int compare(double a, double b);
+ double resetValue();
+}
+
+class DoubleAsc implements DoubleComp {
+ public double resetValue() {
+ return Double.MAX_VALUE;
+ }
+
+ public int compare(double a, double b) {
+ return Double.compare(b, a);
+ }
+}
+
+class DoubleDesc implements DoubleComp {
+ public double resetValue() {
+ return -Double.MAX_VALUE;
+ }
+
+ public int compare(double a, double b) {
+ return Double.compare(a, b);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
new file mode 100644
index 0000000..1b953c5
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleFieldWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class DoubleFieldWriter extends FieldWriter {
+ private String field;
+
+ public DoubleFieldWriter(String field) {
+ this.field = field;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ return false;
+ }
+ ew.put(this.field, Double.longBitsToDouble(val));
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
new file mode 100644
index 0000000..78d0795
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -0,0 +1,448 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.solr.client.solrj.impl.BinaryResponseParser;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.MapWriter.EntryWriter;
+import org.apache.solr.common.PushWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrRequestInfo;
+import org.apache.solr.response.BinaryResponseWriter;
+import org.apache.solr.response.JSONResponseWriter;
+import org.apache.solr.response.QueryResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.DateValueFieldType;
+import org.apache.solr.schema.DoubleValueFieldType;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.FloatValueFieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.IntValueFieldType;
+import org.apache.solr.schema.LongValueFieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.StrField;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.SortSpec;
+import org.apache.solr.search.SyntaxError;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.Collections.singletonList;
+import static java.util.Collections.singletonMap;
+import static org.apache.solr.common.util.Utils.makeMap;
+
+public class ExportWriter implements SolrCore.RawWriter, Closeable {
+ private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private OutputStreamWriter respWriter;
+ final SolrQueryRequest req;
+ final SolrQueryResponse res;
+ FieldWriter[] fieldWriters;
+ int totalHits = 0;
+ FixedBitSet[] sets = null;
+ PushWriter writer;
+ private String wt;
+
+
+ public ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
+ this.req = req;
+ this.res = res;
+ this.wt = wt;
+
+ }
+
+ @Override
+ public String getContentType() {
+ if ("javabin".equals(wt)) {
+ return BinaryResponseParser.BINARY_CONTENT_TYPE;
+ } else return "json";
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (writer != null) writer.close();
+ if (respWriter != null) {
+ respWriter.flush();
+ respWriter.close();
+ }
+
+ }
+
+ protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
+ w.writeMap(mw -> {
+ mw.put("responseHeader", singletonMap("status", 400))
+ .put("response", makeMap(
+ "numFound", 0,
+ "docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
+ });
+ if (log) {
+ SolrException.log(logger, e);
+ }
+ }
+
+ public void write(OutputStream os) throws IOException {
+ QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
+ if (rw instanceof BinaryResponseWriter) {
+ //todo add support for other writers after testing
+ writer = new JavaBinCodec(os, null);
+ } else {
+ respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
+ writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
+ }
+ Exception exception = res.getException();
+ if (exception != null) {
+ if (!(exception instanceof IgnoreException)) {
+ writeException(exception, writer, false);
+ }
+ return;
+ }
+
+ SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
+ SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
+
+ if (sortSpec == null) {
+ writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
+ return;
+ }
+
+ SolrIndexSearcher searcher = req.getSearcher();
+ Sort sort = searcher.weightSort(sortSpec.getSort());
+
+ if (sort == null) {
+ writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
+ return;
+ }
+
+ if (sort != null && sort.needsScores()) {
+ writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
+ return;
+ }
+
+ // There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
+ // if (lastDocRequested <= 0) {
+ // That causes the totalHits and export entries in the context to _not_ get set.
+ // The only time that really matters is when we search against an _empty_ set. That's too obscure
+ // a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
+ // ant test -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
+ // You'll have to uncomment the if below to hit the null pointer exception.
+ // This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
+ // This came to light in the very artifical case of indexing a single doc to Cloud.
+ if (req.getContext().get("totalHits") != null) {
+ totalHits = ((Integer) req.getContext().get("totalHits")).intValue();
+ sets = (FixedBitSet[]) req.getContext().get("export");
+ if (sets == null) {
+ writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
+ return;
+ }
+ }
+ SolrParams params = req.getParams();
+ String fl = params.get("fl");
+
+ String[] fields = null;
+
+ if (fl == null) {
+ writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
+ return;
+ } else {
+ fields = fl.split(",");
+
+ for (int i = 0; i < fields.length; i++) {
+
+ fields[i] = fields[i].trim();
+
+ if (fields[i].equals("score")) {
+ writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
+ return;
+ }
+ }
+ }
+
+ try {
+ fieldWriters = getFieldWriters(fields, req.getSearcher());
+ } catch (Exception e) {
+ writeException(e, writer, true);
+ return;
+ }
+
+ writer.writeMap(m -> {
+ m.put("responseHeader", singletonMap("status", 0));
+ m.put("response", (MapWriter) mw -> {
+ mw.put("numFound", totalHits);
+ mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
+ });
+ });
+
+ }
+
+ protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
+ //Write the data.
+ List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
+ SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
+ int count = 0;
+ int queueSize = 30000;
+ if (totalHits < 30000) {
+ queueSize = totalHits;
+ }
+ SortQueue queue = new SortQueue(queueSize, sortDoc);
+ SortDoc[] outDocs = new SortDoc[queueSize];
+
+ while (count < totalHits) {
+ //long begin = System.nanoTime();
+ queue.reset();
+ SortDoc top = queue.top();
+ for (int i = 0; i < leaves.size(); i++) {
+ sortDoc.setNextReader(leaves.get(i));
+ DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
+ int docId;
+ while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ sortDoc.setValues(docId);
+ if (top.lessThan(sortDoc)) {
+ top.setValues(sortDoc);
+ top = queue.updateTop();
+ }
+ }
+ }
+
+ int outDocsIndex = -1;
+
+ for (int i = 0; i < queueSize; i++) {
+ SortDoc s = queue.pop();
+ if (s.docId > -1) {
+ outDocs[++outDocsIndex] = s;
+ }
+ }
+
+ //long end = System.nanoTime();
+
+ count += (outDocsIndex + 1);
+
+ try {
+ for (int i = outDocsIndex; i >= 0; --i) {
+ SortDoc s = outDocs[i];
+ writer.add((MapWriter) ew -> {
+ writeDoc(s, leaves, ew);
+ s.reset();
+ });
+ }
+ } catch (Throwable e) {
+ Throwable ex = e;
+ while (ex != null) {
+ String m = ex.getMessage();
+ if (m != null && m.contains("Broken pipe")) {
+ throw new IgnoreException();
+ }
+ ex = ex.getCause();
+ }
+
+ if (e instanceof IOException) {
+ throw ((IOException) e);
+ } else {
+ throw new IOException(e);
+ }
+ }
+ }
+ }
+
+ protected void writeDoc(SortDoc sortDoc,
+ List<LeafReaderContext> leaves,
+ EntryWriter ew) throws IOException {
+
+ int ord = sortDoc.ord;
+ FixedBitSet set = sets[ord];
+ set.clear(sortDoc.docId);
+ LeafReaderContext context = leaves.get(ord);
+ int fieldIndex = 0;
+ for (FieldWriter fieldWriter : fieldWriters) {
+ if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
+ ++fieldIndex;
+ }
+ }
+ }
+
+ protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
+ IndexSchema schema = searcher.getSchema();
+ FieldWriter[] writers = new FieldWriter[fields.length];
+ for (int i = 0; i < fields.length; i++) {
+ String field = fields[i];
+ SchemaField schemaField = null;
+
+ try {
+ schemaField = schema.getField(field);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+
+ if (!schemaField.hasDocValues()) {
+ throw new IOException(field + " must have DocValues to use this feature.");
+ }
+
+ boolean multiValued = schemaField.multiValued();
+ FieldType fieldType = schemaField.getType();
+ if (fieldType instanceof IntValueFieldType) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+ } else {
+ writers[i] = new IntFieldWriter(field);
+ }
+ } else if (fieldType instanceof LongValueFieldType) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+ } else {
+ writers[i] = new LongFieldWriter(field);
+ }
+ } else if (fieldType instanceof FloatValueFieldType) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+ } else {
+ writers[i] = new FloatFieldWriter(field);
+ }
+ } else if (fieldType instanceof DoubleValueFieldType) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+ } else {
+ writers[i] = new DoubleFieldWriter(field);
+ }
+ } else if (fieldType instanceof StrField) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
+ } else {
+ writers[i] = new StringFieldWriter(field, fieldType);
+ }
+ } else if (fieldType instanceof DateValueFieldType) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
+ } else {
+ writers[i] = new DateFieldWriter(field);
+ }
+ } else if (fieldType instanceof BoolField) {
+ if (multiValued) {
+ writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
+ } else {
+ writers[i] = new BoolFieldWriter(field, fieldType);
+ }
+ } else {
+ throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
+ }
+ }
+ return writers;
+ }
+
+ private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
+ SortValue[] sortValues = new SortValue[sortFields.length];
+ IndexSchema schema = searcher.getSchema();
+ for (int i = 0; i < sortFields.length; ++i) {
+ SortField sf = sortFields[i];
+ String field = sf.getField();
+ boolean reverse = sf.getReverse();
+ SchemaField schemaField = schema.getField(field);
+ FieldType ft = schemaField.getType();
+
+ if (!schemaField.hasDocValues()) {
+ throw new IOException(field + " must have DocValues to use this feature.");
+ }
+
+ if (ft instanceof IntValueFieldType) {
+ if (reverse) {
+ sortValues[i] = new IntValue(field, new IntDesc());
+ } else {
+ sortValues[i] = new IntValue(field, new IntAsc());
+ }
+ } else if (ft instanceof FloatValueFieldType) {
+ if (reverse) {
+ sortValues[i] = new FloatValue(field, new FloatDesc());
+ } else {
+ sortValues[i] = new FloatValue(field, new FloatAsc());
+ }
+ } else if (ft instanceof DoubleValueFieldType) {
+ if (reverse) {
+ sortValues[i] = new DoubleValue(field, new DoubleDesc());
+ } else {
+ sortValues[i] = new DoubleValue(field, new DoubleAsc());
+ }
+ } else if (ft instanceof LongValueFieldType) {
+ if (reverse) {
+ sortValues[i] = new LongValue(field, new LongDesc());
+ } else {
+ sortValues[i] = new LongValue(field, new LongAsc());
+ }
+ } else if (ft instanceof StrField) {
+ LeafReader reader = searcher.getSlowAtomicReader();
+ SortedDocValues vals = reader.getSortedDocValues(field);
+ if (reverse) {
+ sortValues[i] = new StringValue(vals, field, new IntDesc());
+ } else {
+ sortValues[i] = new StringValue(vals, field, new IntAsc());
+ }
+ } else if (ft instanceof DateValueFieldType) {
+ if (reverse) {
+ sortValues[i] = new LongValue(field, new LongDesc());
+ } else {
+ sortValues[i] = new LongValue(field, new LongAsc());
+ }
+ } else if (ft instanceof BoolField) {
+ // This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
+ // _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
+ // we can just use the existing StringValue here.
+ LeafReader reader = searcher.getSlowAtomicReader();
+ SortedDocValues vals = reader.getSortedDocValues(field);
+ if (reverse) {
+ sortValues[i] = new StringValue(vals, field, new IntDesc());
+ } else {
+ sortValues[i] = new StringValue(vals, field, new IntAsc());
+ }
+ } else {
+ throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
+ }
+ }
+
+ return new SortDoc(sortValues);
+ }
+
+ public static class IgnoreException extends IOException {
+ public void printStackTrace(PrintWriter pw) {
+ pw.print("Early Client Disconnect");
+ }
+
+ public String getMessage() {
+ return "Early Client Disconnect";
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
new file mode 100644
index 0000000..bd69fc0
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/FieldWriter.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReader;
+import org.apache.solr.common.MapWriter;
+
+abstract class FieldWriter {
+ public abstract boolean write(int docId, LeafReader reader, MapWriter.EntryWriter out, int fieldIndex) throws IOException;
+}
\ No newline at end of file
[2/3] lucene-solr:master: SOLR-11598: Support more than 4 sort fields
in the export writer
Posted by va...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/FloatCmp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/FloatCmp.java b/solr/core/src/java/org/apache/solr/handler/export/FloatCmp.java
new file mode 100644
index 0000000..7ef078c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/FloatCmp.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+interface FloatComp {
+ int compare(float a, float b);
+ float resetValue();
+}
+
+class FloatAsc implements FloatComp {
+ public float resetValue() {
+ return Float.MAX_VALUE;
+ }
+
+ public int compare(float a, float b) {
+ return Float.compare(b, a);
+ }
+}
+
+class FloatDesc implements FloatComp {
+ public float resetValue() {
+ return -Float.MAX_VALUE;
+ }
+
+ public int compare(float a, float b) {
+ return Float.compare(a, b);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java
new file mode 100644
index 0000000..cfb2e25
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/FloatFieldWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class FloatFieldWriter extends FieldWriter {
+ private String field;
+
+ public FloatFieldWriter(String field) {
+ this.field = field;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ int val;
+ if (vals.advance(docId) == docId) {
+ val = (int)vals.longValue();
+ } else {
+ return false;
+ }
+ ew.put(this.field, Float.intBitsToFloat(val));
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/FloatValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/FloatValue.java b/solr/core/src/java/org/apache/solr/handler/export/FloatValue.java
new file mode 100644
index 0000000..f6a5bbd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/FloatValue.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+
+class FloatValue implements SortValue {
+
+ protected NumericDocValues vals;
+ protected String field;
+ protected float currentValue;
+ protected FloatComp comp;
+ private int lastDocID;
+
+ public FloatValue(String field, FloatComp comp) {
+ this.field = field;
+ this.comp = comp;
+ this.currentValue = comp.resetValue();
+ }
+
+ public FloatValue copy() {
+ return new FloatValue(field, comp);
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.vals = DocValues.getNumeric(context.reader(), field);
+ lastDocID = 0;
+ }
+
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = Float.intBitsToFloat((int)vals.longValue());
+ } else {
+ currentValue = 0f;
+ }
+ }
+
+ public void setCurrentValue(SortValue sv) {
+ FloatValue fv = (FloatValue)sv;
+ this.currentValue = fv.currentValue;
+ }
+
+ public void reset() {
+ this.currentValue = comp.resetValue();
+ }
+
+ public int compareTo(SortValue o) {
+ FloatValue fv = (FloatValue)o;
+ return comp.compare(currentValue, fv.currentValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/IntComp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/IntComp.java b/solr/core/src/java/org/apache/solr/handler/export/IntComp.java
new file mode 100644
index 0000000..ac83d5d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/IntComp.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+public interface IntComp {
+ int compare(int a, int b);
+ int resetValue();
+}
+
+class IntAsc implements IntComp {
+
+ public int resetValue() {
+ return Integer.MAX_VALUE;
+ }
+
+ public int compare(int a, int b) {
+ return Integer.compare(b, a);
+ }
+}
+
+class IntDesc implements IntComp {
+
+ public int resetValue() {
+ return Integer.MIN_VALUE;
+ }
+
+ public int compare(int a, int b) {
+ return Integer.compare(a, b);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java
new file mode 100644
index 0000000..66e5936
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/IntFieldWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class IntFieldWriter extends FieldWriter {
+ private String field;
+
+ public IntFieldWriter(String field) {
+ this.field = field;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ int val;
+ if (vals.advance(docId) == docId) {
+ val = (int) vals.longValue();
+ } else {
+ return false;
+ }
+ ew.put(this.field, val);
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/IntValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/IntValue.java b/solr/core/src/java/org/apache/solr/handler/export/IntValue.java
new file mode 100644
index 0000000..d222aa3
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/IntValue.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+
+public class IntValue implements SortValue {
+
+ protected NumericDocValues vals;
+ protected String field;
+ protected int currentValue;
+ protected IntComp comp;
+ private int lastDocID;
+
+ public IntValue copy() {
+ return new IntValue(field, comp);
+ }
+
+ public IntValue(String field, IntComp comp) {
+ this.field = field;
+ this.comp = comp;
+ this.currentValue = comp.resetValue();
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.vals = DocValues.getNumeric(context.reader(), field);
+ lastDocID = 0;
+ }
+
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = (int) vals.longValue();
+ } else {
+ currentValue = 0;
+ }
+ }
+
+ public int compareTo(SortValue o) {
+ IntValue iv = (IntValue)o;
+ return comp.compare(currentValue, iv.currentValue);
+ }
+
+ public void setCurrentValue (SortValue value) {
+ currentValue = ((IntValue)value).currentValue;
+ }
+
+ public void reset() {
+ currentValue = comp.resetValue();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/LongCmp.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/LongCmp.java b/solr/core/src/java/org/apache/solr/handler/export/LongCmp.java
new file mode 100644
index 0000000..7d997ac
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/LongCmp.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+interface LongComp {
+ int compare(long a, long b);
+ long resetValue();
+}
+
+class LongAsc implements LongComp {
+
+ public long resetValue() {
+ return Long.MAX_VALUE;
+ }
+
+ public int compare(long a, long b) {
+ return Long.compare(b, a);
+ }
+}
+
+class LongDesc implements LongComp {
+
+ public long resetValue() {
+ return Long.MIN_VALUE;
+ }
+
+ public int compare(long a, long b) {
+ return Long.compare(a, b);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java
new file mode 100644
index 0000000..87f2a32
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/LongFieldWriter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.solr.common.MapWriter;
+
+class LongFieldWriter extends FieldWriter {
+ private String field;
+
+ public LongFieldWriter(String field) {
+ this.field = field;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ NumericDocValues vals = DocValues.getNumeric(reader, this.field);
+ long val;
+ if (vals.advance(docId) == docId) {
+ val = vals.longValue();
+ } else {
+ return false;
+ }
+ ew.put(field, val);
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/LongValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/LongValue.java b/solr/core/src/java/org/apache/solr/handler/export/LongValue.java
new file mode 100644
index 0000000..28b9a07
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/LongValue.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+
+public class LongValue implements SortValue {
+
+ protected NumericDocValues vals;
+ protected String field;
+ protected long currentValue;
+ protected LongComp comp;
+ private int lastDocID;
+
+ public LongValue(String field, LongComp comp) {
+ this.field = field;
+ this.comp = comp;
+ this.currentValue = comp.resetValue();
+ }
+
+ public LongValue copy() {
+ return new LongValue(field, comp);
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.vals = DocValues.getNumeric(context.reader(), field);
+ lastDocID = 0;
+ }
+
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+ lastDocID = docId;
+ int curDocID = vals.docID();
+ if (docId > curDocID) {
+ curDocID = vals.advance(docId);
+ }
+ if (docId == curDocID) {
+ currentValue = vals.longValue();
+ } else {
+ currentValue = 0;
+ }
+ }
+
+ public void setCurrentValue(SortValue sv) {
+ LongValue lv = (LongValue)sv;
+ this.currentValue = lv.currentValue;
+ }
+
+ public int compareTo(SortValue o) {
+ LongValue l = (LongValue)o;
+ return comp.compare(currentValue, l.currentValue);
+ }
+
+ public void reset() {
+ this.currentValue = comp.resetValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java
new file mode 100644
index 0000000..2e43207
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/MultiFieldWriter.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.function.LongFunction;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.solr.common.IteratorWriter;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+
+class MultiFieldWriter extends FieldWriter {
+ private String field;
+ private FieldType fieldType;
+ private SchemaField schemaField;
+ private boolean numeric;
+ private CharsRefBuilder cref = new CharsRefBuilder();
+ private final LongFunction<Object> bitsToValue;
+
+ public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
+ this.field = field;
+ this.fieldType = fieldType;
+ this.schemaField = schemaField;
+ this.numeric = numeric;
+ if (this.fieldType.isPointField()) {
+ bitsToValue = bitsToValue(fieldType);
+ } else {
+ bitsToValue = null;
+ }
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter out, int fieldIndex) throws IOException {
+ if (this.fieldType.isPointField()) {
+ SortedNumericDocValues vals = DocValues.getSortedNumeric(reader, this.field);
+ if (!vals.advanceExact(docId)) return false;
+ out.put(this.field,
+ (IteratorWriter) w -> {
+ for (int i = 0; i < vals.docValueCount(); i++) {
+ w.add(bitsToValue.apply(vals.nextValue()));
+ }
+ });
+ return true;
+ } else {
+ SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
+ if (vals.advance(docId) != docId) return false;
+ out.put(this.field,
+ (IteratorWriter) w -> {
+ long o;
+ while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
+ BytesRef ref = vals.lookupOrd(o);
+ fieldType.indexedToReadable(ref, cref);
+ IndexableField f = fieldType.createField(schemaField, cref.toString());
+ if (f == null) w.add(cref.toString());
+ else w.add(fieldType.toObject(f));
+ }
+ });
+ return true;
+ }
+
+ }
+
+
+ static LongFunction<Object> bitsToValue(FieldType fieldType) {
+ switch (fieldType.getNumberType()) {
+ case LONG:
+ return (bits)-> bits;
+ case DATE:
+ return (bits)-> new Date(bits);
+ case INTEGER:
+ return (bits)-> (int)bits;
+ case FLOAT:
+ return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
+ case DOUBLE:
+ return (bits)-> NumericUtils.sortableLongToDouble(bits);
+ default:
+ throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/PriorityQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/PriorityQueue.java b/solr/core/src/java/org/apache/solr/handler/export/PriorityQueue.java
new file mode 100644
index 0000000..1552060
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/PriorityQueue.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import org.apache.lucene.util.ArrayUtil;
+
+public abstract class PriorityQueue<T> {
+ protected int size = 0;
+ protected final int maxSize;
+ private final T[] heap;
+
+ public PriorityQueue(int maxSize) {
+ this(maxSize, true);
+ }
+
+ public PriorityQueue(int maxSize, boolean prepopulate) {
+ final int heapSize;
+ if (0 == maxSize) {
+ // We allocate 1 extra to avoid if statement in top()
+ heapSize = 2;
+ } else {
+ if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
+ // Don't wrap heapSize to -1, in this case, which
+ // causes a confusing NegativeArraySizeException.
+ // Note that very likely this will simply then hit
+ // an OOME, but at least that's more indicative to
+ // caller that this values is too big. We don't +1
+ // in this case, but it's very unlikely in practice
+ // one will actually insert this many objects into
+ // the PQ:
+ // Throw exception to prevent confusing OOME:
+ throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
+ } else {
+ // NOTE: we add +1 because all access to heap is
+ // 1-based not 0-based. heap[0] is unused.
+ heapSize = maxSize + 1;
+ }
+ }
+ // T is unbounded type, so this unchecked cast works always:
+ @SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
+ this.heap = h;
+ this.maxSize = maxSize;
+
+ if (prepopulate) {
+ // If sentinel objects are supported, populate the queue with them
+ T sentinel = getSentinelObject();
+ if (sentinel != null) {
+ heap[1] = sentinel;
+ for (int i = 2; i < heap.length; i++) {
+ heap[i] = getSentinelObject();
+ }
+ size = maxSize;
+ }
+ }
+ }
+
+ /** Determines the ordering of objects in this priority queue. Subclasses
+ * must define this one method.
+ * @return <code>true</code> iff parameter <tt>a</tt> is less than parameter <tt>b</tt>.
+ */
+ protected abstract boolean lessThan(T a, T b);
+
+
+ protected T getSentinelObject() {
+ return null;
+ }
+
+ /**
+ * Adds an Object to a PriorityQueue in log(size) time. If one tries to add
+ * more objects than maxSize from initialize an
+ *
+ * @return the new 'top' element in the queue.
+ */
+ public final T add(T element) {
+ size++;
+ heap[size] = element;
+ upHeap();
+ return heap[1];
+ }
+
+ /**
+ * Adds an Object to a PriorityQueue in log(size) time.
+ * It returns the object (if any) that was
+ * dropped off the heap because it was full. This can be
+ * the given parameter (in case it is smaller than the
+ * full heap's minimum, and couldn't be added), or another
+ * object that was previously the smallest value in the
+ * heap and now has been replaced by a larger one, or null
+ * if the queue wasn't yet full with maxSize elements.
+ */
+ public T insertWithOverflow(T element) {
+ if (size < maxSize) {
+ add(element);
+ return null;
+ } else if (size > 0 && !lessThan(element, heap[1])) {
+ T ret = heap[1];
+ heap[1] = element;
+ updateTop();
+ return ret;
+ } else {
+ return element;
+ }
+ }
+
+ /** Returns the least element of the PriorityQueue in constant time. */
+ public final T top() {
+ // We don't need to check size here: if maxSize is 0,
+ // then heap is length 2 array with both entries null.
+ // If size is 0 then heap[1] is already null.
+ return heap[1];
+ }
+
+ /** Removes and returns the least element of the PriorityQueue in log(size)
+ time. */
+ public final T pop() {
+ if (size > 0) {
+ T result = heap[1]; // save first value
+ heap[1] = heap[size]; // move last to first
+ heap[size] = null; // permit GC of objects
+ size--;
+ downHeap(); // adjust heap
+ return result;
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Should be called when the Object at top changes values. Still log(n) worst
+ * case, but it's at least twice as fast to
+ *
+ * <pre class="prettyprint">
+ * pq.top().change();
+ * pq.updateTop();
+ * </pre>
+ *
+ * instead of
+ *
+ * <pre class="prettyprint">
+ * o = pq.pop();
+ * o.change();
+ * pq.push(o);
+ * </pre>
+ *
+ * @return the new 'top' element.
+ */
+ public final T updateTop() {
+ downHeap();
+ return heap[1];
+ }
+
+ /** Returns the number of elements currently stored in the PriorityQueue. */
+ public final int size() {
+ return size;
+ }
+
+ /** Removes all entries from the PriorityQueue. */
+ public final void clear() {
+ for (int i = 0; i <= size; i++) {
+ heap[i] = null;
+ }
+ size = 0;
+ }
+
+ private final void upHeap() {
+ int i = size;
+ T node = heap[i]; // save bottom node
+ int j = i >>> 1;
+ while (j > 0 && lessThan(node, heap[j])) {
+ heap[i] = heap[j]; // shift parents down
+ i = j;
+ j = j >>> 1;
+ }
+ heap[i] = node; // install saved node
+ }
+
+ private final void downHeap() {
+ int i = 1;
+ T node = heap[i]; // save top node
+ int j = i << 1; // find smaller child
+ int k = j + 1;
+ if (k <= size && lessThan(heap[k], heap[j])) {
+ j = k;
+ }
+ while (j <= size && lessThan(heap[j], node)) {
+ heap[i] = heap[j]; // shift up child
+ i = j;
+ j = i << 1;
+ k = j + 1;
+ if (k <= size && lessThan(heap[k], heap[j])) {
+ j = k;
+ }
+ }
+ heap[i] = node; // install saved node
+ }
+
+ /** This method returns the internal heap array as Object[].
+ * @lucene.internal
+ */
+ public final Object[] getHeapArray() {
+ return (Object[]) heap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
new file mode 100644
index 0000000..3cf478b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class SortDoc {
+
+ protected int docId = -1;
+ protected int ord = -1;
+ protected int docBase = -1;
+
+ private SortValue[] sortValues;
+
+ public SortDoc(SortValue[] sortValues) {
+ this.sortValues = sortValues;
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ for (SortValue value : sortValues) {
+ value.setNextReader(context);
+ }
+ }
+
+ public void reset() {
+ this.docId = -1;
+ for (SortValue value : sortValues) {
+ value.reset();
+ }
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ for(SortValue sortValue : sortValues) {
+ sortValue.setCurrentValue(docId);
+ }
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ SortValue[] vals = sortDoc.sortValues;
+ for(int i=0; i<vals.length; i++) {
+ sortValues[i].setCurrentValue(vals[i]);
+ }
+ }
+
+ public SortDoc copy() {
+ SortValue[] svs = new SortValue[sortValues.length];
+ for(int i=0; i<sortValues.length; i++) {
+ svs[i] = sortValues[i].copy();
+ }
+
+ return new SortDoc(svs);
+ }
+
+ public boolean lessThan(Object o) {
+ if(docId == -1) {
+ return true;
+ }
+ SortDoc sd = (SortDoc)o;
+ SortValue[] sortValues1 = sd.sortValues;
+ for(int i=0; i<sortValues.length; i++) {
+ int comp = sortValues[i].compareTo(sortValues1[i]);
+ if(comp < 0) {
+ return true;
+ } if(comp > 0) {
+ return false;
+ }
+ }
+ return docId+docBase > sd.docId+sd.docBase; //index order
+ }
+
+ public int compareTo(Object o) {
+ SortDoc sd = (SortDoc)o;
+ for (int i=0; i<sortValues.length; i++) {
+ int comp = sortValues[i].compareTo(sd.sortValues[i]);
+ if (comp != 0) {
+ return comp;
+ }
+ }
+ return 0;
+ }
+
+
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("docId: " + docId + "; ");
+ for (int i=0; i < sortValues.length; i++) {
+ builder.append("value" + i + ": " + sortValues[i] + ", ");
+ }
+ return builder.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/SortQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortQueue.java b/solr/core/src/java/org/apache/solr/handler/export/SortQueue.java
new file mode 100644
index 0000000..cf99d53
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortQueue.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+class SortQueue extends PriorityQueue<SortDoc> {
+
+ private SortDoc proto;
+ private Object[] cache;
+
+ public SortQueue(int len, SortDoc proto) {
+ super(len);
+ this.proto = proto;
+ }
+
+ protected boolean lessThan(SortDoc t1, SortDoc t2) {
+ return t1.lessThan(t2);
+ }
+
+ protected void populate() {
+ Object[] heap = getHeapArray();
+ cache = new SortDoc[heap.length];
+ for (int i = 1; i < heap.length; i++) {
+ cache[i] = heap[i] = proto.copy();
+ }
+ size = maxSize;
+ }
+
+ protected void reset() {
+ Object[] heap = getHeapArray();
+ if(cache != null) {
+ System.arraycopy(cache, 1, heap, 1, heap.length-1);
+ size = maxSize;
+ } else {
+ populate();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/SortValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortValue.java b/solr/core/src/java/org/apache/solr/handler/export/SortValue.java
new file mode 100644
index 0000000..98afd0a
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortValue.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+public interface SortValue extends Comparable<SortValue> {
+ public void setCurrentValue(int docId) throws IOException;
+ public void setNextReader(LeafReaderContext context) throws IOException;
+ public void setCurrentValue(SortValue value);
+ public void reset();
+ public SortValue copy();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
new file mode 100644
index 0000000..9008f71
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringFieldWriter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.schema.FieldType;
+
+class StringFieldWriter extends FieldWriter {
+ private String field;
+ private FieldType fieldType;
+ private CharsRefBuilder cref = new CharsRefBuilder();
+
+ public StringFieldWriter(String field, FieldType fieldType) {
+ this.field = field;
+ this.fieldType = fieldType;
+ }
+
+ public boolean write(int docId, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
+ SortedDocValues vals = DocValues.getSorted(reader, this.field);
+ if (vals.advance(docId) != docId) {
+ return false;
+ }
+ int ord = vals.ordValue();
+
+ BytesRef ref = vals.lookupOrd(ord);
+ fieldType.indexedToReadable(ref, cref);
+ ew.put(this.field, cref.toString());
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/StringValue.java b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
new file mode 100644
index 0000000..23b1413
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/StringValue.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.OrdinalMap;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.LongValues;
+
+class StringValue implements SortValue {
+
+ protected SortedDocValues globalDocValues;
+
+ protected OrdinalMap ordinalMap;
+ protected LongValues toGlobal = LongValues.IDENTITY; // this segment to global ordinal. NN;
+ protected SortedDocValues docValues;
+
+ protected String field;
+ protected int currentOrd;
+ protected IntComp comp;
+ protected int lastDocID;
+
+ public StringValue(SortedDocValues globalDocValues, String field, IntComp comp) {
+ this.globalDocValues = globalDocValues;
+ this.docValues = globalDocValues;
+ if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
+ this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) globalDocValues).mapping;
+ }
+ this.field = field;
+ this.comp = comp;
+ this.currentOrd = comp.resetValue();
+ }
+
+ public StringValue copy() {
+ return new StringValue(globalDocValues, field, comp);
+ }
+
+ public void setCurrentValue(int docId) throws IOException {
+ if (docId < lastDocID) {
+ throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
+ }
+
+ lastDocID = docId;
+
+ if (docId > docValues.docID()) {
+ docValues.advance(docId);
+ }
+ if (docId == docValues.docID()) {
+ currentOrd = (int) toGlobal.get(docValues.ordValue());
+ } else {
+ currentOrd = -1;
+ }
+ }
+
+ public void setCurrentValue(SortValue sv) {
+ StringValue v = (StringValue)sv;
+ this.currentOrd = v.currentOrd;
+ }
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
+ toGlobal = ordinalMap.getGlobalOrds(context.ord);
+ docValues = DocValues.getSorted(context.reader(), field);
+ }
+ lastDocID = 0;
+ }
+
+ public void reset() {
+ this.currentOrd = comp.resetValue();
+ }
+
+ public int compareTo(SortValue o) {
+ StringValue sv = (StringValue)o;
+ return comp.compare(currentOrd, sv.currentOrd);
+ }
+
+ public String toString() {
+ return Integer.toString(this.currentOrd);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/java/org/apache/solr/handler/export/package-info.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/package-info.java b/solr/core/src/java/org/apache/solr/handler/export/package-info.java
new file mode 100644
index 0000000..a516904
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Solr's Export Handler Functionality
+ */
+package org.apache.solr.handler.export;
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
index c45f178..57e684d 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-sortingresponse.xml
@@ -40,7 +40,7 @@
<fieldType name="uuid" class="solr.UUIDField"/>
- <field name="id" type="string" required="true" indexed="true"/>
+ <field name="id" type="string" required="true" indexed="true" docValues="true"/>
<field name="floatdv_m" type="float" indexed="false" stored="false" docValues="true" multiValued="true"/>
<field name="intdv_m" type="int" indexed="false" stored="false" docValues="true" multiValued="true"/>
<field name="doubledv_m" type="double" indexed="false" stored="false" docValues="true" multiValued="true"/>
@@ -54,6 +54,9 @@
<field name="longdv" type="long" indexed="false" stored="false" docValues="true"/>
<field name="datedv" type="date" indexed="false" stored="false" docValues="true"/>
<field name="stringdv" type="string" indexed="false" stored="false" docValues="true"/>
+ <field name="booleandv" type="boolean" indexed="false" stored="false" docValues="true" />
+
+ <dynamicField name="*_s_dv" type="string" indexed="true" stored="true" docValues="true" multiValued="false"/>
<!-- Point fields explicitly -->
<dynamicField name="*_i_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="false"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9d9c3a0c/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
new file mode 100644
index 0000000..55efdee
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/export/TestExportWriter.java
@@ -0,0 +1,749 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.export;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.SuppressForbidden;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.ResultContext;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocList;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestExportWriter extends SolrTestCaseJ4 {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ System.setProperty("export.test", "true");
+ initCore("solrconfig-sortingresponse.xml","schema-sortingresponse.xml");
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ assertU(delQ("*:*"));
+ assertU(commit());
+ createIndex();
+ }
+
+ public static void createIndex() {
+ assertU(adoc("id","1",
+ "floatdv","2.1",
+ "intdv", "1",
+ "stringdv", "hello world",
+ "longdv", "323223232323",
+ "doubledv","2344.345",
+ "intdv_m","100",
+ "intdv_m","250",
+ "floatdv_m", "123.321",
+ "floatdv_m", "345.123",
+ "doubledv_m", "3444.222",
+ "doubledv_m", "23232.2",
+ "longdv_m", "43434343434",
+ "longdv_m", "343332",
+ "stringdv_m", "manchester \"city\"",
+ "stringdv_m", "liverpool",
+ "stringdv_m", "Everton",
+ "datedv", "2017-06-16T07:00:00Z",
+ "datedv_m", "2017-06-16T01:00:00Z",
+ "datedv_m", "2017-06-16T02:00:00Z",
+ "datedv_m", "2017-06-16T03:00:00Z",
+ "datedv_m", "2017-06-16T04:00:00Z"));
+
+ assertU(adoc("id","7",
+ "floatdv","2.1",
+ "intdv", "7",
+ "longdv", "323223232323",
+ "doubledv","2344.345",
+ "floatdv_m", "123.321",
+ "floatdv_m", "345.123",
+ "doubledv_m", "3444.222",
+ "doubledv_m", "23232.2",
+ "longdv_m", "43434343434",
+ "longdv_m", "343332"));
+
+ assertU(commit());
+ assertU(adoc("id","2", "floatdv","2.1", "intdv", "2", "stringdv", "hello world", "longdv", "323223232323","doubledv","2344.344"));
+ assertU(commit());
+ assertU(adoc("id","3",
+ "floatdv","2.1",
+ "intdv", "3",
+ "stringdv", "chello world",
+ "longdv", "323223232323",
+ "doubledv","2344.346",
+ "intdv_m","100",
+ "intdv_m","250",
+ "floatdv_m", "123.321",
+ "floatdv_m", "345.123",
+ "doubledv_m", "3444.222",
+ "doubledv_m", "23232.2",
+ "longdv_m", "43434343434",
+ "longdv_m", "343332",
+ "stringdv_m", "manchester \"city\"",
+ "stringdv_m", "liverpool",
+ "stringdv_m", "everton",
+ "int_is_t", "1",
+ "int_is_t", "1",
+ "int_is_t", "1",
+ "int_is_t", "1"));
+ assertU(commit());
+ assertU(adoc("id","8",
+ "floatdv","2.1",
+ "intdv", "10000000",
+ "stringdv", "chello \"world\"",
+ "longdv", "323223232323",
+ "doubledv","2344.346",
+ "intdv_m","100",
+ "intdv_m","250",
+ "floatdv_m", "123.321",
+ "floatdv_m", "345.123",
+ "doubledv_m", "3444.222",
+ "doubledv_m", "23232.2",
+ "longdv_m", "43434343434",
+ "longdv_m", "343332",
+ "stringdv_m", "manchester \"city\"",
+ "stringdv_m", "liverpool",
+ "stringdv_m", "everton",
+ "datedv", "2017-01-01T00:00:00Z",
+ "datedv_m", "2017-01-01T01:00:00Z",
+ "datedv_m", "2017-01-01T02:00:00Z",
+ "int_is_p", "1",
+ "int_is_p", "1",
+ "int_is_p", "1",
+ "int_is_p", "1"));
+ assertU(commit());
+
+
+ }
+
+ @Test
+ public void testSmallChains() throws Exception {
+ clearIndex();
+
+ assertU(adoc("id","1",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p","1"));
+ assertU(commit());
+
+ assertU(adoc("id","2",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p",Integer.toString(Integer.MIN_VALUE + 1)));
+ assertU(commit());
+
+ assertU(adoc("id","3",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p",Integer.toString(Integer.MIN_VALUE)));
+ assertU(commit());
+
+ //Test single value DocValue output
+ //Expected for asc sort doc3 -> doc2 -> doc1
+ String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "field1_i_p asc,field2_i_p asc"));
+ assertJsonEquals(s, "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":3,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"3\"}\n" +
+ " ,{\n" +
+ " \"id\":\"2\"}\n" +
+ " ,{\n" +
+ " \"id\":\"1\"}]}}");
+
+ clearIndex();
+ //Adding 3 docs of integers with the following values
+ // doc1: Integer.MIN_VALUE,1,2,Integer.MAX_VALUE,3,4,5,6
+ // doc2: Integer.MIN_VALUE,Integer.MIN_VALUE,2,Integer.MAX_VALUE,4,4,5,6
+ // doc3: Integer.MIN_VALUE,Integer.MIN_VALUE,2,Integer.MAX_VALUE,3,4,5,6
+
+ assertU(adoc("id","1",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p","1",
+ "field3_i_p","2",
+ "field4_i_p",Integer.toString(Integer.MAX_VALUE),
+ "field5_i_p","3",
+ "field6_i_p","4",
+ "field7_i_p","5",
+ "field8_i_p","6"));
+ assertU(commit());
+
+ assertU(adoc("id","2",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field3_i_p","2",
+ "field4_i_p",Integer.toString(Integer.MAX_VALUE),
+ "field5_i_p","4",
+ "field6_i_p","4",
+ "field7_i_p","5",
+ "field8_i_p","6"));
+ assertU(commit());
+
+ assertU(adoc("id","3",
+ "field1_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field2_i_p",Integer.toString(Integer.MIN_VALUE),
+ "field3_i_p","2",
+ "field4_i_p",Integer.toString(Integer.MAX_VALUE),
+ "field5_i_p","3",
+ "field6_i_p","4",
+ "field7_i_p","5",
+ "field8_i_p","6"));
+ assertU(commit());
+
+ s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "field1_i_p asc,field2_i_p asc,field3_i_p asc,field4_i_p asc,field5_i_p desc,field6_i_p desc,field7_i_p desc,field8_i_p asc"));
+ assertJsonEquals(s, "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":3,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"2\"}\n" +
+ " ,{\n" +
+ " \"id\":\"3\"}\n" +
+ " ,{\n" +
+ " \"id\":\"1\"}]}}");
+
+ }
+
+ @Test
+ public void testIndexOrder() throws Exception {
+ clearIndex();
+
+ assertU(adoc("id","1", "stringdv","a"));
+ assertU(adoc("id","2", "stringdv","a"));
+
+ assertU(commit());
+
+ assertU(adoc("id","3", "stringdv","a"));
+ assertU(adoc("id","4", "stringdv","a"));
+
+ assertU(commit());
+
+ String expectedResult = "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":4,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"1\"}\n" +
+ " ,{\n" +
+ " \"id\":\"2\"}\n" +
+ " ,{\n" +
+ " \"id\":\"3\"}\n" +
+ " ,{\n" +
+ " \"id\":\"4\"}]}}";
+
+ String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv asc"));
+ assertJsonEquals(s, expectedResult);
+
+ s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv desc"));
+ assertJsonEquals(s, expectedResult);
+
+ }
+
+ @Test
+ public void testStringWithCase() throws Exception {
+ clearIndex();
+
+ assertU(adoc("id","1", "stringdv","a"));
+ assertU(adoc("id","2", "stringdv","ABC"));
+
+ assertU(commit());
+
+ assertU(adoc("id","3", "stringdv","xyz"));
+ assertU(adoc("id","4", "stringdv","a"));
+
+ assertU(commit());
+
+ String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "stringdv desc"));
+ assertJsonEquals(s, "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":4,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"3\"}\n" +
+ " ,{\n" +
+ " \"id\":\"1\"}\n" +
+ " ,{\n" +
+ " \"id\":\"4\"}\n" +
+ " ,{\n" +
+ " \"id\":\"2\"}]}}");
+ }
+
+ @Test
+ public void testBooleanField() throws Exception {
+ clearIndex();
+
+ assertU(adoc("id","1",
+ "booleandv","true"));
+ assertU(commit());
+
+ assertU(adoc("id","2",
+ "booleandv","false"));
+ assertU(commit());
+
+ String s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "booleandv asc"));
+ assertJsonEquals(s, "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":2,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"2\"}\n" +
+ " ,{\n" +
+ " \"id\":\"1\"}]}}");
+
+ s = h.query(req("q", "*:*", "qt", "/export", "fl", "id", "sort", "booleandv desc"));
+ assertJsonEquals(s, "{\n" +
+ " \"responseHeader\":{\"status\":0},\n" +
+ " \"response\":{\n" +
+ " \"numFound\":2,\n" +
+ " \"docs\":[{\n" +
+ " \"id\":\"1\"}\n" +
+ " ,{\n" +
+ " \"id\":\"2\"}]}}");
+ }
+
+ @Test
+ public void testSortingOutput() throws Exception {
+
+ //Test single value DocValue output
+ String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":1,\"stringdv\":\"hello world\",\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
+
+ //Test null value string:
+ s = h.query(req("q", "id:7", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv\":2.1,\"intdv\":7,\"longdv\":323223232323,\"doubledv\":2344.345}]}}");
+
+ //Test multiValue docValues output
+ s = h.query(req("q", "id:1", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"intdv_m\":[100,250],\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434],\"stringdv_m\":[\"Everton\",\"liverpool\",\"manchester \\\"city\\\"\"]}]}}");
+
+ //Test multiValues docValues output with nulls
+ s = h.query(req("q", "id:7", "qt", "/export", "fl", "intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"floatdv_m\":[123.321,345.123],\"doubledv_m\":[3444.222,23232.2],\"longdv_m\":[343332,43434343434]}]}}");
+
+ //Test single sort param is working
+ s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
+
+ s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
+
+ // Test sort on String will null value. Null value should sort last on desc and first on asc.
+
+ s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":7}]}}");
+
+ s = h.query(req("q", "id:(1 7)", "qt", "/export", "fl", "intdv", "sort", "stringdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":7},{\"intdv\":1}]}}");
+
+
+ //Test multi-sort params
+ s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":2},{\"intdv\":1}]}}");
+
+ s = h.query(req("q", "id:(1 2)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":2, \"docs\":[{\"intdv\":1},{\"intdv\":2}]}}");
+
+ //Test three sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+
+ //Test three sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,stringdv desc,intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+
+ //Test four sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,floatdv desc,floatdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+
+ //Test five sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc,floatdv desc,floatdv desc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+
+ //Test six sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+
+ //Test seven sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv desc,intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+
+ //Test eight sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,intdv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+
+ //Test nine sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv asc,intdv desc,floatdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv desc,intdv asc,floatdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+
+ //Test ten sort fields
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "intdv asc,floatdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv asc,intdv desc,floatdv desc,floatdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":1},{\"intdv\":2},{\"intdv\":3}]}}");
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "floatdv asc,intdv desc,floatdv asc,floatdv desc,floatdv asc,floatdv desc,intdv desc,intdv asc,floatdv desc,floatdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":2},{\"intdv\":1}]}}");
+
+ s = h.query(req("q", "id:(1 2 3)", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":1},{\"intdv\":2}]}}");
+
+ s = h.query(req("q", "intdv:[2 TO 1000]", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":3, \"docs\":[{\"intdv\":3},{\"intdv\":7},{\"intdv\":2}]}}");
+
+ s = h.query(req("q", "stringdv:blah", "qt", "/export", "fl", "intdv", "sort", "doubledv desc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":0, \"docs\":[]}}");
+
+ s = h.query(req("q", "id:8", "qt", "/export", "fl", "stringdv", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"stringdv\":\"chello \\\"world\\\"\"}]}}");
+ }
+
+ private void assertJsonEquals(String actual, String expected) {
+ assertEquals(Utils.toJSONString(Utils.fromJSONString(expected)), Utils.toJSONString(Utils.fromJSONString(actual)));
+ }
+
+ @Test
+ public void testExportRequiredParams() throws Exception {
+
+ //Test whether missing required parameters returns expected errors.
+
+ //String s = h.query(req("q", "id:1", "qt", "/export", "fl", "floatdv,intdv,stringdv,longdv,doubledv", "sort", "intdv asc"));
+ String s;
+ s = h.query(req("qt", "/export"));
+ assertTrue("Should have had a sort error", s.contains("No sort criteria"));
+ s = h.query(req("sort", "intdv asc", "qt", "/export"));
+ assertTrue("Should have had fl error", s.contains("export field list (fl) must be specified"));
+ s = h.query(req("sort", "intdv asc", "qt", "/export", "fl", "stringdv"));
+ // Interesting you don't even need to specify a "q" parameter.
+
+ }
+
+ @Test
+ public void testDates() throws Exception {
+ String s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv", "sort", "datedv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv\":\"2017-06-16T07:00:00Z\"}]}}");
+ s = h.query(req("q", "id:1", "qt", "/export", "fl", "datedv_m", "sort", "datedv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"datedv_m\":[\"2017-06-16T01:00:00Z\",\"2017-06-16T02:00:00Z\",\"2017-06-16T03:00:00Z\",\"2017-06-16T04:00:00Z\"]}]}}");
+ }
+
+ @Test
+ public void testDuplicates() throws Exception {
+ // see SOLR-10924
+ String expected = h.getCore().getLatestSchema().getField("int_is_t").getType().isPointField()
+ ? "1,1,1,1" : "1";
+ String s = h.query(req("q", "id:3", "qt", "/export", "fl", "int_is_t", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_t\":["+expected+"]}]}}");
+ expected = h.getCore().getLatestSchema().getField("int_is_p").getType().isPointField()
+ ? "1,1,1,1" : "1";
+ s = h.query(req("q", "id:8", "qt", "/export", "fl", "int_is_p", "sort", "intdv asc"));
+ assertJsonEquals(s, "{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":1, \"docs\":[{\"int_is_p\":[1,1,1,1]}]}}");
+ }
+
+ /**
+ * This test doesn't validate the correctness of results, it just compares the response of the same request
+ * when asking for Trie fields vs Point fields. Can be removed once Trie fields are no longer supported
+ */
+ @Test
+ @SuppressForbidden(reason="using new Date(time) to create random dates")
+ public void testRandomNumerics() throws Exception {
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT);
+ assertU(delQ("*:*"));
+ assertU(commit());
+ List<String> trieFields = new ArrayList<String>();
+ List<String> pointFields = new ArrayList<String>();
+ for (String mv:new String[]{"s", ""}) {
+ for (String indexed:new String[]{"_ni", ""}) {
+ for (String type:new String[]{"i", "l", "f", "d", "dt"}) {
+ String field = "number_" + type + mv + indexed;
+ SchemaField sf = h.getCore().getLatestSchema().getField(field + "_t");
+ assertTrue(sf.hasDocValues());
+ assertTrue(sf.getType().getNumberType() != null);
+
+ sf = h.getCore().getLatestSchema().getField(field + "_p");
+ assertTrue(sf.hasDocValues());
+ assertTrue(sf.getType().getNumberType() != null);
+ assertTrue(sf.getType().isPointField());
+
+ trieFields.add(field + "_t");
+ pointFields.add(field + "_p");
+ }
+ }
+ }
+ for (int i = 0; i < atLeast(100); i++) {
+ if (random().nextInt(20) == 0) {
+ //have some empty docs
+ assertU(adoc("id", String.valueOf(i)));
+ continue;
+ }
+
+ if (random().nextInt(20) == 0 && i > 0) {
+ //delete some docs
+ assertU(delI(String.valueOf(i - 1)));
+ }
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", String.valueOf(i));
+ addInt(doc, random().nextInt(), false);
+ addLong(doc, random().nextLong(), false);
+ addFloat(doc, random().nextFloat() * 3000 * (random().nextBoolean()?1:-1), false);
+ addDouble(doc, random().nextDouble() * 3000 * (random().nextBoolean()?1:-1), false);
+ addDate(doc, dateFormat.format(new Date()), false);
+
+ // MV need to be unique in order to be the same in Trie vs Points
+ Set<Integer> ints = new HashSet<>();
+ Set<Long> longs = new HashSet<>();
+ Set<Float> floats = new HashSet<>();
+ Set<Double> doubles = new HashSet<>();
+ Set<String> dates = new HashSet<>();
+ for (int j=0; j < random().nextInt(20); j++) {
+ ints.add(random().nextInt());
+ longs.add(random().nextLong());
+ floats.add(random().nextFloat() * 3000 * (random().nextBoolean()?1:-1));
+ doubles.add(random().nextDouble() * 3000 * (random().nextBoolean()?1:-1));
+ dates.add(dateFormat.format(new Date(System.currentTimeMillis() + random().nextInt())));
+ }
+ ints.stream().forEach((val)->addInt(doc, val, true));
+ longs.stream().forEach((val)->addLong(doc, val, true));
+ floats.stream().forEach((val)->addFloat(doc, val, true));
+ doubles.stream().forEach((val)->addDouble(doc, val, true));
+ dates.stream().forEach((val)->addDate(doc, val, true));
+
+ assertU(adoc(doc));
+ if (random().nextInt(20) == 0) {
+ assertU(commit());
+ }
+ }
+ assertU(commit());
+ doTestQuery("id:1", trieFields, pointFields);
+ doTestQuery("*:*", trieFields, pointFields);
+ doTestQuery("id:[0 TO 2]", trieFields, pointFields);// "id" field is really a string, this is not a numeric range query
+ doTestQuery("id:[0 TO 9]", trieFields, pointFields);
+ doTestQuery("id:DOES_NOT_EXIST", trieFields, pointFields);
+ }
+
+ @Test
+ public void testMultipleSorts() throws Exception {
+ assertU(delQ("*:*"));
+ assertU(commit());
+
+ int numDocs = 1000;
+
+ //10 unique values
+ String[] str_vals = new String[10];
+ for (int i=0; i<str_vals.length; i++) {
+ str_vals[i] = TestUtil.randomSimpleString(random(), 10);
+ }
+
+ float[] float_vals = new float[10];
+ float_vals[0] = 0.0f;
+ float_vals[1] = +0.0f;
+ float_vals[2] = -0.0f;
+ float_vals[3] = +0.00001f;
+ float_vals[4] = +0.000011f;
+ float_vals[5] = Float.MAX_VALUE;
+ float_vals[6] = Float.MIN_VALUE;
+ float_vals[7] = 1/3f; //0.33333334
+ float_vals[8] = 0.33333333f;
+ float_vals[9] = random().nextFloat();
+
+ for (int i = 0; i < numDocs; i++) {
+ int number = TestUtil.nextInt(random(), 0, 9);
+ assertU(adoc("id", String.valueOf(i),
+ "floatdv", String.valueOf(number),
+ "intdv", String.valueOf(number),
+ "stringdv", String.valueOf(str_vals[number]),
+ "longdv", String.valueOf(number),
+ "doubledv", String.valueOf(number),
+ "datedv", randomSkewedDate(),
+ "booleandv", String.valueOf(random().nextBoolean()),
+ "field1_s_dv", String.valueOf(str_vals[number]),
+ "field2_i_p", String.valueOf(number),
+ "field3_l_p", String.valueOf(number)));
+ if (numDocs % 3000 ==0) {
+ assertU(commit());
+ }
+ }
+ assertU(commit());
+
+ validateSort(numDocs);
+ }
+
+ private void validateSort(int numDocs) throws Exception {
+ // 10 fields
+ List<String> fieldNames = new ArrayList<>(Arrays.asList("floatdv", "intdv", "stringdv", "longdv", "doubledv",
+ "datedv", "booleandv", "field1_s_dv", "field2_i_p", "field3_l_p"));
+
+ SortFields[] fieldSorts = new SortFields[TestUtil.nextInt(random(), 1, fieldNames.size())];
+ for (int i = 0; i < fieldSorts.length; i++) {
+ fieldSorts[i] = new SortFields(fieldNames.get(TestUtil.nextInt(random(), 0, fieldNames.size() - 1)));
+ fieldNames.remove(fieldSorts[i].getField());
+ }
+ String[] fieldWithOrderStrs = new String[fieldSorts.length];
+ String[] fieldStrs = new String[fieldSorts.length];
+ for (int i = 0; i < fieldSorts.length; i++) {
+ fieldWithOrderStrs[i] = fieldSorts[i].getFieldWithOrder();
+ fieldStrs[i] = fieldSorts[i].getField();
+ }
+
+ String sortStr = String.join(",", fieldWithOrderStrs); // sort : field1 asc, field2 desc
+ String fieldsStr = String.join(",", fieldStrs); // fl : field1, field2
+
+ String resp = h.query(req("q", "*:*", "qt", "/export", "fl", "id," + fieldsStr, "sort", sortStr));
+
+ //We cannot compare /select vs /export as for docs with the same values ( ties ) the ordering is different
+ SolrQueryRequest req = null;
+ try {
+ req = req("q", "*:*", "qt", "/select", "fl", "id," + fieldsStr, "sort", sortStr, "rows", Integer.toString(numDocs));
+
+ SolrQueryResponse selectRsp = h.queryAndResponse("", req);
+ DocList selectDocList = ((ResultContext)selectRsp.getResponse()).getDocList();
+ assert selectDocList.size() == numDocs;
+ DocIterator selectDocListIter = selectDocList.iterator();
+
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap respMap = mapper.readValue(resp, HashMap.class);
+ List docs = (ArrayList) ((HashMap) respMap.get("response")).get("docs");
+ assert docs.size() == numDocs;
+
+ for (int i = 0; i < docs.size() - 1; i++) { // docs..
+ assertEquals("Position:" + i + " has different id value" , String.valueOf(selectDocListIter.nextDoc()), String.valueOf(((HashMap) docs.get(i)).get("id")));
+
+ for (int j = 0; j < fieldSorts.length; j++) { // fields ..
+ String field = fieldSorts[j].getField();
+ String sort = fieldSorts[j].getSort();
+ String fieldVal1 = String.valueOf(((HashMap) docs.get(i)).get(field)); // 1st doc
+ String fieldVal2 = String.valueOf(((HashMap) docs.get(i + 1)).get(field)); // 2nd obj
+ if (fieldVal1.equals(fieldVal2)) {
+ continue;
+ } else {
+ if (sort.equals("asc")) {
+ if (field.equals("stringdv") || field.equals("field1_s_dv")|| field.equals("datedv") || field.equals("booleandv")) { // use string comparator
+ assertTrue(fieldVal1.compareTo(fieldVal2) < 0);
+ } else if (field.equals("doubledv")){
+ assertTrue(Double.compare(Double.valueOf(fieldVal1), Double.valueOf(fieldVal2)) <= 0);
+ } else if(field.equals("floatdv")) {
+ assertTrue(Float.compare(Float.valueOf(fieldVal1), Float.valueOf(fieldVal2)) <= 0);
+ } else if(field.equals("intdv") || "field2_i_p".equals(field)) {
+ assertTrue(Integer.compare(Integer.valueOf(fieldVal1), Integer.valueOf(fieldVal2)) <= 0);
+ } else if(field.equals("longdv") || field.equals("field3_l_p")) {
+ assertTrue(Long.compare(Integer.valueOf(fieldVal1), Long.valueOf(fieldVal2)) <= 0);
+ }
+ } else {
+ if (field.equals("stringdv") || field.equals("field1_s_dv")|| field.equals("datedv") || field.equals("booleandv")) { // use string comparator
+ assertTrue(fieldVal1.compareTo(fieldVal2) > 0);
+ } else if (field.equals("doubledv")){
+ assertTrue(Double.compare(Double.valueOf(fieldVal1), Double.valueOf(fieldVal2)) >= 0);
+ } else if(field.equals("floatdv")) {
+ assertTrue(Float.compare(Float.valueOf(fieldVal1), Float.valueOf(fieldVal2)) >= 0);
+ } else if(field.equals("intdv") || "field2_i_p".equals(field)) {
+ assertTrue(Integer.compare(Integer.valueOf(fieldVal1), Integer.valueOf(fieldVal2)) >= 0);
+ } else if(field.equals("longdv") || field.equals("field3_l_p")) {
+ assertTrue(Long.compare(Integer.valueOf(fieldVal1), Long.valueOf(fieldVal2)) >= 0);
+ }
+ }
+ break;
+ }
+ }
+ }
+ } finally {
+ if (req != null) {
+ req.close();
+ }
+ }
+ }
+
+ private class SortFields {
+ String fieldName;
+ String sortOrder;
+ String[] orders = {"asc", "desc"};
+
+ SortFields(String fn) {
+ this.fieldName = fn;
+ this.sortOrder = orders[random().nextInt(2)];
+ }
+
+ public String getFieldWithOrder() {
+ return this.fieldName + " " + this.sortOrder;
+ }
+
+ public String getField() {
+ return this.fieldName;
+ }
+
+ public String getSort() {
+ return this.sortOrder;
+ }
+ }
+
+ private void doTestQuery(String query, List<String> trieFields, List<String> pointFields) throws Exception {
+ String trieFieldsFl = String.join(",", trieFields);
+ String pointFieldsFl = String.join(",", pointFields);
+ String sort = pickRandom((String)pickRandom(trieFields.toArray()), (String)pickRandom(pointFields.toArray())).replace("s_", "_") + pickRandom(" asc", " desc");
+ String resultPoints = h.query(req("q", query, "qt", "/export", "fl", pointFieldsFl, "sort", sort));
+ String resultTries = h.query(req("q", query, "qt", "/export", "fl", trieFieldsFl, "sort", sort));
+ assertJsonEquals(resultPoints.replaceAll("_p", ""), resultTries.replaceAll("_t", ""));
+ }
+
+ private void addFloat(SolrInputDocument doc, float value, boolean mv) {
+ addField(doc, "f", String.valueOf(value), mv);
+ }
+
+ private void addDouble(SolrInputDocument doc, double value, boolean mv) {
+ addField(doc, "d", String.valueOf(value), mv);
+ }
+
+ private void addLong(SolrInputDocument doc, long value, boolean mv) {
+ addField(doc, "l", String.valueOf(value), mv);
+ }
+
+ private void addInt(SolrInputDocument doc, int value, boolean mv) {
+ addField(doc, "i", String.valueOf(value), mv);
+ }
+
+ private void addDate(SolrInputDocument doc, String value, boolean mv) {
+ addField(doc, "dt", value, mv);
+ }
+
+ private void addField(SolrInputDocument doc, String type, String value, boolean mv) {
+ doc.addField("number_" + type + (mv?"s":"") + "_t", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_p", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_ni_t", value);
+ doc.addField("number_" + type + (mv?"s":"") + "_ni_p", value);
+ }
+
+}