You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2019/04/22 20:29:48 UTC
[lucene-solr] branch branch_8x updated: SOLR-13337: faster /terms
response in distributed mode.
This is an automated email from the ASF dual-hosted git repository.
mkhl pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 075c123 SOLR-13337: faster /terms response in distributed mode.
075c123 is described below
commit 075c123cb164f13a20796b088646425a76d98419
Author: Mikhail Khludnev <mk...@apache.org>
AuthorDate: Mon Apr 22 00:16:04 2019 +0300
SOLR-13337: faster /terms response in distributed mode.
---
solr/CHANGES.txt | 3 ++
.../solr/handler/component/TermsComponent.java | 29 ++++++++-----
.../component/DistributedTermsComponentTest.java | 48 ++++++++++++++++++----
.../solr/handler/component/TermsComponentTest.java | 23 +++++++++++
4 files changed, 85 insertions(+), 18 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index fda0351..173a3b2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -212,6 +212,9 @@ Improvements
* SOLR-11035: (at least) 2 distinct failures possible when clients attempt searches during SolrCore reload,
added test band-aid for DocValuesNotIndexedTest.
+* SOLR-13337: Only request the minimum required number of terms from each shard when using terms.sort=index and none
+ are discarded due to terms.min/maxcount (Morten Bøgeskov,Munendra S N via Mikhail Khludnev)
+
Other Changes
----------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
index 1258e60..fb61a43 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
@@ -445,22 +445,31 @@ public class TermsComponent extends SearchComponent {
rb._termsHelper = null;
}
- private static ShardRequest createShardQuery(SolrParams params) {
+ static ShardRequest createShardQuery(SolrParams params) {
ShardRequest sreq = new ShardRequest();
sreq.purpose = ShardRequest.PURPOSE_GET_TERMS;
// base shard request on original parameters
sreq.params = new ModifiableSolrParams(params);
- // remove any limits for shards, we want them to return all possible
- // responses
- // we want this so we can calculate the correct counts
- // dont sort by count to avoid that unnecessary overhead on the shards
- sreq.params.remove(TermsParams.TERMS_MAXCOUNT);
- sreq.params.remove(TermsParams.TERMS_MINCOUNT);
- sreq.params.set(TermsParams.TERMS_LIMIT, -1);
- sreq.params.set(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_INDEX);
-
+ // if using index-order, we can send all parameters to all shards
+ // since all required data are returned within the first n rows
+ String actualSort = sreq.params.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT);
+
+ boolean fast = actualSort.equals(TermsParams.TERMS_SORT_INDEX) &&
+ sreq.params.getLong(TermsParams.TERMS_MINCOUNT, 0) <= 1 &&
+ sreq.params.getLong(TermsParams.TERMS_MAXCOUNT, -1) <=0;
+
+ if (!fast) {
+ // remove any limits for shards, we want them to return all possible
+ // responses
+ // we want this so we can calculate the correct counts
+ // dont sort by count to avoid that unnecessary overhead on the shards
+ sreq.params.remove(TermsParams.TERMS_MAXCOUNT);
+ sreq.params.remove(TermsParams.TERMS_MINCOUNT);
+ sreq.params.set(TermsParams.TERMS_LIMIT, -1);
+ sreq.params.set(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_INDEX);
+ }
return sreq;
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
index 7b1fd8c..329cd80 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedTermsComponentTest.java
@@ -16,7 +16,14 @@
*/
package org.apache.solr.handler.component;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+import java.util.stream.Stream;
+
import org.apache.solr.BaseDistributedSearchTestCase;
+import org.apache.solr.client.solrj.response.QueryResponse;
import org.junit.Test;
/**
@@ -30,15 +37,16 @@ public class DistributedTermsComponentTest extends BaseDistributedSearchTestCase
@Test
public void test() throws Exception {
+ Random random = random();
del("*:*");
- index(id, 18, "b_t", "snake a,b spider shark snail slug seal", "foo_i", "1");
- index(id, 19, "b_t", "snake spider shark snail slug", "foo_i", "2");
- index(id, 20, "b_t", "snake spider shark snail", "foo_i", "3");
- index(id, 21, "b_t", "snake spider shark", "foo_i", "2");
- index(id, 22, "b_t", "snake spider", "c_t", "snake spider");
- index(id, 23, "b_t", "snake", "c_t", "snake");
- index(id, 24, "b_t", "ant zebra", "c_t", "ant zebra");
- index(id, 25, "b_t", "zebra", "c_t", "zebra");
+ index(id, random.nextInt(), "b_t", "snake a,b spider shark snail slug seal", "foo_i", "1");
+ index(id, random.nextInt(), "b_t", "snake spider shark snail slug", "foo_i", "2");
+ index(id, random.nextInt(), "b_t", "snake spider shark snail", "foo_i", "3");
+ index(id, random.nextInt(), "b_t", "snake spider shark", "foo_i", "2");
+ index(id, random.nextInt(), "b_t", "snake spider", "c_t", "snake spider");
+ index(id, random.nextInt(), "b_t", "snake", "c_t", "snake");
+ index(id, random.nextInt(), "b_t", "ant zebra", "c_t", "ant zebra");
+ index(id, random.nextInt(), "b_t", "zebra", "c_t", "zebra");
commit();
handle.clear();
@@ -48,13 +56,37 @@ public class DistributedTermsComponentTest extends BaseDistributedSearchTestCase
query("qt", "/terms", "shards.qt", "/terms", "terms.limit", 5, "terms", "true", "terms.fl", "b_t", "terms.lower", "s");
query("qt", "/terms", "shards.qt", "/terms", "terms.limit", 5, "terms", "true", "terms.fl", "b_t", "terms.prefix", "sn", "terms.lower", "sn");
query("qt", "/terms", "shards.qt", "/terms", "terms.limit", 5, "terms", "true", "terms.fl", "b_t", "terms.prefix", "s", "terms.lower", "s", "terms.upper", "sn");
+ // terms.sort
query("qt", "/terms", "shards.qt", "/terms", "terms.limit", 5, "terms", "true", "terms.fl", "b_t", "terms.prefix", "s", "terms.lower", "s", "terms.sort", "index");
query("qt", "/terms", "shards.qt", "/terms", "terms.limit", 5, "terms", "true", "terms.fl", "b_t", "terms.prefix", "s", "terms.lower", "s", "terms.upper", "sn", "terms.sort", "index");
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "b_t", "terms.sort", "index");
+ // terms.list
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "b_t", "terms.list", "snake,zebra,ant,bad");
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "foo_i", "terms.list", "2,3,1");
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "foo_i", "terms.stats", "true","terms.list", "2,3,1");
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "b_t", "terms.list", "snake,zebra", "terms.ttf", "true");
query("qt", "/terms", "shards.qt", "/terms", "terms", "true", "terms.fl", "b_t", "terms.fl", "c_t", "terms.list", "snake,ant,zebra", "terms.ttf", "true");
}
+
+ protected QueryResponse query(Object... q) throws Exception {
+ if (Stream.of(q).noneMatch(s->s.equals("terms.list"))) {
+ // SOLR-9243 doesn't support max/min count
+ for (int i = 0; i < q.length; i+=2) {
+ if (q[i].equals("terms.sort") && q[i+1].equals("index") || rarely()) {
+ List<Object> params = new ArrayList<Object>(Arrays.asList(q));
+ if (usually()) {
+ params.add("terms.mincount");
+ params.add(random().nextInt(4)-1);
+ }
+ if (usually()) {
+ params.add("terms.maxcount");
+ params.add(random().nextInt(4)-1);
+ }
+ q = params.toArray(new Object[params.size()]);
+ break;
+ }
+ }
+ }
+ return super.query(q);
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java
index adaf26d..1fa100e 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/TermsComponentTest.java
@@ -511,4 +511,27 @@ public class TermsComponentTest extends SolrTestCaseJ4 {
assertU(commit());
}
}
+
+ @Test
+ public void testTermsSortIndexDistribution() {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_INDEX);
+ params.set(TermsParams.TERMS_LIMIT, "any-number");
+ assertEquals(params.toString(), createShardQueryParamsString(params));
+ params.set(TermsParams.TERMS_MINCOUNT, "0");
+ assertEquals(params.toString(), createShardQueryParamsString(params));
+ params.set(TermsParams.TERMS_MINCOUNT, "1");
+ assertEquals(params.toString(), createShardQueryParamsString(params));
+ // include all (also lower mincount) since 2 shards can have one each
+ params.set(TermsParams.TERMS_MINCOUNT, "2");
+ assertNotEquals(params.toString(), createShardQueryParamsString(params));
+ // "unlimited" since 2 shards can have 30 each, and term then should not be included
+ params.remove(TermsParams.TERMS_MINCOUNT);
+ params.set(TermsParams.TERMS_MAXCOUNT, "32");
+ assertNotEquals(params.toString(), createShardQueryParamsString(params));
+ }
+
+ private static String createShardQueryParamsString(ModifiableSolrParams params) {
+ return TermsComponent.createShardQuery(params).params.toString();
+ }
}