You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ep...@apache.org on 2022/08/08 14:53:15 UTC
[solr] branch branch_9x updated: SOLR-9359: Make Config API work for warming queries (#875)
This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new fa8b7eb6d74 SOLR-9359: Make Config API work for warming queries (#875)
fa8b7eb6d74 is described below
commit fa8b7eb6d749919c1cc8c8b1518347215d80b5be
Author: Andy Webb <an...@gmail.com>
AuthorDate: Mon Aug 8 15:52:17 2022 +0100
SOLR-9359: Make Config API work for warming queries (#875)
Co-authored-by: Andy Webb <an...@gmail.com>
Co-authored-by: Eric Pugh <ep...@opensourceconnections.com>
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/QuerySenderListener.java | 37 +++++++++++++-
.../apache/solr/core/QuerySenderListenerTest.java | 49 ++++++++++++++++++
.../apache/solr/core/TestSolrConfigHandler.java | 45 +++++++++++++++++
.../configuration-guide/pages/caches-warming.adoc | 58 ++++++++++++++++++++++
5 files changed, 190 insertions(+), 1 deletion(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6343babd18c..5e444c59248 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -55,6 +55,8 @@ Optimizations
* SOLR-16266: Eliminate unneccessary byte[] copy in Http2SolrClient (hossman)
+* SOLR-9359: Enable warming queries to be managed using Config API (Andy Webb, Eric Pugh)
+
Bug Fixes
---------------------
* SOLR-15918: Skip repetitive parent znode creation on config set upload (Mike Drob)
diff --git a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
index 273bc70faa2..30c310ca0e3 100644
--- a/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
+++ b/solr/core/src/java/org/apache/solr/core/QuerySenderListener.java
@@ -19,6 +19,7 @@ package org.apache.solr.core;
import static org.apache.solr.common.params.CommonParams.DISTRIB;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.List;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
@@ -46,7 +47,8 @@ public class QuerySenderListener extends AbstractSolrEventListener {
final SolrIndexSearcher searcher = newSearcher;
log.debug("QuerySenderListener sending requests to {}", newSearcher);
@SuppressWarnings("unchecked")
- List<NamedList<Object>> allLists = (List<NamedList<Object>>) getArgs().get("queries");
+ List<NamedList<Object>> allLists =
+ convertQueriesToList((ArrayList<Object>) getArgs().getAll("queries"));
if (allLists == null) return;
for (NamedList<Object> nlst : allLists) {
try {
@@ -102,4 +104,37 @@ public class QuerySenderListener extends AbstractSolrEventListener {
}
log.info("QuerySenderListener done.");
}
+
+ protected static List<NamedList<Object>> convertQueriesToList(ArrayList<Object> queries) {
+
+ List<NamedList<Object>> allLists = new ArrayList<NamedList<Object>>();
+
+ for (Object o : queries) {
+ if (o instanceof ArrayList) {
+ // XML config from solrconfig.xml triggers this path
+ for (Object o2 : (ArrayList) o) {
+ if (o2 instanceof NamedList) {
+ @SuppressWarnings("unchecked")
+ NamedList<Object> o3 = (NamedList<Object>) o2;
+ allLists.add(o3);
+ } else {
+ // this is triggered by unexpected <str> elements
+ // (unexpected <arr> is ignored)
+ // also by nested lists in JSON from Config API
+ log.warn("ignoring unsupported warming config ({})", o2);
+ }
+ }
+ } else if (o instanceof NamedList) {
+ // JSON config from Config API triggers this path
+ @SuppressWarnings("unchecked")
+ NamedList<Object> o3 = (NamedList<Object>) o;
+ allLists.add(o3);
+ } else {
+ // NB different message format to above so messages can be differentiated
+ log.warn("ignoring unsupported warming config - {}", o);
+ }
+ }
+
+ return allLists;
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/core/QuerySenderListenerTest.java b/solr/core/src/test/org/apache/solr/core/QuerySenderListenerTest.java
new file mode 100644
index 00000000000..ebde00c623a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/QuerySenderListenerTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.util.NamedList;
+import org.junit.Test;
+
+public class QuerySenderListenerTest extends SolrTestCaseJ4 {
+
+ @Test
+ public void testConvertQueriesToList() {
+
+ // represents a warming query
+ NamedList<Object> query = new NamedList<Object>();
+
+ // in the JSON config model, queries is an ArrayList of NamedLists
+ ArrayList<Object> queries = new ArrayList<Object>();
+ queries.add(query);
+
+ // in the XML config model, queries is an ArrayList of ArrayLists of NamedLists
+ ArrayList<Object> queriesList = new ArrayList<Object>();
+ queriesList.add(query);
+ queries.add(queriesList);
+
+ // this unexpected item should be ignored
+ queries.add("test");
+
+ // the output having a length of 2 proves both expected models were handled
+ List<NamedList<Object>> allLists = QuerySenderListener.convertQueriesToList(queries);
+ assertEquals(allLists.size(), 2);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
index 6c42c5b702b..415eb0c31c8 100644
--- a/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
+++ b/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
@@ -373,6 +373,51 @@ public class TestSolrConfigHandler extends RestTestBase {
null,
TIMEOUT_S);
+ payload =
+ "{\n"
+ + "'add-listener' : { 'event' : 'firstSearcher', 'class': 'solr.QuerySenderListener', "
+ + "'name':'f7fb2d87bea44464af2401cf33f42b69', "
+ + "'queries':[{'q':'static firstSearcher warming in solrconfig.xml'}]"
+ + "}\n"
+ + "}";
+ runConfigCommand(writeHarness, "/config", payload);
+ testForResponseElement(
+ writeHarness,
+ testServerBaseUrl,
+ "/config",
+ cloudSolrClient,
+ asList("config", "listener[0]", "class"),
+ "solr.QuerySenderListener",
+ TIMEOUT_S);
+
+ payload =
+ "{\n"
+ + "'update-listener' : { 'event' : 'firstSearcher', 'class': 'org.apache.solr.core.QuerySenderListener', "
+ + "'name':'f7fb2d87bea44464af2401cf33f42b69', "
+ + "'queries':[{'q':'static firstSearcher warming in solrconfig.xml'}]"
+ + "}\n"
+ + "}";
+ runConfigCommand(writeHarness, "/config", payload);
+ testForResponseElement(
+ writeHarness,
+ testServerBaseUrl,
+ "/config",
+ cloudSolrClient,
+ asList("config", "listener[0]", "class"),
+ "org.apache.solr.core.QuerySenderListener",
+ TIMEOUT_S);
+
+ payload = "{\n" + "'delete-listener' : 'f7fb2d87bea44464af2401cf33f42b69'" + "}";
+ runConfigCommand(writeHarness, "/config", payload);
+ testForResponseElement(
+ writeHarness,
+ testServerBaseUrl,
+ "/config",
+ cloudSolrClient,
+ asList("config", "listener"),
+ null,
+ TIMEOUT_S);
+
payload =
"{\n"
+ "'create-searchcomponent' : { 'name' : 'tc', 'class': 'org.apache.solr.handler.component.TermsComponent'}\n"
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc
index 67e447ef959..4a8e584f125 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/caches-warming.adoc
@@ -342,3 +342,61 @@ A key best practice is to modify these defaults before taking your application t
There is no point in auto-warming your Searcher with the query string "static firstSearcher warming in solrconfig.xml" if that is not relevant to your search application.
====
+
+### Managing warming queries with the Config API
+
+Warming queries may be managed using the xref:config-api.adoc[Config API] using commands such as the below.
+
+Multiple sets with different `name` attributes may be provided and any changes will take place immediately without a restart being necessary.
+
+#### adding warming query sets
+
+To add a set of warming queries, use the `add-listener` command. (You may wish to provide `firstSearcher` queries too with a separate command and `name`.)
+
+[source,json]
+----
+{
+ "add-listener": {
+ "name": "my-warming-queries",
+ "event": "newSearcher",
+ "class": "solr.QuerySenderListener",
+ "queries": [
+ { "q": "solr", "sort": "price asc" }
+ ]
+ }
+}
+----
+
+#### updating query sets
+
+To update a set, use the `update-listener` command. Note that the whole set will be replaced.
+
+[source,json]
+----
+{
+ "update-listener": {
+ "name": "my-warming-queries",
+ "event": "newSearcher",
+ "class": "solr.QuerySenderListener",
+ "queries": [
+ { "q": "solr", "sort": "price asc" },
+ { "q": "rocks", "sort": "weight asc" }
+ ]
+ }
+}
+----
+
+If warming queries are defined in `solrconfig.xml` they may be overridden using the Config API if a `name` attribute is added to each `<listener>` element.
+
+#### removing query sets
+
+To delete a set of queries, use `delete-listener`:
+
+[source,json]
+----
+{
+ "delete-listener": "my-warming-queries"
+}
+----
+
+Note that for warming query sets originally defined in `solrconfig.xml`, using the `delete-listener` command will revert to the `solrconfig.xml` set rather than removing the warming queries.