You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2023/07/17 19:19:08 UTC
[solr] branch main updated: SOLR-16856: Set autoSoftCommit for built-in configSets (#1777)
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 0697fde07a2 SOLR-16856: Set autoSoftCommit for built-in configSets (#1777)
0697fde07a2 is described below
commit 0697fde07a2e423106db7e355461a6a471f9fae5
Author: Houston Putman <ho...@apache.org>
AuthorDate: Mon Jul 17 15:19:02 2023 -0400
SOLR-16856: Set autoSoftCommit for built-in configSets (#1777)
The default is 3 seconds.
---
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/cli/RunExampleTool.java | 32 ------
.../apache/solr/cloud/SolrCloudExampleTest.java | 111 ---------------------
.../solr/configsets/_default/conf/solrconfig.xml | 2 +-
.../conf/solrconfig.xml | 2 +-
.../pages/commits-transaction-logs.adoc | 6 +-
.../pages/taking-solr-to-production.adoc | 2 +-
.../pages/tutorial-techproducts.adoc | 31 +-----
.../pages/major-changes-in-solr-9.adoc | 9 ++
9 files changed, 18 insertions(+), 179 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index f42986cad3c..268cc31326e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -81,6 +81,8 @@ Other Changes
* SOLR-16842: Eliminate special case code in Solr CLI by introducing explicit VersionTool.java. (Eric Pugh, Will White)
+* SOLR-16856: The default auto-soft-commit time has been set to 3 seconds. Previously, auto-soft-commit was disabled by default. (Houston Putman)
+
================== 9.3.0 ==================
Upgrade Notes
diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
index bcd76b1d1cb..f5ba560e8dc 100644
--- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java
@@ -516,41 +516,9 @@ public class RunExampleTool extends ToolBase {
// create the collection
String collectionName = createCloudExampleCollection(numNodes, readInput, prompt, solrUrl);
- // update the config to enable soft auto-commit
- echo("\nEnabling auto soft-commits with maxTime 3 secs using the Config API");
- setCollectionConfigProperty(solrUrl, collectionName);
-
echo("\n\nSolrCloud example running, please visit: " + solrUrl + " \n");
}
- protected void setCollectionConfigProperty(String solrUrl, String collectionName) {
- ConfigTool configTool = new ConfigTool(stdout);
- String[] configArgs =
- new String[] {
- "-collection",
- collectionName,
- "-property",
- "updateHandler.autoSoftCommit.maxTime",
- "-value",
- "3000",
- "-solrUrl",
- solrUrl
- };
-
- // let's not fail if we get this far ... just report error and finish up
- try {
- configTool.runTool(
- SolrCLI.processCommandLineArgs(
- configTool.getName(), configTool.getOptions(), configArgs));
- } catch (Exception exc) {
- CLIO.err(
- "Failed to update '"
- + "updateHandler.autoSoftCommit.maxTime"
- + "' property due to: "
- + exc);
- }
- }
-
/** wait until the number of live nodes == numNodes. */
protected void waitToSeeLiveNodes(String zkHost, int numNodes) {
try (CloudSolrClient cloudClient =
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
index 2b7a74a2f67..29080392e98 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrCloudExampleTest.java
@@ -21,34 +21,23 @@ import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.cli.CommandLine;
-import org.apache.solr.cli.ConfigTool;
import org.apache.solr.cli.CreateCollectionTool;
import org.apache.solr.cli.DeleteTool;
import org.apache.solr.cli.HealthcheckTool;
import org.apache.solr.cli.SolrCLI;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.StreamingUpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.ExternalPaths;
import org.junit.Test;
import org.slf4j.Logger;
@@ -175,9 +164,6 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {
}
assertEquals("*:* found unexpected number of documents", expectedXmlDocCount, numFound);
- log.info("Updating Config for {}", testCollectionName);
- doTestConfigUpdate(testCollectionName, solrUrl);
-
log.info("Running healthcheck for {}", testCollectionName);
doTestHealthcheck(testCollectionName, cloudClient.getClusterStateProvider().getQuorumHosts());
@@ -212,101 +198,4 @@ public class SolrCloudExampleTest extends AbstractFullDistribZkTestBase {
SolrCLI.safeCheckCollectionExists(
solrUrl, testCollectionName)); // it should not exist anymore
}
-
- /**
- * Uses the SolrCLI config action to activate soft auto-commits for the getting started
- * collection.
- */
- protected void doTestConfigUpdate(String testCollectionName, String solrUrl) throws Exception {
-
- try (SolrClient solrClient = SolrCLI.getSolrClient(solrUrl)) {
- NamedList<Object> configJson =
- solrClient.request(
- new GenericSolrRequest(SolrRequest.METHOD.GET, "/" + testCollectionName + "/config"));
- Object maxTimeFromConfig =
- configJson._get("/config/updateHandler/autoSoftCommit/maxTime", Collections.emptyMap());
- assertNotNull(maxTimeFromConfig);
- assertEquals(-1, maxTimeFromConfig);
-
- String prop = "updateHandler.autoSoftCommit.maxTime";
- Integer maxTime = 3000;
- String[] args =
- new String[] {
- "-name", testCollectionName,
- "-property", prop,
- "-value", maxTime.toString(),
- "-solrUrl", solrUrl
- };
-
- Map<String, Integer> startTimes = getSoftAutocommitInterval(testCollectionName, solrClient);
-
- ConfigTool tool = new ConfigTool();
- CommandLine cli = SolrCLI.processCommandLineArgs(tool.getName(), tool.getOptions(), args);
- log.info("Sending set-property '{}'={} to SolrCLI.ConfigTool.", prop, maxTime);
- assertEquals("Set config property failed!", 0, tool.runTool(cli));
-
- configJson =
- solrClient.request(
- new GenericSolrRequest(SolrRequest.METHOD.GET, "/" + testCollectionName + "/config"));
- maxTimeFromConfig =
- configJson._get("/config/updateHandler/autoSoftCommit/maxTime", Collections.emptyMap());
- assertNotNull(maxTimeFromConfig);
- assertEquals(maxTime, maxTimeFromConfig);
-
- if (log.isInfoEnabled()) {
- log.info("live_nodes_count : {}", cloudClient.getClusterState().getLiveNodes());
- }
-
- // Need to use the _get(List, Object) here because of /query in the path
- assertEquals(
- "Should have been able to get a value from the /query request handler",
- "explicit",
- configJson._get(
- Arrays.asList("config", "requestHandler", "/query", "defaults", "echoParams"),
- Collections.emptyMap()));
-
- // Since it takes some time for this command to complete we need to make sure all the reloads
- // for all the cores have been done.
- boolean allGood = false;
- Map<String, Integer> curSoftCommitInterval = null;
- for (int idx = 0; idx < 600 && !allGood; ++idx) {
- curSoftCommitInterval = getSoftAutocommitInterval(testCollectionName, solrClient);
- // no point in even trying if they're not the same size!
- if (curSoftCommitInterval.size() > 0 && curSoftCommitInterval.size() == startTimes.size()) {
- allGood = true;
- for (Map.Entry<String, Integer> currEntry : curSoftCommitInterval.entrySet()) {
- if (!currEntry.getValue().equals(maxTime)) {
- allGood = false;
- }
- }
- }
- if (!allGood) {
- Thread.sleep(100);
- }
- }
- assertTrue("All cores should have been reloaded within 60 seconds!!!", allGood);
- }
- }
-
- // Collect all the autoSoftCommit intervals.
- private Map<String, Integer> getSoftAutocommitInterval(String collection, SolrClient solrClient)
- throws Exception {
- Map<String, Integer> ret = new HashMap<>();
- DocCollection coll = cloudClient.getClusterState().getCollection(collection);
- for (Slice slice : coll.getActiveSlices()) {
- for (Replica replica : slice.getReplicas()) {
- NamedList<Object> configJson =
- solrClient.request(
- new GenericSolrRequest(
- SolrRequest.METHOD.GET,
- "/" + replica.get(ZkStateReader.CORE_NAME_PROP) + "/config"));
- Integer maxTime =
- (Integer)
- configJson._get(
- "/config/updateHandler/autoSoftCommit/maxTime", Collections.emptyMap());
- ret.put(replica.getCoreName(), maxTime);
- }
- }
- return ret;
- }
}
diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml
index b56305e6529..8023d7bb98f 100644
--- a/solr/server/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml
@@ -301,7 +301,7 @@
-->
<autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
+ <maxTime>${solr.autoSoftCommit.maxTime:3000}</maxTime>
</autoSoftCommit>
<!-- Update Related Event Listeners
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
index b553b37ea83..bd61f96dc82 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -313,7 +313,7 @@
-->
<autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
+ <maxTime>${solr.autoSoftCommit.maxTime:3000}</maxTime>
</autoSoftCommit>
<!-- Update Related Event Listeners
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
index 23b2955d3b6..424591309de 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/commits-transaction-logs.adoc
@@ -144,7 +144,7 @@ You can also specify 'soft' autoCommits with the `autoSoftCommit` tag.
[source,xml]
----
<autoSoftCommit>
- <maxTime>60000</maxTime>
+ <maxTime>5000</maxTime>
</autoSoftCommit>
----
@@ -154,7 +154,7 @@ Determining the best `autoCommit` settings is a tradeoff between performance and
Settings that cause frequent updates will improve the accuracy of searches because new content will be searchable more quickly, but performance may suffer because of the frequent updates.
Less frequent updates may improve performance but it will take longer for updates to show up in queries.
-Here is an example NRT configuration for the two flavors of commit, a hard commit every 60 seconds and a soft commit every 30 seconds.
+Here is an example NRT configuration for the two flavors of commit, a hard commit every 60 seconds and a soft commit every 10 seconds.
Note that these are _not_ the values in the examples shipped with Solr!
[source,xml]
@@ -165,7 +165,7 @@ Note that these are _not_ the values in the examples shipped with Solr!
</autoCommit>
<autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:30000}</maxTime>
+ <maxTime>${solr.autoSoftCommit.maxTime:10000}</maxTime>
</autoSoftCommit>
----
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/taking-solr-to-production.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/taking-solr-to-production.adoc
index 48fe805ac29..7cd071695b9 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/taking-solr-to-production.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/taking-solr-to-production.adoc
@@ -337,7 +337,7 @@ For instance, in `solrconfig.xml`, the default auto soft commit settings are set
[source,xml]
----
<autoSoftCommit>
- <maxTime>${solr.autoSoftCommit.maxTime:-1}</maxTime>
+ <maxTime>${solr.autoSoftCommit.maxTime:3000}</maxTime>
</autoSoftCommit>
----
diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc
index 42ef7c3a5be..874acf5fdf4 100644
--- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc
+++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc
@@ -137,36 +137,7 @@ At this point, Solr will create the collection and again output to the screen th
[,console]
----
-Uploading /solr-{solr-full-version}/server/solr/configsets/_default/conf for config techproducts to ZooKeeper at localhost:9983
-
-Connecting to ZooKeeper at localhost:9983 ...
-INFO - 2017-07-27 12:48:59.289; org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider; Cluster at localhost:9983 ready
-Uploading /solr-{solr-full-version}/server/solr/configsets/sample_techproducts_configs/conf for config techproducts to ZooKeeper at localhost:9983
-
-Creating new collection 'techproducts' using command:
-http://localhost:8983/solr/admin/collections?action=CREATE&name=techproducts&numShards=2&replicationFactor=2&collection.configName=techproducts
-
-{
- "responseHeader":{
- "status":0,
- "QTime":5460},
- "success":{
- "192.168.0.110:7574_solr":{
- "responseHeader":{
- "status":0,
- "QTime":4056},
- "core":"techproducts_shard1_replica_n1"},
- "192.168.0.110:8983_solr":{
- "responseHeader":{
- "status":0,
- "QTime":4056},
- "core":"techproducts_shard2_replica_n2"}}}
-
-Enabling auto soft-commits with maxTime 3 secs using the Config API
-
-POSTing request to Config API: http://localhost:8983/solr/techproducts/config
-{"set-property":{"updateHandler.autoSoftCommit.maxTime":"3000"}}
-Successfully set-property updateHandler.autoSoftCommit.maxTime to 3000
+Created collection 'techproducts' with 2 shard(s), 2 replica(s) with config-set 'techproducts'
SolrCloud example running, please visit: http://localhost:8983/solr
----
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index dd413832d54..c380fa78048 100644
--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -64,6 +64,15 @@ It is always strongly recommended that you fully reindex your documents after a
In Solr 8, it was possible to add docValues to a schema without re-indexing via `UninvertDocValuesMergePolicy`, an advanced/expert utility.
Due to changes in Lucene 9, that isn't possible any more.
+== Solr 9.4
+=== The Built-In Config Sets
+* The build in ConfigSets (`_default` and `sample_techproducts_configs`), now use a default `autoSoftCommit` time of 3 seconds,
+meaning that documents will be searchable within 3 seconds of uploading them.
+Please refer to the xref:configuration-guide:commits-transaction-logs.html#hard-commits-vs-soft-commits[Soft Commit documentation]
+for more information on how this change will effect you.
+Upgrading existing clouds and use-cases that have custom configSets will not be affected by this change.
+
+
== Solr 9.3
=== Binary Releases
* Solr now comes with two xref:deployment-guide:installing-solr.adoc#available-solr-packages[binary releases] and xref:deployment-guide:solr-in-docker.adoc#available-images[docker images], **full** and **slim**.