You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by js...@apache.org on 2023/03/11 17:19:12 UTC
[solr] branch branch_9x updated: SOLR-16487: Improve Pull Replica Interval (#1437)
This is an automated email from the ASF dual-hosted git repository.
jsweeney pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new a9421cc0932 SOLR-16487: Improve Pull Replica Interval (#1437)
a9421cc0932 is described below
commit a9421cc0932281230a4029ba7fb90e7a057f4465
Author: Justin Sweeney <ju...@fullstory.com>
AuthorDate: Sat Mar 11 10:17:31 2023 -0700
SOLR-16487: Improve Pull Replica Interval (#1437)
* Updating replication polling to factor in if a new searcher is opened or not as well as adding a test for poll interval logic
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/ReplicateFromLeader.java | 46 ++++++++++++--
.../apache/solr/cloud/ReplicateFromLeaderTest.java | 71 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 4 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 52e302b59a6..7b96b468d6c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -161,6 +161,8 @@ Optimizations
* SOLR-16615: Jersey 'ApplicationHandlers' are now shared by compatible cores where possible (Jason Gerlowski, Houston Putman)
+* SOLR-16487: Replication pooling is optimized based on hard and soft commit settings for a given collection (Justin Sweeney)
+
Bug Fixes
---------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
index 2755102ad21..7abe46fe0c3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ReplicateFromLeader.java
@@ -80,10 +80,10 @@ public class ReplicateFromLeader {
if (System.getProperty("jetty.testMode") != null) {
pollIntervalStr = "00:00:01";
}
- if (uinfo.autoCommmitMaxTime != -1) {
- pollIntervalStr = toPollIntervalStr(uinfo.autoCommmitMaxTime / 2);
- } else if (uinfo.autoSoftCommmitMaxTime != -1) {
- pollIntervalStr = toPollIntervalStr(uinfo.autoSoftCommmitMaxTime / 2);
+
+ String calculatedPollIntervalString = determinePollInterval(uinfo);
+ if (calculatedPollIntervalString != null) {
+ pollIntervalStr = calculatedPollIntervalString;
}
log.info("Will start replication from leader with poll interval: {}", pollIntervalStr);
@@ -155,6 +155,44 @@ public class ReplicateFromLeader {
}
}
+ /**
+ * Determine the poll interval for replicas based on the auto soft/hard commit schedule
+ *
+ * @param uinfo the update handler info containing soft/hard commit configuration
+ * @return a poll interval string representing a cadence of polling frequency in the form of
+ * hh:mm:ss
+ */
+ public static String determinePollInterval(SolrConfig.UpdateHandlerInfo uinfo) {
+ int hardCommitMaxTime = uinfo.autoCommmitMaxTime;
+ int softCommitMaxTime = uinfo.autoSoftCommmitMaxTime;
+ boolean hardCommitNewSearcher = uinfo.openSearcher;
+ String pollIntervalStr = null;
+ if (hardCommitMaxTime != -1) {
+ // configured hardCommit places a ceiling on the interval at which new segments will be
+ // available to replicate
+ if (softCommitMaxTime != -1
+ && (!hardCommitNewSearcher || softCommitMaxTime <= hardCommitMaxTime)) {
+ /*
+ * softCommit is configured.
+ * Usually if softCommit is configured, `hardCommitNewSearcher==false`,
+ * in which case you want to calculate poll interval wrt the max of hardCommitTime
+ * (when segments are available to replicate) and softCommitTime (when changes are visible).
+ * But in the unusual case that hardCommit _does_ open a new searcher and
+ * `hardCommitMaxTime < softCommitMaxTime`, then fallback to `else` clause,
+ * setting poll interval wrt `hardCommitMaxTime` alone.
+ */
+ pollIntervalStr = toPollIntervalStr(Math.max(hardCommitMaxTime, softCommitMaxTime) / 2);
+ } else {
+ pollIntervalStr = toPollIntervalStr(hardCommitMaxTime / 2);
+ }
+ } else if (softCommitMaxTime != -1) {
+ // visibility of changes places a ceiling on polling frequency
+ pollIntervalStr = toPollIntervalStr(softCommitMaxTime / 2);
+ }
+
+ return pollIntervalStr;
+ }
+
private static String toPollIntervalStr(int ms) {
int sec = ms / 1000;
int hour = sec / 3600;
diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
new file mode 100644
index 00000000000..723eda5874d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/ReplicateFromLeaderTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.solr.core.SolrConfig;
+import org.junit.Test;
+
+public class ReplicateFromLeaderTest {
+
+ @Test
+ public void determinePollIntervalString() {
+ SolrConfig.UpdateHandlerInfo updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 15000, -1, true, -1, 60000, false);
+ String pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:7", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, 15000, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:30", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, 60000, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:30", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 60000, -1, false, -1, 15000, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:30", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, -1, -1, false, -1, 60000, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:30", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 15000, -1, false, -1, -1, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:7", pollInterval);
+
+ updateHandlerInfo =
+ new SolrConfig.UpdateHandlerInfo(
+ "solr.DirectUpdateHandler2", -1, 60000, -1, true, -1, -1, false);
+ pollInterval = ReplicateFromLeader.determinePollInterval(updateHandlerInfo);
+ assertEquals("0:0:30", pollInterval);
+ }
+}