You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2020/08/10 08:34:52 UTC
[lucene-solr] branch branch_8x updated: SOLR-14641: PeerSync,
remove canHandleVersionRanges check (#1663)
This is an automated email from the ASF dual-hosted git repository.
datcm pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new d52628d SOLR-14641: PeerSync, remove canHandleVersionRanges check (#1663)
d52628d is described below
commit d52628d9facfc13d8c29a7ecaf646a3b90263f8c
Author: Cao Manh Dat <da...@apache.org>
AuthorDate: Mon Aug 10 15:16:53 2020 +0700
SOLR-14641: PeerSync, remove canHandleVersionRanges check (#1663)
---
.../src/java/org/apache/solr/core/SolrConfig.java | 9 ---
.../handler/component/RealTimeGetComponent.java | 8 ++-
.../src/java/org/apache/solr/update/PeerSync.java | 69 +---------------------
.../org/apache/solr/update/PeerSyncWithLeader.java | 25 +-------
.../resources/EditableSolrConfigAttributes.json | 3 -
.../solr/collection1/conf/solrconfig-tlog.xml | 4 --
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 2 -
7 files changed, 11 insertions(+), 109 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index a674148..96377cf 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -228,8 +228,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
queryResultMaxDocsCached = getInt("query/queryResultMaxDocsCached", Integer.MAX_VALUE);
enableLazyFieldLoading = getBool("query/enableLazyFieldLoading", false);
- useRangeVersionsForPeerSync = getBool("peerSync/useRangeVersions", true);
-
filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
@@ -525,9 +523,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
public final int queryResultWindowSize;
public final int queryResultMaxDocsCached;
public final boolean enableLazyFieldLoading;
-
- public final boolean useRangeVersionsForPeerSync;
-
// IndexConfig settings
public final SolrIndexConfig indexConfig;
@@ -914,10 +909,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
"addHttpRequestToContext", addHttpRequestToContext));
if (indexConfig != null) result.put("indexConfig", indexConfig);
- m = new LinkedHashMap();
- result.put("peerSync", m);
- m.put("useRangeVersions", useRangeVersionsForPeerSync);
-
//TODO there is more to add
return result;
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 093c419..0f6b567 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -136,9 +136,11 @@ public class RealTimeGetComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
-
- // This seems rather kludgey, may there is better way to indicate
- // that replica can support handling version ranges
+
+ //TODO remove this at Solr 10
+ //After SOLR-14641 other nodes won't call RTG with this param.
+ //Just keeping here for backward-compatibility, if we remove this, nodes with older versions will
+ //assume that this node can't handle version ranges.
String val = params.get("checkCanHandleVersionRanges");
if(val != null) {
rb.rsp.add("canHandleVersionRanges", true);
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index 53e6ec6..b9684bd 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -22,17 +22,13 @@ import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
import java.util.Optional;
-import java.util.Set;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import org.apache.http.NoHttpResponseException;
-import org.apache.http.client.HttpClient;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.client.solrj.SolrServerException;
@@ -41,7 +37,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.handler.component.ShardHandler;
@@ -84,7 +79,6 @@ public class PeerSync implements SolrMetricProducer {
private final boolean cantReachIsSuccess;
private final boolean doFingerprint;
- private final HttpClient client;
private final boolean onlyIfActive;
private SolrCore core;
private Updater updater;
@@ -116,7 +110,6 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
this.cantReachIsSuccess = cantReachIsSuccess;
this.doFingerprint = doFingerprint && !("true".equals(System.getProperty("solr.disableFingerprint")));
- this.client = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
this.onlyIfActive = onlyIfActive;
uhandler = core.getUpdateHandler();
@@ -399,31 +392,6 @@ public class PeerSync implements SolrMetricProducer {
}
}
- private boolean canHandleVersionRanges(String replica) {
- SyncShardRequest sreq = new SyncShardRequest();
- requests.add(sreq);
-
- // determine if leader can handle version ranges
- sreq.shards = new String[] {replica};
- sreq.actualShards = sreq.shards;
- sreq.params = new ModifiableSolrParams();
- sreq.params.set("qt", "/get");
- sreq.params.set(DISTRIB, false);
- sreq.params.set("checkCanHandleVersionRanges", false);
-
- ShardHandler sh = shardHandlerFactory.getShardHandler();
- sh.submit(sreq, replica, sreq.params);
-
- ShardResponse srsp = sh.takeCompletedIncludingErrors();
- Boolean canHandleVersionRanges = srsp.getSolrResponse().getResponse().getBooleanArg("canHandleVersionRanges");
-
- if (canHandleVersionRanges == null || canHandleVersionRanges.booleanValue() == false) {
- return false;
- }
-
- return true;
- }
-
private boolean handleVersions(ShardResponse srsp) {
// we retrieved the last N updates from the replica
@SuppressWarnings({"unchecked"})
@@ -446,8 +414,7 @@ public class PeerSync implements SolrMetricProducer {
}
MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(
- otherVersions, sreq.shards[0],
- () -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges(sreq.shards[0]));
+ otherVersions, sreq.shards[0]);
if (updatesRequest == MissedUpdatesRequest.ALREADY_IN_SYNC) {
return true;
@@ -710,16 +677,12 @@ public class PeerSync implements SolrMetricProducer {
}
static abstract class MissedUpdatesFinderBase {
- private Set<Long> ourUpdateSet;
- private Set<Long> requestedUpdateSet = new HashSet<>();
-
long ourLowThreshold; // 20th percentile
List<Long> ourUpdates;
MissedUpdatesFinderBase(List<Long> ourUpdates, long ourLowThreshold) {
assert sorted(ourUpdates);
this.ourUpdates = ourUpdates;
- this.ourUpdateSet = new HashSet<>(ourUpdates);
this.ourLowThreshold = ourLowThreshold;
}
@@ -776,26 +739,6 @@ public class PeerSync implements SolrMetricProducer {
String rangesToRequestStr = rangesToRequest.stream().collect(Collectors.joining(","));
return MissedUpdatesRequest.of(rangesToRequestStr, totalRequestedVersions);
}
-
- MissedUpdatesRequest handleIndividualVersions(List<Long> otherVersions, boolean completeList) {
- List<Long> toRequest = new ArrayList<>();
- for (Long otherVersion : otherVersions) {
- // stop when the entries get old enough that reorders may lead us to see updates we don't need
- if (!completeList && Math.abs(otherVersion) < ourLowThreshold) break;
-
- if (ourUpdateSet.contains(otherVersion) || requestedUpdateSet.contains(otherVersion)) {
- // we either have this update, or already requested it
- // TODO: what if the shard we previously requested this from returns failure (because it goes
- // down)
- continue;
- }
-
- toRequest.add(otherVersion);
- requestedUpdateSet.add(otherVersion);
- }
-
- return MissedUpdatesRequest.of(StrUtils.join(toRequest, ','), toRequest.size());
- }
}
/**
@@ -817,7 +760,7 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
- public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
+ public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom) {
otherVersions.sort(absComparator);
if (debug) {
log.debug("{} sorted versions from {} = {}", logPrefix, otherVersions, updateFrom);
@@ -851,13 +794,7 @@ public class PeerSync implements SolrMetricProducer {
boolean completeList = otherVersions.size() < nUpdates;
- MissedUpdatesRequest updatesRequest;
- if (canHandleVersionRanges.get()) {
- updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
- } else {
- updatesRequest = handleIndividualVersions(otherVersions, completeList);
- }
-
+ MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);
return MissedUpdatesRequest.UNABLE_TO_SYNC;
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
index 751aa17..e62075e 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Set;
-import java.util.function.Supplier;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
@@ -231,7 +230,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return MissedUpdatesRequest.UNABLE_TO_SYNC;
}
- MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl, () -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges());
+ MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl);
if (updatesRequest == MissedUpdatesRequest.EMPTY) {
if (doFingerprint) return MissedUpdatesRequest.UNABLE_TO_SYNC;
return MissedUpdatesRequest.ALREADY_IN_SYNC;
@@ -306,19 +305,6 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return true;
}
- // determine if leader can handle version ranges
- private boolean canHandleVersionRanges() {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("qt", "/get");
- params.set(DISTRIB, false);
- params.set("checkCanHandleVersionRanges", false);
-
- NamedList<Object> rsp = request(params, "Failed on determine if leader can handle version ranges");
- Boolean canHandleVersionRanges = rsp.getBooleanArg("canHandleVersionRanges");
-
- return canHandleVersionRanges != null && canHandleVersionRanges;
- }
-
private NamedList<Object> request(ModifiableSolrParams params, String onFail) {
try {
QueryResponse rsp = new QueryRequest(params, SolrRequest.METHOD.POST).process(clientToLeader);
@@ -397,7 +383,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
- public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
+ public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom) {
leaderVersions.sort(absComparator);
log.debug("{} sorted versions from {} = {}", logPrefix, leaderVersions, updateFrom);
@@ -411,12 +397,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
// In that case, we will fail on compute fingerprint with the current leader and start segments replication
boolean completeList = leaderVersions.size() < nUpdates;
- MissedUpdatesRequest updatesRequest;
- if (canHandleVersionRanges.get()) {
- updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
- } else {
- updatesRequest = handleIndividualVersions(leaderVersions, completeList);
- }
+ MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);
diff --git a/solr/core/src/resources/EditableSolrConfigAttributes.json b/solr/core/src/resources/EditableSolrConfigAttributes.json
index ed61e1f..03bb1b6 100644
--- a/solr/core/src/resources/EditableSolrConfigAttributes.json
+++ b/solr/core/src/resources/EditableSolrConfigAttributes.json
@@ -68,7 +68,4 @@
"enableRemoteStreaming":10,
"enableStreamBody":10,
"addHttpRequestToContext":0}},
- "peerSync":{
- "useRangeVersions":11
- }
}
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
index 989023d..b5e542d 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
@@ -49,10 +49,6 @@
<requestHandler name="/select" class="solr.SearchHandler" />
- <peerSync>
- <useRangeVersions>${solr.peerSync.useRangeVersions:true}</useRangeVersions>
- </peerSync>
-
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met -->
<autoCommit>
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index b24c04b..a3f325b 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -287,7 +287,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.setProperty("enable.update.log", usually() ? "true" : "false");
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
System.setProperty("solr.clustering.enabled", "false");
- System.setProperty("solr.peerSync.useRangeVersions", String.valueOf(random().nextBoolean()));
System.setProperty("solr.cloud.wait-for-updates-with-stale-state-pause", "500");
System.setProperty(ZK_WHITELIST_PROPERTY, "*");
startTrackingSearchers();
@@ -341,7 +340,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.clearProperty("enable.update.log");
System.clearProperty("useCompoundFile");
System.clearProperty("urlScheme");
- System.clearProperty("solr.peerSync.useRangeVersions");
System.clearProperty("solr.cloud.wait-for-updates-with-stale-state-pause");
System.clearProperty("solr.zkclienttmeout");
System.clearProperty(ZK_WHITELIST_PROPERTY);