You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2020/07/10 08:07:56 UTC
[lucene-solr] 01/01: SOLR-14641: PeerSync,
remove canHandleVersionRanges check
This is an automated email from the ASF dual-hosted git repository.
datcm pushed a commit to branch jira/SOLR-14641
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 9562dd887a31102f86f710fd99ea53b019a85fcf
Author: Cao Manh Dat <da...@apache.org>
AuthorDate: Fri Jul 10 15:07:33 2020 +0700
SOLR-14641: PeerSync, remove canHandleVersionRanges check
---
.../src/java/org/apache/solr/core/SolrConfig.java | 10 +---
.../handler/component/RealTimeGetComponent.java | 12 +---
.../src/java/org/apache/solr/update/PeerSync.java | 68 +---------------------
.../org/apache/solr/update/PeerSyncWithLeader.java | 24 +-------
.../resources/EditableSolrConfigAttributes.json | 3 -
.../solr/collection1/conf/solrconfig-tlog.xml | 4 --
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 2 -
7 files changed, 9 insertions(+), 114 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
index 2daaa95..4a759bb 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java
@@ -230,8 +230,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
validateMemoryBreakerThreshold();
- useRangeVersionsForPeerSync = getBool("peerSync/useRangeVersions", true);
-
filterCacheConfig = CacheConfig.getConfig(this, "query/filterCache");
queryResultCacheConfig = CacheConfig.getConfig(this, "query/queryResultCache");
documentCacheConfig = CacheConfig.getConfig(this, "query/documentCache");
@@ -531,9 +529,7 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
// Circuit Breaker Configuration
public final boolean useCircuitBreakers;
public final int memoryCircuitBreakerThresholdPct;
-
- public final boolean useRangeVersionsForPeerSync;
-
+
// IndexConfig settings
public final SolrIndexConfig indexConfig;
@@ -930,10 +926,6 @@ public class SolrConfig extends XmlConfigFile implements MapSerializable {
"addHttpRequestToContext", addHttpRequestToContext));
if (indexConfig != null) result.put("indexConfig", indexConfig);
- m = new LinkedHashMap();
- result.put("peerSync", m);
- m.put("useRangeVersions", useRangeVersionsForPeerSync);
-
//TODO there is more to add
return result;
diff --git a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
index 093c419..48b082c 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java
@@ -136,16 +136,8 @@ public class RealTimeGetComponent extends SearchComponent
if (!params.getBool(COMPONENT_NAME, true)) {
return;
}
-
- // This seems rather kludgey, may there is better way to indicate
- // that replica can support handling version ranges
- String val = params.get("checkCanHandleVersionRanges");
- if(val != null) {
- rb.rsp.add("canHandleVersionRanges", true);
- return;
- }
-
- val = params.get("getFingerprint");
+
+ String val = params.get("getFingerprint");
if(val != null) {
processGetFingeprint(rb);
return;
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index 09d2ce0..c23bfb1 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -22,17 +22,13 @@ import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
import java.util.Optional;
-import java.util.Set;
-import java.util.function.Supplier;
import java.util.stream.Collectors;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Timer;
import org.apache.http.NoHttpResponseException;
-import org.apache.http.client.HttpClient;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.client.solrj.SolrServerException;
@@ -41,7 +37,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.IOUtils;
-import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.handler.component.ShardHandler;
@@ -84,7 +79,6 @@ public class PeerSync implements SolrMetricProducer {
private final boolean cantReachIsSuccess;
private final boolean doFingerprint;
- private final HttpClient client;
private final boolean onlyIfActive;
private SolrCore core;
private Updater updater;
@@ -117,7 +111,6 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
this.cantReachIsSuccess = cantReachIsSuccess;
this.doFingerprint = doFingerprint && !("true".equals(System.getProperty("solr.disableFingerprint")));
- this.client = core.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient();
this.onlyIfActive = onlyIfActive;
uhandler = core.getUpdateHandler();
@@ -406,31 +399,6 @@ public class PeerSync implements SolrMetricProducer {
}
}
- private boolean canHandleVersionRanges(String replica) {
- SyncShardRequest sreq = new SyncShardRequest();
- requests.add(sreq);
-
- // determine if leader can handle version ranges
- sreq.shards = new String[] {replica};
- sreq.actualShards = sreq.shards;
- sreq.params = new ModifiableSolrParams();
- sreq.params.set("qt", "/get");
- sreq.params.set(DISTRIB, false);
- sreq.params.set("checkCanHandleVersionRanges", false);
-
- ShardHandler sh = shardHandlerFactory.getShardHandler();
- sh.submit(sreq, replica, sreq.params);
-
- ShardResponse srsp = sh.takeCompletedIncludingErrors();
- Boolean canHandleVersionRanges = srsp.getSolrResponse().getResponse().getBooleanArg("canHandleVersionRanges");
-
- if (canHandleVersionRanges == null || canHandleVersionRanges.booleanValue() == false) {
- return false;
- }
-
- return true;
- }
-
private boolean handleVersions(ShardResponse srsp) {
// we retrieved the last N updates from the replica
@SuppressWarnings({"unchecked"})
@@ -453,8 +421,7 @@ public class PeerSync implements SolrMetricProducer {
}
MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(
- otherVersions, sreq.shards[0],
- () -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges(sreq.shards[0]));
+ otherVersions, sreq.shards[0]);
if (updatesRequest == MissedUpdatesRequest.ALREADY_IN_SYNC) {
return true;
@@ -717,16 +684,12 @@ public class PeerSync implements SolrMetricProducer {
}
static abstract class MissedUpdatesFinderBase {
- private Set<Long> ourUpdateSet;
- private Set<Long> requestedUpdateSet = new HashSet<>();
-
long ourLowThreshold; // 20th percentile
List<Long> ourUpdates;
MissedUpdatesFinderBase(List<Long> ourUpdates, long ourLowThreshold) {
assert sorted(ourUpdates);
this.ourUpdates = ourUpdates;
- this.ourUpdateSet = new HashSet<>(ourUpdates);
this.ourLowThreshold = ourLowThreshold;
}
@@ -783,26 +746,6 @@ public class PeerSync implements SolrMetricProducer {
String rangesToRequestStr = rangesToRequest.stream().collect(Collectors.joining(","));
return MissedUpdatesRequest.of(rangesToRequestStr, totalRequestedVersions);
}
-
- MissedUpdatesRequest handleIndividualVersions(List<Long> otherVersions, boolean completeList) {
- List<Long> toRequest = new ArrayList<>();
- for (Long otherVersion : otherVersions) {
- // stop when the entries get old enough that reorders may lead us to see updates we don't need
- if (!completeList && Math.abs(otherVersion) < ourLowThreshold) break;
-
- if (ourUpdateSet.contains(otherVersion) || requestedUpdateSet.contains(otherVersion)) {
- // we either have this update, or already requested it
- // TODO: what if the shard we previously requested this from returns failure (because it goes
- // down)
- continue;
- }
-
- toRequest.add(otherVersion);
- requestedUpdateSet.add(otherVersion);
- }
-
- return MissedUpdatesRequest.of(StrUtils.join(toRequest, ','), toRequest.size());
- }
}
/**
@@ -824,7 +767,7 @@ public class PeerSync implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
- public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
+ public MissedUpdatesRequest find(List<Long> otherVersions, Object updateFrom) {
otherVersions.sort(absComparator);
if (debug) {
log.debug("{} sorted versions from {} = {}", logPrefix, otherVersions, updateFrom);
@@ -858,12 +801,7 @@ public class PeerSync implements SolrMetricProducer {
boolean completeList = otherVersions.size() < nUpdates;
- MissedUpdatesRequest updatesRequest;
- if (canHandleVersionRanges.get()) {
- updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
- } else {
- updatesRequest = handleIndividualVersions(otherVersions, completeList);
- }
+ MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(otherVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
index 5e81b9d..c448d42 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
@@ -238,7 +238,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return MissedUpdatesRequest.UNABLE_TO_SYNC;
}
- MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl, () -> core.getSolrConfig().useRangeVersionsForPeerSync && canHandleVersionRanges());
+ MissedUpdatesRequest updatesRequest = missedUpdatesFinder.find(otherVersions, leaderUrl);
if (updatesRequest == MissedUpdatesRequest.EMPTY) {
if (doFingerprint) return MissedUpdatesRequest.UNABLE_TO_SYNC;
return MissedUpdatesRequest.ALREADY_IN_SYNC;
@@ -313,19 +313,6 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
return true;
}
- // determine if leader can handle version ranges
- private boolean canHandleVersionRanges() {
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.set("qt", "/get");
- params.set(DISTRIB, false);
- params.set("checkCanHandleVersionRanges", false);
-
- NamedList<Object> rsp = request(params, "Failed on determine if leader can handle version ranges");
- Boolean canHandleVersionRanges = rsp.getBooleanArg("canHandleVersionRanges");
-
- return canHandleVersionRanges != null && canHandleVersionRanges;
- }
-
private NamedList<Object> request(ModifiableSolrParams params, String onFail) {
try {
QueryResponse rsp = new QueryRequest(params, SolrRequest.METHOD.POST).process(clientToLeader);
@@ -404,7 +391,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
this.nUpdates = nUpdates;
}
- public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom, Supplier<Boolean> canHandleVersionRanges) {
+ public MissedUpdatesRequest find(List<Long> leaderVersions, Object updateFrom) {
leaderVersions.sort(absComparator);
log.debug("{} sorted versions from {} = {}", logPrefix, leaderVersions, updateFrom);
@@ -418,12 +405,7 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
// In that case, we will fail on compute fingerprint with the current leader and start segments replication
boolean completeList = leaderVersions.size() < nUpdates;
- MissedUpdatesRequest updatesRequest;
- if (canHandleVersionRanges.get()) {
- updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
- } else {
- updatesRequest = handleIndividualVersions(leaderVersions, completeList);
- }
+ MissedUpdatesRequest updatesRequest = handleVersionsWithRanges(leaderVersions, completeList);
if (updatesRequest.totalRequestedUpdates > nUpdates) {
log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);
diff --git a/solr/core/src/resources/EditableSolrConfigAttributes.json b/solr/core/src/resources/EditableSolrConfigAttributes.json
index c441c30..0ed8333 100644
--- a/solr/core/src/resources/EditableSolrConfigAttributes.json
+++ b/solr/core/src/resources/EditableSolrConfigAttributes.json
@@ -70,7 +70,4 @@
"enableRemoteStreaming":10,
"enableStreamBody":10,
"addHttpRequestToContext":0}},
- "peerSync":{
- "useRangeVersions":11
- }
}
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
index 2fc3c52..f2cad61 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
@@ -49,10 +49,6 @@
<requestHandler name="/select" class="solr.SearchHandler" />
- <peerSync>
- <useRangeVersions>${solr.peerSync.useRangeVersions:true}</useRangeVersions>
- </peerSync>
-
<updateHandler class="solr.DirectUpdateHandler2">
<!-- autocommit pending docs if certain criteria are met -->
<autoCommit>
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index d70013b..09b98e0 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -291,7 +291,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.setProperty("enable.update.log", usually() ? "true" : "false");
System.setProperty("tests.shardhandler.randomSeed", Long.toString(random().nextLong()));
System.setProperty("solr.clustering.enabled", "false");
- System.setProperty("solr.peerSync.useRangeVersions", String.valueOf(random().nextBoolean()));
System.setProperty("solr.cloud.wait-for-updates-with-stale-state-pause", "500");
System.setProperty("pkiHandlerPrivateKeyPath", SolrTestCaseJ4.class.getClassLoader().getResource("cryptokeys/priv_key512_pkcs8.pem").toExternalForm());
@@ -349,7 +348,6 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
System.clearProperty("enable.update.log");
System.clearProperty("useCompoundFile");
System.clearProperty("urlScheme");
- System.clearProperty("solr.peerSync.useRangeVersions");
System.clearProperty("solr.cloud.wait-for-updates-with-stale-state-pause");
System.clearProperty("solr.zkclienttmeout");
System.clearProperty(ZK_WHITELIST_PROPERTY);