You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2020/01/16 09:43:46 UTC
[lucene-solr] branch master updated: SOLR-14128: Tentative fix: put
replicas on other nodes than overseer,
wait for all replicas to complete the reload.
This is an automated email from the ASF dual-hosted git repository.
ab pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new 5435054 SOLR-14128: Tentative fix: put replicas on other nodes than overseer, wait for all replicas to complete the reload.
5435054 is described below
commit 543505470c26f1ebb3ecd5ca57c411c03941a6a1
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Thu Jan 16 10:42:26 2020 +0100
SOLR-14128: Tentative fix: put replicas on other nodes than overseer, wait for
all replicas to complete the reload.
---
.../solr/cloud/SystemCollectionCompatTest.java | 48 +++++++++++++++-------
1 file changed, 33 insertions(+), 15 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
index a37c791..3e3ac2a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SystemCollectionCompatTest.java
@@ -20,11 +20,13 @@ package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -36,11 +38,13 @@ import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.RetryUtil;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.logging.LogWatcher;
import org.apache.solr.logging.LogWatcherConfig;
import org.apache.solr.util.IdUtils;
@@ -74,13 +78,21 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
@Before
public void setupSystemCollection() throws Exception {
- CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 2)
- .process(cluster.getSolrClient());
- cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 2);
ZkController zkController = cluster.getJettySolrRunner(0).getCoreContainer().getZkController();
cloudManager = zkController.getSolrCloudManager();
solrClient = new CloudSolrClientBuilder(Collections.singletonList(zkController.getZkServerAddress()),
Optional.empty()).build();
+ CollectionAdminRequest.OverseerStatus status = new CollectionAdminRequest.OverseerStatus();
+ CollectionAdminResponse adminResponse = status.process(solrClient);
+ String overseerLeader = (String) adminResponse.getResponse().get("leader");
+ Set<String> nodes = new HashSet<>(cloudManager.getClusterStateProvider().getLiveNodes());
+ nodes.remove(overseerLeader);
+ // put .system replicas on other nodes that the overseer
+ CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 2)
+ .setCreateNodeSet(String.join(",", nodes))
+ .setMaxShardsPerNode(2)
+ .process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 2);
// send a dummy doc to the .system collection
SolrInputDocument doc = new SolrInputDocument(
"id", IdUtils.timeRandomId(),
@@ -90,9 +102,11 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
solrClient.add(CollectionAdminParams.SYSTEM_COLL, doc);
solrClient.commit(CollectionAdminParams.SYSTEM_COLL);
- Replica leader
- = solrClient.getZkStateReader().getLeaderRetry(CollectionAdminParams.SYSTEM_COLL, "shard1", DEFAULT_TIMEOUT);
- final AtomicReference<Long> coreStartTime = new AtomicReference<>(getCoreStatus(leader).getCoreStartTime().getTime());
+ Map<String, Long> coreStartTimes = new HashMap<>();
+ DocCollection coll = cloudManager.getClusterStateProvider().getCollection(CollectionAdminParams.SYSTEM_COLL);
+ for (Replica r : coll.getReplicas()) {
+ coreStartTimes.put(r.getName(), getCoreStatus(r).getCoreStartTime().getTime());
+ }
// trigger compat report by changing the schema
SchemaRequest req = new SchemaRequest();
SchemaResponse rsp = req.process(solrClient, CollectionAdminParams.SYSTEM_COLL);
@@ -107,16 +121,20 @@ public class SystemCollectionCompatTest extends SolrCloudTestCase {
CollectionAdminResponse response = reloadRequest.process(solrClient);
assertEquals(0, response.getStatus());
assertTrue(response.isSuccess());
- // wait for the reload to complete
+ // wait for the reload of all replicas to complete
RetryUtil.retryUntil("Timed out waiting for core to reload", 30, 1000, TimeUnit.MILLISECONDS, () -> {
- long restartTime = 0;
- try {
- restartTime = getCoreStatus(leader).getCoreStartTime().getTime();
- } catch (Exception e) {
- log.warn("Exception getting core start time: {}", e.getMessage());
- return false;
+ boolean allReloaded = true;
+ for (Replica r : coll.getReplicas()) {
+ long previousTime = coreStartTimes.get(r.getName());
+ try {
+ long currentTime = getCoreStatus(r).getCoreStartTime().getTime();
+ allReloaded = allReloaded && (previousTime < currentTime);
+ } catch (Exception e) {
+ log.warn("Error retrieving replica status of " + Utils.toJSONString(r), e);
+ allReloaded = false;
+ }
}
- return restartTime > coreStartTime.get();
+ return allReloaded;
});
cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 2);