You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2019/05/03 19:00:33 UTC
[lucene-solr] branch branch_8_1 updated: SOLR-13410: Designated
overseer wasn't able to rejoin election queue upon restart
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch branch_8_1
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8_1 by this push:
new 85b4f26 SOLR-13410: Designated overseer wasn't able to rejoin election queue upon restart
85b4f26 is described below
commit 85b4f2634bd2a04fe2771b7f6cafd87296b0100e
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Sat May 4 00:29:07 2019 +0530
SOLR-13410: Designated overseer wasn't able to rejoin election queue upon restart
---
solr/CHANGES.txt | 3 ++
.../java/org/apache/solr/cloud/ZkController.java | 14 ++---
.../org/apache/solr/cloud/OverseerRolesTest.java | 61 ++++++++++++++++++----
3 files changed, 62 insertions(+), 16 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d9ea0cc..3c2e479 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -201,6 +201,9 @@ Bug Fixes
* SOLR-12291: prematurely reporting not yet finished async Collections API call as completed
when collection's replicas are collocated at least at one node (Varun Thacker, Mikhail Khludnev)
+* SOLR-13410: Designated overseer wasn't able to rejoin election queue upon restart (Ishan Chattopadhyaya,
+ Kesharee Nandan Vishwakarma)
+
Improvements
----------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index 9225472..285e265 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -2143,13 +2143,9 @@ public class ZkController implements Closeable {
public void rejoinOverseerElection(String electionNode, boolean joinAtHead) {
try {
if (electionNode != null) {
- //this call is from inside the JVM . not from CoreAdminHandler
- if (overseerElector.getContext() == null || overseerElector.getContext().leaderSeqPath == null) {
- overseerElector.retryElection(new OverseerElectionContext(zkClient,
- overseer, getNodeName()), joinAtHead);
- return;
- }
- if (!overseerElector.getContext().leaderSeqPath.endsWith(electionNode)) {
+ // Check whether we came to this node by mistake
+ if ( overseerElector.getContext() != null && overseerElector.getContext().leaderSeqPath == null
+ && !overseerElector.getContext().leaderSeqPath.endsWith(electionNode)) {
log.warn("Asked to rejoin with wrong election node : {}, current node is {}", electionNode, overseerElector.getContext().leaderSeqPath);
//however delete it . This is possible when the last attempt at deleting the election node failed.
if (electionNode.startsWith(getNodeName())) {
@@ -2163,6 +2159,10 @@ public class ZkController implements Closeable {
log.warn("Old election node exists , could not be removed ", e);
}
}
+ } else { // We're in the right place, now attempt to rejoin
+ overseerElector.retryElection(new OverseerElectionContext(zkClient,
+ overseer, getNodeName()), joinAtHead);
+ return;
}
} else {
overseerElector.retryElection(overseerElector.getContext(), joinAtHead);
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 895d81b..64a4d01 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -46,25 +46,33 @@ public class OverseerRolesTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
- configureCluster(4)
+ configureCluster(6)
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
- private void waitForNewOverseer(int seconds, Predicate<String> state) throws Exception {
+
+ private void waitForNewOverseer(int seconds, Predicate<String> state, boolean failOnIntermediateTransition) throws Exception {
TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS, TimeSource.NANO_TIME);
String current = null;
while (timeout.hasTimedOut() == false) {
+ String prev = current;
current = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
if (state.test(current))
return;
+ else if (failOnIntermediateTransition) {
+ if (prev != null && !current.equals(prev)) {
+ fail ("There was an intermediate transition, previous: "+prev+", intermediate transition: "+current);
+ }
+ }
Thread.sleep(100);
}
- fail("Timed out waiting for overseer state change");
+ fail("Timed out waiting for overseer state change. The current overseer is: "+current);
}
- private void waitForNewOverseer(int seconds, String expected) throws Exception {
- waitForNewOverseer(seconds, s -> Objects.equals(s, expected));
+ private void waitForNewOverseer(int seconds, String expected, boolean failOnIntermediateTransition) throws Exception {
+ log.info("Expecting node: "+expected);
+ waitForNewOverseer(seconds, s -> Objects.equals(s, expected), failOnIntermediateTransition);
}
private JettySolrRunner getOverseerJetty() throws Exception {
@@ -93,6 +101,10 @@ public class OverseerRolesTest extends SolrCloudTestCase {
logOverseerState();
List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
+ // Remove the OVERSEER role, in case it was already assigned by another test in this suite
+ for (String node: nodes) {
+ CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
+ }
String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
nodes.remove(overseer1);
@@ -102,7 +114,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
CollectionAdminRequest.addRole(overseer2, "overseer").process(cluster.getSolrClient());
- waitForNewOverseer(15, overseer2);
+ waitForNewOverseer(15, overseer2, false);
//add another node as overseer
nodes.remove(overseer2);
@@ -117,7 +129,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
logOverseerState();
leaderJetty.stop();
- waitForNewOverseer(10, overseer3);
+ waitForNewOverseer(10, overseer3, false);
// add another node as overseer
nodes.remove(overseer3);
@@ -129,7 +141,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
// remove the overseer role from the current overseer
CollectionAdminRequest.removeRole(overseer3, "overseer").process(cluster.getSolrClient());
- waitForNewOverseer(15, overseer4);
+ waitForNewOverseer(15, overseer4, false);
// Add it back again - we now have two delegates, 4 and 3
CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());
@@ -142,7 +154,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
.offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
"id", leaderId)));
- waitForNewOverseer(15, s -> Objects.equals(leader, s) == false);
+ waitForNewOverseer(15, s -> Objects.equals(leader, s) == false, false);
Thread.sleep(1000);
@@ -150,6 +162,37 @@ public class OverseerRolesTest extends SolrCloudTestCase {
assertTrue("The old leader should have rejoined election",
OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()).contains(leader));
+ leaderJetty.start(); // starting this back, just for good measure
}
+ @Test
+ public void testDesignatedOverseerRestarts() throws Exception {
+ logOverseerState();
+ List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
+ // Remove the OVERSEER role, in case it was already assigned by another test in this suite
+ for (String node: nodes) {
+ CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
+ }
+ String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ nodes.remove(overseer1);
+
+ // Setting overseer role to the current overseer
+ CollectionAdminRequest.addRole(overseer1, "overseer").process(cluster.getSolrClient());
+ waitForNewOverseer(15, overseer1, false);
+
+ // kill the current overseer, and check that the next node in the election queue assumes leadership
+ JettySolrRunner leaderJetty = getOverseerJetty();
+ logOverseerState();
+ leaderJetty.stop();
+ log.info("Killing designated overseer: "+overseer1);
+ waitForNewOverseer(10, nodes.get(0), false);
+
+ // after 5 seconds, bring back dead designated overseer and assert that it assumes leadership "right away",
+ // i.e. without any other node assuming leadership before this node becomes leader.
+ Thread.sleep(5);
+ logOverseerState();
+ log.info("Starting back the prioritized overseer..");
+ leaderJetty.start();
+ waitForNewOverseer(1500000, overseer1, true); // assert that there is just a single leadership transition
+ }
}