You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2017/01/19 17:46:12 UTC
[1/2] lucene-solr:branch_6x: SOLR-10001: Fix overseer-roles test bug
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 5c5a6db45 -> 9ee34e36d
refs/heads/master a14d79366 -> eba939096
SOLR-10001: Fix overseer-roles test bug
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9ee34e36
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9ee34e36
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9ee34e36
Branch: refs/heads/branch_6x
Commit: 9ee34e36d70c41063bcc46dcc416c94c8ba29486
Parents: 5c5a6db
Author: Alan Woodward <ro...@apache.org>
Authored: Thu Jan 19 17:16:42 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Jan 19 17:17:01 2017 +0000
----------------------------------------------------------------------
.../solr/cloud/CollectionsAPISolrJTest.java | 34 ----
.../apache/solr/cloud/OverseerRolesTest.java | 173 +++++++++----------
2 files changed, 77 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ee34e36/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 616b657..3e0d840 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -21,7 +21,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import org.apache.lucene.util.LuceneTestCase;
@@ -255,39 +254,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
}
@Test
- @SuppressWarnings("unchecked")
- public void testAddAndRemoveRole() throws InterruptedException, IOException, SolrServerException {
-
- String node = cluster.getRandomJetty(random()).getNodeName();
-
- CollectionAdminRequest.addRole(node, "overseer").process(cluster.getSolrClient());
-
- CollectionAdminResponse response = CollectionAdminRequest.getClusterStatus().process(cluster.getSolrClient());
-
- NamedList<Object> rsp = response.getResponse();
- NamedList<Object> cs = (NamedList<Object>) rsp.get("cluster");
- assertNotNull("Cluster state should not be null", cs);
- Map<String, Object> roles = (Map<String, Object>) cs.get("roles");
- assertNotNull("Role information should not be null", roles);
- List<String> overseer = (List<String>) roles.get("overseer");
- assertNotNull(overseer);
- assertEquals(1, overseer.size());
- assertTrue(overseer.contains(node));
-
- // Remove role
- CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
-
- response = CollectionAdminRequest.getClusterStatus().process(cluster.getSolrClient());
- rsp = response.getResponse();
- cs = (NamedList<Object>) rsp.get("cluster");
- assertNotNull("Cluster state should not be null", cs);
- roles = (Map<String, Object>) cs.get("roles");
- assertNotNull("Role information should not be null", roles);
- overseer = (List<String>) roles.get("overseer");
- assertFalse(overseer.contains(node));
- }
-
- @Test
public void testOverseerStatus() throws IOException, SolrServerException {
CollectionAdminResponse response = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9ee34e36/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 762bbeb..3c2ca87 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -17,28 +17,27 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
+import java.net.URL;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.overseer.OverseerAction;
-import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.data.Stat;
-import org.junit.Before;
+import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getLeaderNode;
-import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames;
-import static org.hamcrest.CoreMatchers.not;
+import static org.apache.solr.cloud.OverseerTaskProcessor.getSortedElectionNodes;
public class OverseerRolesTest extends SolrCloudTestCase {
@@ -51,117 +50,99 @@ public class OverseerRolesTest extends SolrCloudTestCase {
.configure();
}
- @Before
- public void clearAllOverseerRoles() throws Exception {
- for (String node : OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient())) {
- CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
+ private void waitForNewOverseer(int seconds, Predicate<String> state) throws Exception {
+ TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS);
+ String current = null;
+ while (timeout.hasTimedOut() == false) {
+ current = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ if (state.test(current))
+ return;
+ Thread.sleep(100);
}
+ fail("Timed out waiting for overseer state change");
}
- @Test
- public void testQuitCommand() throws Exception {
-
- SolrZkClient zk = zkClient();
- byte[] data = zk.getData("/overseer_elect/leader", null, new Stat(), true);
- Map m = (Map) Utils.fromJSON(data);
- String s = (String) m.get("id");
- String leader = LeaderElector.getNodeName(s);
- log.info("Current overseer: {}", leader);
- Overseer.getStateUpdateQueue(zk)
- .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
- "id", s)));
- final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS);
- String newLeader = null;
- for(;! timeout.hasTimedOut();){
- newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zk);
- if (newLeader != null && !newLeader.equals(leader))
- break;
- Thread.sleep(100);
+ private void waitForNewOverseer(int seconds, String expected) throws Exception {
+ waitForNewOverseer(seconds, s -> Objects.equals(s, expected));
+ }
+
+ private JettySolrRunner getOverseerJetty() throws Exception {
+ String overseer = getLeaderNode(zkClient());
+ URL overseerUrl = new URL("http://" + overseer.substring(0, overseer.indexOf('_')));
+ int hostPort = overseerUrl.getPort();
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ if (jetty.getBaseUrl().getPort() == hostPort)
+ return jetty;
}
- assertThat("Leader not changed yet", newLeader, not(leader));
+ fail("Couldn't find overseer node " + overseer);
+ return null; // to keep the compiler happy
+ }
- assertTrue("The old leader should have rejoined election",
- OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zk).contains(leader));
+ private void logOverseerState() throws KeeperException, InterruptedException {
+ log.info("Overseer: {}", getLeaderNode(zkClient()));
+ log.info("Election queue: ", getSortedElectionNodes(zkClient(), "/overseer_elect/election"));
}
@Test
public void testOverseerRole() throws Exception {
- List<String> l = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()) ;
+ logOverseerState();
+ List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
+ String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ nodes.remove(overseer1);
- log.info("All nodes {}", l);
- String currentLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
- log.info("Current leader {} ", currentLeader);
- l.remove(currentLeader);
+ Collections.shuffle(nodes, random());
+ String overseer2 = nodes.get(0);
+ log.info("### Setting overseer designate {}", overseer2);
- Collections.shuffle(l, random());
- String overseerDesignate = l.get(0);
- log.info("overseerDesignate {}", overseerDesignate);
+ CollectionAdminRequest.addRole(overseer2, "overseer").process(cluster.getSolrClient());
- CollectionAdminRequest.addRole(overseerDesignate, "overseer").process(cluster.getSolrClient());
-
- TimeOut timeout = new TimeOut(15, TimeUnit.SECONDS);
-
- boolean leaderchanged = false;
- for (;!timeout.hasTimedOut();) {
- if (overseerDesignate.equals(OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()))) {
- log.info("overseer designate is the new overseer");
- leaderchanged =true;
- break;
- }
- Thread.sleep(100);
- }
- assertTrue("could not set the new overseer . expected "+
- overseerDesignate + " current order : " +
- getSortedOverseerNodeNames(zkClient()) +
- " ldr :"+ OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()) ,leaderchanged);
+ waitForNewOverseer(15, overseer2);
//add another node as overseer
- l.remove(overseerDesignate);
- Collections.shuffle(l, random());
-
- String anotherOverseer = l.get(0);
- log.info("Adding another overseer designate {}", anotherOverseer);
- CollectionAdminRequest.addRole(anotherOverseer, "overseer").process(cluster.getSolrClient());
-
- String currentOverseer = getLeaderNode(zkClient());
+ nodes.remove(overseer2);
+ Collections.shuffle(nodes, random());
- log.info("Current Overseer {}", currentOverseer);
+ String overseer3 = nodes.get(0);
+ log.info("### Adding another overseer designate {}", overseer3);
+ CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());
- String hostPort = currentOverseer.substring(0, currentOverseer.indexOf('_'));
+ // kill the current overseer, and check that the new designate becomes the new overseer
+ JettySolrRunner leaderJetty = getOverseerJetty();
+ logOverseerState();
- StringBuilder sb = new StringBuilder();
- log.info("hostPort : {}", hostPort);
-
- JettySolrRunner leaderJetty = null;
+ ChaosMonkey.stop(leaderJetty);
+ waitForNewOverseer(10, overseer3);
+
+ // add another node as overseer
+ nodes.remove(overseer3);
+ Collections.shuffle(nodes, random());
+ String overseer4 = nodes.get(0);
+ log.info("### Adding last overseer designate {}", overseer4);
+ CollectionAdminRequest.addRole(overseer4, "overseer").process(cluster.getSolrClient());
+ logOverseerState();
+
+ // remove the overseer role from the current overseer
+ CollectionAdminRequest.removeRole(overseer3, "overseer").process(cluster.getSolrClient());
+ waitForNewOverseer(15, overseer4);
+
+ // Add it back again - we now have two delegates, 4 and 3
+ CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());
+
+ // explicitly tell the overseer to quit
+ String leaderId = OverseerCollectionConfigSetProcessor.getLeaderId(zkClient());
+ String leader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ log.info("### Sending QUIT to overseer {}", leader);
+ Overseer.getStateUpdateQueue(zkClient())
+ .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
+ "id", leaderId)));
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- String s = jetty.getBaseUrl().toString();
- log.info("jetTy {}",s);
- sb.append(s).append(" , ");
- if (s.contains(hostPort)) {
- leaderJetty = jetty;
- break;
- }
- }
+ waitForNewOverseer(10, s -> Objects.equals(leader, s) == false);
- assertNotNull("Could not find a jetty2 kill", leaderJetty);
+ logOverseerState();
+ assertTrue("The old leader should have rejoined election",
+ OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()).contains(leader));
- log.info("leader node {}", leaderJetty.getBaseUrl());
- log.info("current election Queue",
- OverseerCollectionConfigSetProcessor.getSortedElectionNodes(zkClient(), "/overseer_elect/election"));
- ChaosMonkey.stop(leaderJetty);
- timeout = new TimeOut(10, TimeUnit.SECONDS);
- leaderchanged = false;
- for (; !timeout.hasTimedOut(); ) {
- currentOverseer = getLeaderNode(zkClient());
- if (anotherOverseer.equals(currentOverseer)) {
- leaderchanged = true;
- break;
- }
- Thread.sleep(100);
- }
- assertTrue("New overseer designate has not become the overseer, expected : " + anotherOverseer + "actual : " + getLeaderNode(zkClient()), leaderchanged);
}
}
[2/2] lucene-solr:master: SOLR-10001: Fix overseer-roles test bug
Posted by ro...@apache.org.
SOLR-10001: Fix overseer-roles test bug
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eba93909
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eba93909
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eba93909
Branch: refs/heads/master
Commit: eba9390965bcf6b2422524a5628a160ce26c1226
Parents: a14d793
Author: Alan Woodward <ro...@apache.org>
Authored: Thu Jan 19 17:16:42 2017 +0000
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Jan 19 17:46:03 2017 +0000
----------------------------------------------------------------------
.../solr/cloud/CollectionsAPISolrJTest.java | 34 ----
.../apache/solr/cloud/OverseerRolesTest.java | 173 +++++++++----------
2 files changed, 77 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eba93909/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 616b657..3e0d840 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -21,7 +21,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.List;
import java.util.Map;
import org.apache.lucene.util.LuceneTestCase;
@@ -255,39 +254,6 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
}
@Test
- @SuppressWarnings("unchecked")
- public void testAddAndRemoveRole() throws InterruptedException, IOException, SolrServerException {
-
- String node = cluster.getRandomJetty(random()).getNodeName();
-
- CollectionAdminRequest.addRole(node, "overseer").process(cluster.getSolrClient());
-
- CollectionAdminResponse response = CollectionAdminRequest.getClusterStatus().process(cluster.getSolrClient());
-
- NamedList<Object> rsp = response.getResponse();
- NamedList<Object> cs = (NamedList<Object>) rsp.get("cluster");
- assertNotNull("Cluster state should not be null", cs);
- Map<String, Object> roles = (Map<String, Object>) cs.get("roles");
- assertNotNull("Role information should not be null", roles);
- List<String> overseer = (List<String>) roles.get("overseer");
- assertNotNull(overseer);
- assertEquals(1, overseer.size());
- assertTrue(overseer.contains(node));
-
- // Remove role
- CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
-
- response = CollectionAdminRequest.getClusterStatus().process(cluster.getSolrClient());
- rsp = response.getResponse();
- cs = (NamedList<Object>) rsp.get("cluster");
- assertNotNull("Cluster state should not be null", cs);
- roles = (Map<String, Object>) cs.get("roles");
- assertNotNull("Role information should not be null", roles);
- overseer = (List<String>) roles.get("overseer");
- assertFalse(overseer.contains(node));
- }
-
- @Test
public void testOverseerStatus() throws IOException, SolrServerException {
CollectionAdminResponse response = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient());
assertEquals(0, response.getStatus());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eba93909/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 762bbeb..3c2ca87 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -17,28 +17,27 @@
package org.apache.solr.cloud;
import java.lang.invoke.MethodHandles;
+import java.net.URL;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.overseer.OverseerAction;
-import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.data.Stat;
-import org.junit.Before;
+import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getLeaderNode;
-import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames;
-import static org.hamcrest.CoreMatchers.not;
+import static org.apache.solr.cloud.OverseerTaskProcessor.getSortedElectionNodes;
public class OverseerRolesTest extends SolrCloudTestCase {
@@ -51,117 +50,99 @@ public class OverseerRolesTest extends SolrCloudTestCase {
.configure();
}
- @Before
- public void clearAllOverseerRoles() throws Exception {
- for (String node : OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient())) {
- CollectionAdminRequest.removeRole(node, "overseer").process(cluster.getSolrClient());
+ private void waitForNewOverseer(int seconds, Predicate<String> state) throws Exception {
+ TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS);
+ String current = null;
+ while (timeout.hasTimedOut() == false) {
+ current = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ if (state.test(current))
+ return;
+ Thread.sleep(100);
}
+ fail("Timed out waiting for overseer state change");
}
- @Test
- public void testQuitCommand() throws Exception {
-
- SolrZkClient zk = zkClient();
- byte[] data = zk.getData("/overseer_elect/leader", null, new Stat(), true);
- Map m = (Map) Utils.fromJSON(data);
- String s = (String) m.get("id");
- String leader = LeaderElector.getNodeName(s);
- log.info("Current overseer: {}", leader);
- Overseer.getStateUpdateQueue(zk)
- .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
- "id", s)));
- final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS);
- String newLeader = null;
- for(;! timeout.hasTimedOut();){
- newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zk);
- if (newLeader != null && !newLeader.equals(leader))
- break;
- Thread.sleep(100);
+ private void waitForNewOverseer(int seconds, String expected) throws Exception {
+ waitForNewOverseer(seconds, s -> Objects.equals(s, expected));
+ }
+
+ private JettySolrRunner getOverseerJetty() throws Exception {
+ String overseer = getLeaderNode(zkClient());
+ URL overseerUrl = new URL("http://" + overseer.substring(0, overseer.indexOf('_')));
+ int hostPort = overseerUrl.getPort();
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ if (jetty.getBaseUrl().getPort() == hostPort)
+ return jetty;
}
- assertThat("Leader not changed yet", newLeader, not(leader));
+ fail("Couldn't find overseer node " + overseer);
+ return null; // to keep the compiler happy
+ }
- assertTrue("The old leader should have rejoined election",
- OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zk).contains(leader));
+ private void logOverseerState() throws KeeperException, InterruptedException {
+ log.info("Overseer: {}", getLeaderNode(zkClient()));
+ log.info("Election queue: ", getSortedElectionNodes(zkClient(), "/overseer_elect/election"));
}
@Test
public void testOverseerRole() throws Exception {
- List<String> l = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()) ;
+ logOverseerState();
+ List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient());
+ String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ nodes.remove(overseer1);
- log.info("All nodes {}", l);
- String currentLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
- log.info("Current leader {} ", currentLeader);
- l.remove(currentLeader);
+ Collections.shuffle(nodes, random());
+ String overseer2 = nodes.get(0);
+ log.info("### Setting overseer designate {}", overseer2);
- Collections.shuffle(l, random());
- String overseerDesignate = l.get(0);
- log.info("overseerDesignate {}", overseerDesignate);
+ CollectionAdminRequest.addRole(overseer2, "overseer").process(cluster.getSolrClient());
- CollectionAdminRequest.addRole(overseerDesignate, "overseer").process(cluster.getSolrClient());
-
- TimeOut timeout = new TimeOut(15, TimeUnit.SECONDS);
-
- boolean leaderchanged = false;
- for (;!timeout.hasTimedOut();) {
- if (overseerDesignate.equals(OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()))) {
- log.info("overseer designate is the new overseer");
- leaderchanged =true;
- break;
- }
- Thread.sleep(100);
- }
- assertTrue("could not set the new overseer . expected "+
- overseerDesignate + " current order : " +
- getSortedOverseerNodeNames(zkClient()) +
- " ldr :"+ OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()) ,leaderchanged);
+ waitForNewOverseer(15, overseer2);
//add another node as overseer
- l.remove(overseerDesignate);
- Collections.shuffle(l, random());
-
- String anotherOverseer = l.get(0);
- log.info("Adding another overseer designate {}", anotherOverseer);
- CollectionAdminRequest.addRole(anotherOverseer, "overseer").process(cluster.getSolrClient());
-
- String currentOverseer = getLeaderNode(zkClient());
+ nodes.remove(overseer2);
+ Collections.shuffle(nodes, random());
- log.info("Current Overseer {}", currentOverseer);
+ String overseer3 = nodes.get(0);
+ log.info("### Adding another overseer designate {}", overseer3);
+ CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());
- String hostPort = currentOverseer.substring(0, currentOverseer.indexOf('_'));
+ // kill the current overseer, and check that the new designate becomes the new overseer
+ JettySolrRunner leaderJetty = getOverseerJetty();
+ logOverseerState();
- StringBuilder sb = new StringBuilder();
- log.info("hostPort : {}", hostPort);
-
- JettySolrRunner leaderJetty = null;
+ ChaosMonkey.stop(leaderJetty);
+ waitForNewOverseer(10, overseer3);
+
+ // add another node as overseer
+ nodes.remove(overseer3);
+ Collections.shuffle(nodes, random());
+ String overseer4 = nodes.get(0);
+ log.info("### Adding last overseer designate {}", overseer4);
+ CollectionAdminRequest.addRole(overseer4, "overseer").process(cluster.getSolrClient());
+ logOverseerState();
+
+ // remove the overseer role from the current overseer
+ CollectionAdminRequest.removeRole(overseer3, "overseer").process(cluster.getSolrClient());
+ waitForNewOverseer(15, overseer4);
+
+ // Add it back again - we now have two delegates, 4 and 3
+ CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient());
+
+ // explicitly tell the overseer to quit
+ String leaderId = OverseerCollectionConfigSetProcessor.getLeaderId(zkClient());
+ String leader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient());
+ log.info("### Sending QUIT to overseer {}", leader);
+ Overseer.getStateUpdateQueue(zkClient())
+ .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
+ "id", leaderId)));
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- String s = jetty.getBaseUrl().toString();
- log.info("jetTy {}",s);
- sb.append(s).append(" , ");
- if (s.contains(hostPort)) {
- leaderJetty = jetty;
- break;
- }
- }
+ waitForNewOverseer(10, s -> Objects.equals(leader, s) == false);
- assertNotNull("Could not find a jetty2 kill", leaderJetty);
+ logOverseerState();
+ assertTrue("The old leader should have rejoined election",
+ OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()).contains(leader));
- log.info("leader node {}", leaderJetty.getBaseUrl());
- log.info("current election Queue",
- OverseerCollectionConfigSetProcessor.getSortedElectionNodes(zkClient(), "/overseer_elect/election"));
- ChaosMonkey.stop(leaderJetty);
- timeout = new TimeOut(10, TimeUnit.SECONDS);
- leaderchanged = false;
- for (; !timeout.hasTimedOut(); ) {
- currentOverseer = getLeaderNode(zkClient());
- if (anotherOverseer.equals(currentOverseer)) {
- leaderchanged = true;
- break;
- }
- Thread.sleep(100);
- }
- assertTrue("New overseer designate has not become the overseer, expected : " + anotherOverseer + "actual : " + getLeaderNode(zkClient()), leaderchanged);
}
}