You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@activemq.apache.org by cl...@apache.org on 2020/08/07 00:20:30 UTC
[activemq-artemis] branch master updated (3c3b7b6 -> f7bc2f7)
This is an automated email from the ASF dual-hosted git repository.
clebertsuconic pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git.
from 3c3b7b6 This closes #3237
new df2f48a NO-JIRA update RandomUtil.randomInterval to deal with (max==min)
new 9842f45 ARTEMIS-2867 Do not cache IPs on DNS Entries for NetworkHealthCheck
new 27cb9b3 ARTEMIS-2868 Protect Topology Updates from Split Brain
new f7bc2f7 This closes #3232
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../artemis/core/server/NetworkHealthCheck.java | 66 ++++--
.../apache/activemq/artemis/utils/RandomUtil.java | 3 +
.../activemq/artemis/utils/NetworkHealthTest.java | 3 +-
.../activemq/artemis/utils/network/NetUtil.java | 8 +-
.../core/client/impl/ServerLocatorImpl.java | 4 +
.../artemis/core/client/impl/Topology.java | 12 ++
...ctiveMQXAResource.java => TopologyManager.java} | 8 +-
.../core/management/impl/AbstractControl.java | 2 +-
.../protocol/core/impl/CoreProtocolManager.java | 1 +
.../artemis/core/server/ActiveMQServerLogger.java | 6 +
.../artemis/core/server/cluster/BackupManager.java | 9 +
.../core/server/cluster/ClusterConnection.java | 7 +
.../server/cluster/impl/ClusterConnectionImpl.java | 36 +++-
.../core/server/impl/ActiveMQServerImpl.java | 11 +-
.../core/server/impl/LiveOnlyActivation.java | 2 +-
.../server/impl/SharedNothingBackupActivation.java | 33 +--
.../server/impl/SharedNothingLiveActivation.java | 3 +-
.../server/impl/SharedStoreBackupActivation.java | 2 +-
.../server/impl/SharedStoreLiveActivation.java | 2 +-
.../cluster/failover/NetworkIsolationTest.java | 11 +-
tests/smoke-tests/pom.xml | 42 ++++
.../broker.xml | 10 +-
.../management.xml | 0
.../servers/dnsswitch-replicated-backup/broker.xml | 6 +-
.../broker.xml | 4 +-
.../broker.xml | 14 +-
.../management.xml | 0
.../servers/dnsswitch-replicated-main/broker.xml | 4 +-
.../tests/smoke/dnsswitch/DNSSwitchTest.java | 240 ++++++++++++++++-----
.../tests/unit/core/util/RandomUtilTest.java | 25 ++-
30 files changed, 432 insertions(+), 142 deletions(-)
copy artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/{ActiveMQXAResource.java => TopologyManager.java} (87%)
copy tests/smoke-tests/src/main/resources/servers/{dnsswitch-replicated-backup => dnsswitch-replicated-backup-withping}/broker.xml (94%)
copy tests/smoke-tests/src/main/resources/servers/{replicated-failback-master2 => dnsswitch-replicated-backup-withping}/management.xml (100%)
copy tests/smoke-tests/src/main/resources/servers/{dnsswitch-replicated-main-noretrydns => dnsswitch-replicated-main-withping}/broker.xml (93%)
copy tests/smoke-tests/src/main/resources/servers/{replicated-failback-master1 => dnsswitch-replicated-main-withping}/management.xml (100%)
copy artemis-service-extensions/src/test/java/org/apache/activemq/artemis/service/extensions/tests/transactions/TransactionManagerLocatorTest.java => tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/util/RandomUtilTest.java (61%)
[activemq-artemis] 01/04: NO-JIRA update RandomUtil.randomInterval
to deal with (max==min)
Posted by cl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
clebertsuconic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git
commit df2f48a48169da3b9af1158fbd4af54fef6910fa
Author: Clebert Suconic <cl...@apache.org>
AuthorDate: Mon Aug 3 10:36:39 2020 -0400
NO-JIRA update RandomUtil.randomInterval to deal with (max==min)
---
.../apache/activemq/artemis/utils/RandomUtil.java | 3 ++
.../tests/unit/core/util/RandomUtilTest.java | 39 ++++++++++++++++++++++
2 files changed, 42 insertions(+)
diff --git a/artemis-commons/src/main/java/org/apache/activemq/artemis/utils/RandomUtil.java b/artemis-commons/src/main/java/org/apache/activemq/artemis/utils/RandomUtil.java
index 1691df1..766c1d6 100644
--- a/artemis-commons/src/main/java/org/apache/activemq/artemis/utils/RandomUtil.java
+++ b/artemis-commons/src/main/java/org/apache/activemq/artemis/utils/RandomUtil.java
@@ -79,10 +79,13 @@ public class RandomUtil {
}
public static int randomInterval(final int min, final int max) {
+ if (min == max) return max;
return min + random.nextInt(max - min);
}
public static int randomMax(final int max) {
+ assert max > 0;
+
int value = randomPositiveInt() % max;
if (value == 0) {
diff --git a/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/util/RandomUtilTest.java b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/util/RandomUtilTest.java
new file mode 100644
index 0000000..908c9e7
--- /dev/null
+++ b/tests/unit-tests/src/test/java/org/apache/activemq/artemis/tests/unit/core/util/RandomUtilTest.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.activemq.artemis.tests.unit.core.util;
+
+import org.apache.activemq.artemis.utils.RandomUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class RandomUtilTest {
+
+
+ @Test
+ public void testInterval() {
+ int i = RandomUtil.randomInterval(0, 1000);
+ Assert.assertTrue(i <= 1000);
+ Assert.assertTrue(i >= 0);
+
+ i = RandomUtil.randomInterval(0, 0);
+ Assert.assertEquals(0, i);
+
+ i = RandomUtil.randomInterval(10, 10);
+ Assert.assertEquals(10, i);
+
+ }
+}
[activemq-artemis] 02/04: ARTEMIS-2867 Do not cache IPs on DNS
Entries for NetworkHealthCheck
Posted by cl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
clebertsuconic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git
commit 9842f45a49afb81ff81d16fcb51bd0c0bb717b62
Author: Clebert Suconic <cl...@apache.org>
AuthorDate: Mon Aug 3 09:12:44 2020 -0400
ARTEMIS-2867 Do not cache IPs on DNS Entries for NetworkHealthCheck
In case of a DNS outage, the pinger should still fail
If we cache the InetAddress this would not be possible
---
.../artemis/core/server/NetworkHealthCheck.java | 66 ++++++---
.../activemq/artemis/utils/NetworkHealthTest.java | 3 +-
.../activemq/artemis/utils/network/NetUtil.java | 8 +-
.../core/management/impl/AbstractControl.java | 2 +-
.../cluster/failover/NetworkIsolationTest.java | 11 +-
tests/smoke-tests/pom.xml | 42 ++++++
.../broker.xml | 10 +-
.../management.xml | 20 +++
.../servers/dnsswitch-replicated-backup/broker.xml | 6 +-
.../broker.xml | 4 +-
.../broker.xml | 14 +-
.../management.xml | 20 +++
.../servers/dnsswitch-replicated-main/broker.xml | 4 +-
.../tests/smoke/dnsswitch/DNSSwitchTest.java | 165 ++++++++++++++-------
14 files changed, 276 insertions(+), 99 deletions(-)
diff --git a/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java b/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java
index 5c8fc82..559ca17 100644
--- a/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java
+++ b/artemis-commons/src/main/java/org/apache/activemq/artemis/core/server/NetworkHealthCheck.java
@@ -46,7 +46,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
private static final Logger logger = Logger.getLogger(NetworkHealthCheck.class);
private final Set<ActiveMQComponent> componentList = new ConcurrentHashSet<>();
- private final Set<InetAddress> addresses = new ConcurrentHashSet<>();
+ private final Set<String> addresses = new ConcurrentHashSet<>();
private final Set<URL> urls = new ConcurrentHashSet<>();
private NetworkInterface networkInterface;
@@ -104,7 +104,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
return this;
}
- public Set<InetAddress> getAddresses() {
+ public Set<String> getAddresses() {
return addresses;
}
@@ -127,7 +127,8 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
for (String address : addresses) {
if (!address.trim().isEmpty()) {
try {
- this.addAddress(InetAddress.getByName(address.trim()));
+ String strAddress = address.trim();
+ this.addAddress(strAddress);
} catch (Exception e) {
ActiveMQUtilLogger.LOGGER.failedToParseAddressList(e, addressList);
}
@@ -204,22 +205,23 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
return this;
}
- public NetworkHealthCheck addAddress(InetAddress address) {
- if (!check(address)) {
- ActiveMQUtilLogger.LOGGER.addressWasntReacheable(address.toString());
+ public NetworkHealthCheck addAddress(String straddress) {
+ InetAddress address = internalCheck(straddress);
+ if (address == null) {
+ ActiveMQUtilLogger.LOGGER.addressWasntReacheable(straddress);
}
- if (!ignoreLoopback && address.isLoopbackAddress()) {
- ActiveMQUtilLogger.LOGGER.addressloopback(address.toString());
+ if (!ignoreLoopback && address != null && address.isLoopbackAddress()) {
+ ActiveMQUtilLogger.LOGGER.addressloopback(straddress);
} else {
- addresses.add(address);
+ addresses.add(straddress);
checkStart();
}
return this;
}
- public NetworkHealthCheck removeAddress(InetAddress address) {
- addresses.remove(address);
+ public NetworkHealthCheck removeAddress(String straddress) {
+ addresses.remove(straddress);
return this;
}
@@ -267,7 +269,11 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
private void checkStart() {
if (!isStarted() && (!addresses.isEmpty() || !urls.isEmpty()) && !componentList.isEmpty()) {
- start();
+ try {
+ this.run(); // run the first check immediately, this is to immediately shutdown the server if there's no net
+ } finally {
+ start();
+ }
}
}
@@ -311,7 +317,7 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
return true;
}
- for (InetAddress address : addresses) {
+ for (String address : addresses) {
if (check(address)) {
return true;
}
@@ -326,23 +332,37 @@ public class NetworkHealthCheck extends ActiveMQScheduledComponent {
return false;
}
- public boolean check(InetAddress address) {
- if (address == null) {
+ public boolean check(String straddress) {
+ if (straddress == null) {
return false;
}
+ return internalCheck(straddress) != null;
+ }
+
+ private InetAddress internalCheck(String straddress) {
try {
- if (!hasCustomPingCommand() && isReachable(address)) {
- if (logger.isTraceEnabled()) {
- logger.tracef(address + " OK");
- }
- return true;
+ InetAddress address = InetAddress.getByName(straddress);
+ address = InetAddress.getByName(address.getHostName());
+ if (check(address)) {
+ return address;
} else {
- return purePing(address);
+ return null;
}
} catch (Exception e) {
- ActiveMQUtilLogger.LOGGER.failedToCheckAddress(e, address.toString());
- return false;
+ ActiveMQUtilLogger.LOGGER.failedToCheckAddress(e, straddress);
+ return null;
+ }
+ }
+
+ public boolean check(InetAddress address) throws IOException, InterruptedException {
+ if (!hasCustomPingCommand() && isReachable(address)) {
+ if (logger.isTraceEnabled()) {
+ logger.tracef(address + " OK");
+ }
+ return true;
+ } else {
+ return purePing(address);
}
}
diff --git a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java
index b790173..a1b5b41 100644
--- a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java
+++ b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/NetworkHealthTest.java
@@ -155,8 +155,7 @@ public class NetworkHealthTest {
}
});
check.addComponent(component);
- InetAddress address = InetAddress.getByName("127.0.0.1");
- check.addAddress(address);
+ check.addAddress("127.0.0.1");
component.stop();
diff --git a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java
index 489d47b..8731372 100644
--- a/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java
+++ b/artemis-commons/src/test/java/org/apache/activemq/artemis/utils/network/NetUtil.java
@@ -109,6 +109,10 @@ public class NetUtil {
public static void netUp(String ip) throws Exception {
String deviceID = "lo:" + nextDevice.incrementAndGet();
+ netUp(ip, deviceID);
+ }
+
+ public static void netUp(String ip, String deviceID) throws Exception {
if (osUsed == OS.MAC) {
if (runCommand("sudo", "-n", "ifconfig", "lo0", "alias", ip) != 0) {
Assert.fail("Cannot sudo ifconfig for ip " + ip);
@@ -138,7 +142,9 @@ public class NetUtil {
netDown(ip, device, force);
}
- private static void netDown(String ip, String device, boolean force) throws Exception {
+ public static void netDown(String ip, String device, boolean force) throws Exception {
+ networks.remove(ip);
+
if (osUsed == OS.MAC) {
if (runCommand("sudo", "-n", "ifconfig", "lo0", "-alias", ip) != 0) {
if (!force) {
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java
index 3f8b83a..ad5b81a 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/management/impl/AbstractControl.java
@@ -70,7 +70,7 @@ public abstract class AbstractControl extends StandardMBean {
protected void blockOnIO() {
// the storage manager could be null on the backup on certain components
- if (storageManager != null) {
+ if (storageManager != null && storageManager.isStarted()) {
try {
storageManager.waitOnOperations();
storageManager.clearContext();
diff --git a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java
index 95db530..5bfd25c 100644
--- a/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java
+++ b/tests/integration-tests/src/test/java/org/apache/activemq/artemis/tests/integration/cluster/failover/NetworkIsolationTest.java
@@ -17,7 +17,6 @@
package org.apache.activemq.artemis.tests.integration.cluster.failover;
-import java.net.InetAddress;
import java.util.concurrent.TimeUnit;
import org.apache.activemq.artemis.api.core.QueueConfiguration;
@@ -71,7 +70,7 @@ public class NetworkIsolationTest extends FailoverTestBase {
Assert.assertTrue(Wait.waitFor(liveServer::isActive));
- liveServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName(badAddress));
+ liveServer.getServer().getNetworkHealthCheck().addAddress(badAddress);
Assert.assertTrue(Wait.waitFor(() -> !liveServer.isStarted()));
@@ -89,20 +88,20 @@ public class NetworkIsolationTest extends FailoverTestBase {
// this block here is just to validate if ignoring loopback addresses logic is in place
{
- backupServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName("127.0.0.1"));
+ backupServer.getServer().getNetworkHealthCheck().addAddress("127.0.0.1");
Assert.assertTrue(AssertionLoggerHandler.findText("AMQ202001"));
AssertionLoggerHandler.clear();
- backupServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress(InetAddress.getByName("127.0.0.1"));
+ backupServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress("127.0.0.1");
Assert.assertFalse(AssertionLoggerHandler.findText("AMQ202001"));
backupServer.getServer().getNetworkHealthCheck().clearAddresses();
}
- backupServer.getServer().getNetworkHealthCheck().addAddress(InetAddress.getByName(badAddress));
+ backupServer.getServer().getNetworkHealthCheck().addAddress(badAddress);
backupServer.getServer().start();
ClientSessionFactory sf = addSessionFactory(locator.createSessionFactory());
@@ -162,7 +161,7 @@ public class NetworkIsolationTest extends FailoverTestBase {
Assert.assertFalse(liveServer.isStarted());
- liveServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress(InetAddress.getByName("127.0.0.1"));
+ liveServer.getServer().getNetworkHealthCheck().setIgnoreLoopback(true).addAddress("127.0.0.1");
timeout = System.currentTimeMillis() + 30000;
while (!liveServer.isStarted() && System.currentTimeMillis() < timeout) {
diff --git a/tests/smoke-tests/pom.xml b/tests/smoke-tests/pom.xml
index ee0a5a9..d083e8d 100644
--- a/tests/smoke-tests/pom.xml
+++ b/tests/smoke-tests/pom.xml
@@ -270,6 +270,48 @@
</execution>
<execution>
<phase>test-compile</phase>
+ <id>create-dnsswitch-main-withping</id>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ <configuration>
+ <configuration>${basedir}/target/classes/servers/dnsswitch-replicated-main-withping</configuration>
+ <allowAnonymous>true</allowAnonymous>
+ <user>admin</user>
+ <password>admin</password>
+ <noWeb>true</noWeb>
+ <instance>${basedir}/target/dnsswitch-replicated-main-withping</instance>
+ <args>
+ <arg>--java-options</arg>
+ <!-- notice these files are only available on dnsswitch, so this is not a copy and paste error
+ where I really meant dnsswitch here -->
+ <arg>-Djdk.net.hosts.file=${basedir}/target/dnsswitch/etc/hosts.conf -Djava.security.properties=${basedir}/target/dnsswitch/etc/zerocache.security -Djava.rmi.server.hostname=localhost</arg>
+ </args>
+ </configuration>
+ </execution>
+ <execution>
+ <phase>test-compile</phase>
+ <id>create-dnsswitch-backup-withping</id>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ <configuration>
+ <configuration>${basedir}/target/classes/servers/dnsswitch-replicated-backup-withping</configuration>
+ <allowAnonymous>true</allowAnonymous>
+ <user>admin</user>
+ <password>admin</password>
+ <noWeb>true</noWeb>
+ <instance>${basedir}/target/dnsswitch-replicated-backup-withping</instance>
+ <args>
+ <arg>--java-options</arg>
+ <!-- notice these files are only available on dnsswitch, so this is not a copy and paste error
+ where I really meant dnsswitch here -->
+ <arg>-Djdk.net.hosts.file=${basedir}/target/dnsswitch/etc/hosts.conf -Djava.security.properties=${basedir}/target/dnsswitch/etc/zerocache.security -Djava.rmi.server.hostname=localhost</arg>
+ </args>
+ </configuration>
+ </execution>
+ <execution>
+ <phase>test-compile</phase>
<id>create-dnsswitch-main-noretrydns</id>
<goals>
<goal>create</goal>
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml
similarity index 94%
copy from tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml
copy to tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml
index 36a07a6..bb08ea5 100644
--- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/broker.xml
@@ -30,6 +30,12 @@ under the License.
<paging-directory>./data/paging</paging-directory>
+ <network-check-list>PINGPLACE</network-check-list>
+
+ <network-check-period>1000</network-check-period>
+
+ <network-check-timeout>200</network-check-timeout>
+
<ha-policy>
<replication>
<slave>
@@ -41,14 +47,14 @@ under the License.
<connectors>
<!-- Connector used to be announced through cluster connections and notifications -->
- <connector name="artemis">tcp://SECOND:61616</connector>
+ <connector name="artemis">tcp://SECOND:61716</connector>
<connector name="main">tcp://FIRST:61616</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
- <acceptor name="artemis">tcp://SECOND:61616</acceptor>
+ <acceptor name="artemis">tcp://0.0.0.0:61716</acceptor>
</acceptors>
<cluster-user>admin</cluster-user>
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml
new file mode 100644
index 0000000..14bbaf2
--- /dev/null
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup-withping/management.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<management-context xmlns="http://activemq.org/schema">
+ <connector connector-port="10199" connector-host="localhost"/>
+</management-context>
\ No newline at end of file
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml
index 36a07a6..b24398e 100644
--- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-backup/broker.xml
@@ -41,14 +41,14 @@ under the License.
<connectors>
<!-- Connector used to be announced through cluster connections and notifications -->
- <connector name="artemis">tcp://SECOND:61616</connector>
+ <connector name="artemis">tcp://SECOND:61716</connector>
<connector name="main">tcp://FIRST:61616</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
- <acceptor name="artemis">tcp://SECOND:61616</acceptor>
+ <acceptor name="artemis">tcp://0.0.0.0:61716</acceptor>
</acceptors>
<cluster-user>admin</cluster-user>
@@ -58,6 +58,8 @@ under the License.
<cluster-connections>
<cluster-connection name="my-cluster">
<connector-ref>artemis</connector-ref>
+ <check-period>100</check-period>
+ <connection-ttl>500</connection-ttl>
<message-load-balancing>OFF</message-load-balancing>
<max-hops>1</max-hops>
<static-connectors>
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml
index bb5995e..44029f6 100644
--- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml
@@ -43,12 +43,12 @@ under the License.
<connectors>
<!-- Connector used to be announced through cluster connections and notifications -->
<connector name="artemis">tcp://FIRST:61616</connector>
- <connector name="backup">tcp://SECOND:61616</connector>
+ <connector name="backup">tcp://SECOND:61716</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
- <acceptor name="artemis">tcp://FIRST:61616</acceptor>
+ <acceptor name="artemis">tcp://0.0.0.0:61616</acceptor>
</acceptors>
<cluster-user>admin</cluster-user>
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml
similarity index 93%
copy from tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml
copy to tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml
index bb5995e..174e8d3 100644
--- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-noretrydns/broker.xml
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/broker.xml
@@ -40,15 +40,21 @@ under the License.
</replication>
</ha-policy>
+ <network-check-list>PINGPLACE</network-check-list>
+
+ <network-check-period>1000</network-check-period>
+
+ <network-check-timeout>200</network-check-timeout>
+
<connectors>
<!-- Connector used to be announced through cluster connections and notifications -->
<connector name="artemis">tcp://FIRST:61616</connector>
- <connector name="backup">tcp://SECOND:61616</connector>
+ <connector name="backup">tcp://SECOND:61716</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
- <acceptor name="artemis">tcp://FIRST:61616</acceptor>
+ <acceptor name="artemis">tcp://0.0.0.0:61616</acceptor>
</acceptors>
<cluster-user>admin</cluster-user>
@@ -60,9 +66,9 @@ under the License.
<connector-ref>artemis</connector-ref>
<message-load-balancing>OFF</message-load-balancing>
<max-hops>1</max-hops>
- <static-connectors>
+ <!--<static-connectors>
<connector-ref>backup</connector-ref>
- </static-connectors>
+ </static-connectors> -->
</cluster-connection>
</cluster-connections>
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml
new file mode 100644
index 0000000..576f1e5
--- /dev/null
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main-withping/management.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<management-context xmlns="http://activemq.org/schema">
+ <connector connector-port="10099" connector-host="localhost"/>
+</management-context>
\ No newline at end of file
diff --git a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml
index bb5995e..44029f6 100644
--- a/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml
+++ b/tests/smoke-tests/src/main/resources/servers/dnsswitch-replicated-main/broker.xml
@@ -43,12 +43,12 @@ under the License.
<connectors>
<!-- Connector used to be announced through cluster connections and notifications -->
<connector name="artemis">tcp://FIRST:61616</connector>
- <connector name="backup">tcp://SECOND:61616</connector>
+ <connector name="backup">tcp://SECOND:61716</connector>
</connectors>
<!-- Acceptors -->
<acceptors>
- <acceptor name="artemis">tcp://FIRST:61616</acceptor>
+ <acceptor name="artemis">tcp://0.0.0.0:61616</acceptor>
</acceptors>
<cluster-user>admin</cluster-user>
diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
index 31a76d0..aa51d9b 100644
--- a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
+++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
@@ -94,12 +94,17 @@ public class DNSSwitchTest extends SmokeTestBase {
private static final String SERVER_LIVE_NORETRYDNS = "dnsswitch-replicated-main-noretrydns";
private static final String SERVER_BACKUP = "dnsswitch-replicated-backup";
+ private static final String SERVER_LIVE_PING = "dnsswitch-replicated-main-withping";
+ private static final String SERVER_BACKUP_PING = "dnsswitch-replicated-backup-withping";
+
// 192.0.2.0 is reserved for documentation (and testing on this case).
private static final String FIRST_IP = "192.0.2.0";
private static final String SECOND_IP = "192.0.3.0";
private static final String THIRD_IP = "192.0.3.0";
private static final String FOURTH_IP = "192.0.4.0";
+ private static final String INVALID_IP = "203.0.113.0";
+
private static String serverLocation;
@Rule
@@ -122,7 +127,6 @@ public class DNSSwitchTest extends SmokeTestBase {
serverControl.isActive(); // making one call to make sure it's working
return serverControl;
} catch (Throwable e) {
- System.err.println("Retrying error : " + e.getMessage());
lastException = e;
Thread.sleep(500);
}
@@ -235,24 +239,20 @@ public class DNSSwitchTest extends SmokeTestBase {
cleanupData(SERVER_LIVE);
cleanupData(SERVER_LIVE_NORETRYDNS);
cleanupData(SERVER_BACKUP);
+ cleanupData(SERVER_LIVE_PING);
+ cleanupData(SERVER_BACKUP_PING);
}
@Test
public void testBackupRedefinition() throws Throwable {
- System.out.println(System.getProperty("java.security.properties"));
-
spawnRun(serverLocation, "testBackupRedefinition", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP));
}
public static void testBackupRedefinition(String[] args) throws Throwable {
- NetUtil.netUp(FIRST_IP);
- NetUtil.netUp(SECOND_IP);
- // NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND");
- //System.out.println("Waiting here");
- //Thread.sleep(300_000);
-
Process serverLive = null;
Process serverBackup = null;
@@ -263,7 +263,7 @@ public class DNSSwitchTest extends SmokeTestBase {
connectAndWaitBackup();
saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND");
- NetUtil.netDown(SECOND_IP, true);
+ NetUtil.netDown(SECOND_IP, "lo:second", true);
serverBackup.destroyForcibly();
Thread.sleep(1000); // wait some time at least until a reconnection is in place
@@ -280,11 +280,6 @@ public class DNSSwitchTest extends SmokeTestBase {
connectAndWaitBackup();
- //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1);
-
- //System.out.println("I'm here!!!");
- //Thread.sleep(300_000);
-
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true");
Assert.assertTrue(connectionFactory.getServerLocator().isHA());
Connection connection = connectionFactory.createConnection();
@@ -299,7 +294,7 @@ public class DNSSwitchTest extends SmokeTestBase {
producer.send(session.createTextMessage("hello"));
session.commit();
- NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(THIRD_IP, "lo:third");
serverLive.destroyForcibly();
Wait.assertTrue(backupControl::isActive);
@@ -320,7 +315,7 @@ public class DNSSwitchTest extends SmokeTestBase {
errors++;
Assert.assertTrue(errors < 20); // I would accept one or two errors, but the code must connect itself
connection.close();
- connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61616?ha=true");
+ connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61716?ha=true");
connection = connectionFactory.createConnection();
connection.start();
session = connection.createSession(true, Session.SESSION_TRANSACTED);
@@ -342,20 +337,15 @@ public class DNSSwitchTest extends SmokeTestBase {
@Test
public void testBackupRedefinition2() throws Throwable {
- System.out.println(System.getProperty("java.security.properties"));
-
spawnRun(serverLocation, "testBackupRedefinition2", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP));
}
public static void testBackupRedefinition2(String[] args) throws Throwable {
- NetUtil.netUp(FIRST_IP);
- NetUtil.netUp(SECOND_IP);
- NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
+ NetUtil.netUp(THIRD_IP, "lo:third");
saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND");
- //System.out.println("Waiting here");
- //Thread.sleep(300_000);
-
Process serverLive = null;
Process serverBackup = null;
@@ -366,7 +356,7 @@ public class DNSSwitchTest extends SmokeTestBase {
connectAndWaitBackup();
saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND");
- NetUtil.netDown(SECOND_IP, true);
+ NetUtil.netDown(SECOND_IP, "lo:second", true);
serverBackup.destroyForcibly();
Thread.sleep(1000); // wait some time at least until a reconnection is in place
@@ -391,11 +381,6 @@ public class DNSSwitchTest extends SmokeTestBase {
Wait.assertTrue(backupControl::isStarted);
Wait.assertTrue(backupControl::isReplicaSync);
- //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1);
-
- //System.out.println("I'm here!!!");
- //Thread.sleep(300_000);
-
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true");
Assert.assertTrue(connectionFactory.getServerLocator().isHA());
Connection connection = connectionFactory.createConnection();
@@ -410,7 +395,7 @@ public class DNSSwitchTest extends SmokeTestBase {
producer.send(session.createTextMessage("hello"));
session.commit();
- NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(THIRD_IP, "lo:third");
serverLive.destroyForcibly();
Wait.assertTrue(backupControl::isActive);
@@ -431,7 +416,7 @@ public class DNSSwitchTest extends SmokeTestBase {
errors++;
Assert.assertTrue(errors < 20); // I would accept one or two errors, but the code must connect itself
connection.close();
- connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61616?ha=true");
+ connectionFactory = new ActiveMQConnectionFactory("tcp://SECOND:61716?ha=true");
connection = connectionFactory.createConnection();
connection.start();
session = connection.createSession(true, Session.SESSION_TRANSACTED);
@@ -452,20 +437,15 @@ public class DNSSwitchTest extends SmokeTestBase {
@Test
public void testBackupRedefinition3() throws Throwable {
- System.out.println(System.getProperty("java.security.properties"));
-
spawnRun(serverLocation, "testBackupRedefinition2", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP));
}
public static void testBackupRedefinition3(String[] args) throws Throwable {
- NetUtil.netUp(FIRST_IP);
- NetUtil.netUp(SECOND_IP);
- NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
+ NetUtil.netUp(THIRD_IP, "lo:third");
saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND");
- //System.out.println("Waiting here");
- //Thread.sleep(300_000);
-
Process serverLive = null;
Process serverBackup = null;
@@ -476,7 +456,7 @@ public class DNSSwitchTest extends SmokeTestBase {
connectAndWaitBackup();
saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND");
- NetUtil.netDown(SECOND_IP, true);
+ NetUtil.netDown(SECOND_IP, "lo:second", true);
serverBackup.destroyForcibly();
Thread.sleep(1000); // wait some time at least until a reconnection is in place
@@ -501,11 +481,6 @@ public class DNSSwitchTest extends SmokeTestBase {
Wait.assertTrue(backupControl::isStarted);
Wait.assertTrue(backupControl::isReplicaSync);
- //waitForTopology(connectionFactory.getServerLocator().getTopology(), 60_000, 1, 1);
-
- //System.out.println("I'm here!!!");
- //Thread.sleep(300_000);
-
ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory("tcp://FIRST:61616?ha=true");
Assert.assertTrue(connectionFactory.getServerLocator().isHA());
Connection connection = connectionFactory.createConnection();
@@ -520,7 +495,7 @@ public class DNSSwitchTest extends SmokeTestBase {
producer.send(session.createTextMessage("hello"));
session.commit();
- NetUtil.netUp(THIRD_IP);
+ NetUtil.netUp(THIRD_IP, "lo:third");
serverLive.destroyForcibly();
Wait.assertTrue(backupControl::isActive);
@@ -564,14 +539,12 @@ public class DNSSwitchTest extends SmokeTestBase {
@Test
public void testCantReachBack() throws Throwable {
- System.out.println(System.getProperty("java.security.properties"));
-
spawnRun(serverLocation, "testCantReachBack", getServerLocation(SERVER_LIVE_NORETRYDNS), getServerLocation(SERVER_BACKUP));
}
public static void testCantReachBack(String[] args) throws Throwable {
- NetUtil.netUp(FIRST_IP);
- NetUtil.netUp(SECOND_IP);
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
// notice there's no THIRD_IP anywhere
saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "SECOND");
@@ -602,6 +575,90 @@ public class DNSSwitchTest extends SmokeTestBase {
if (serverLive != null) {
serverLive.destroyForcibly();
}
+ }
+
+ }
+
+
+ @Test
+ public void testWithPing() throws Throwable {
+ spawnRun(serverLocation, "testWithPing", getServerLocation(SERVER_LIVE_PING), getServerLocation(SERVER_BACKUP_PING));
+ }
+
+ public static void testWithPing(String[] args) throws Throwable {
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
+ NetUtil.netUp(THIRD_IP, "lo:third");
+
+ // notice there's no THIRD_IP anywhere
+ saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND", THIRD_IP, "PINGPLACE");
+
+ Process serverLive = null;
+ Process serverBackup = null;
+
+ try {
+ serverLive = ServerUtil.startServer(args[1], "live", "tcp://FIRST:61616", 30_000);
+ ActiveMQServerControl liveControl = getServerControl(liveURI, liveNameBuilder, 20_000);
+
+ Wait.assertTrue(liveControl::isStarted);
+
+ // notice the first server does not know about this server at all
+ serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0);
+ ActiveMQServerControl backupControl = getServerControl(backupURI, backupNameBuilder, 20_000);
+
+ Wait.assertTrue(backupControl::isStarted);
+ Wait.assertTrue(backupControl::isReplicaSync);
+ // Removing PINGPLACE from DNS
+ saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND", INVALID_IP, "PINGPLACE");
+
+ Wait.assertFalse(liveControl::isStarted);
+
+ serverBackup.destroyForcibly();
+
+
+ //Thread.sleep(10_000);
+ serverLive.destroyForcibly();
+ serverLive = ServerUtil.startServer(args[1], "live", "tcp://FIRST:61616", 0);
+
+ Thread.sleep(1_000);
+
+
+ logger.debug("going to re-enable ping");
+ // Enable the address just for ping now
+ saveConf(hostsFile, THIRD_IP, "PINGPLACE");
+ liveControl = getServerControl(liveURI, liveNameBuilder, 20_000);
+ Wait.assertTrue(liveControl::isStarted);
+
+ // Waiting some time as to the retry logic to kick in
+ Thread.sleep(5_000);
+
+ // the backup will know about the live, but live doesn't have a direct DNS to backup.. lets see what happens
+ saveConf(hostsFile, FIRST_IP, "FIRST", THIRD_IP, "PINGPLACE");
+
+ boolean ok = false;
+ for (int i = 0; i < 5; i++) {
+ serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0);
+ backupControl = getServerControl(backupURI, backupNameBuilder, 20_000);
+ Wait.assertTrue(backupControl::isStarted);
+ if (!Wait.waitFor(backupControl::isReplicaSync, 5000, 100)) {
+ serverBackup.destroyForcibly();
+ } else {
+ ok = true;
+ break;
+ }
+ }
+
+ Assert.assertTrue(ok);
+
+ //connectAndWaitBackup();
+
+ } finally {
+ if (serverBackup != null) {
+ serverBackup.destroyForcibly();
+ }
+ if (serverLive != null) {
+ serverLive.destroyForcibly();
+ }
}
@@ -752,8 +809,8 @@ public class DNSSwitchTest extends SmokeTestBase {
String securityProperties = System.getProperty("java.security.properties");
- if (securityProperties != null && securityProperties.equals(location + "/etc/zerocache.security3")) {
- System.out.println("No need to spawn a VM, the zerocache is already in place");
+ if (securityProperties != null && securityProperties.equals(location + "/etc/zerocache.security")) {
+ logger.info("No need to spawn a VM, the zerocache is already in place");
System.setProperty("artemis.config.location", location);
USING_SPAWN = false;
main(args);
[activemq-artemis] 04/04: This closes #3232
Posted by cl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
clebertsuconic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git
commit f7bc2f78de0bb9db43ce33f0aaf96b1d6c7a3cdb
Merge: 3c3b7b6 27cb9b3
Author: Clebert Suconic <cl...@apache.org>
AuthorDate: Thu Aug 6 20:20:13 2020 -0400
This closes #3232
.../artemis/core/server/NetworkHealthCheck.java | 66 ++++--
.../apache/activemq/artemis/utils/RandomUtil.java | 3 +
.../activemq/artemis/utils/NetworkHealthTest.java | 3 +-
.../activemq/artemis/utils/network/NetUtil.java | 8 +-
.../core/client/impl/ServerLocatorImpl.java | 4 +
.../artemis/core/client/impl/Topology.java | 12 ++
.../artemis/core/client/impl/TopologyManager.java | 22 ++
.../core/management/impl/AbstractControl.java | 2 +-
.../protocol/core/impl/CoreProtocolManager.java | 1 +
.../artemis/core/server/ActiveMQServerLogger.java | 6 +
.../artemis/core/server/cluster/BackupManager.java | 9 +
.../core/server/cluster/ClusterConnection.java | 7 +
.../server/cluster/impl/ClusterConnectionImpl.java | 36 +++-
.../core/server/impl/ActiveMQServerImpl.java | 11 +-
.../core/server/impl/LiveOnlyActivation.java | 2 +-
.../server/impl/SharedNothingBackupActivation.java | 33 +--
.../server/impl/SharedNothingLiveActivation.java | 3 +-
.../server/impl/SharedStoreBackupActivation.java | 2 +-
.../server/impl/SharedStoreLiveActivation.java | 2 +-
.../cluster/failover/NetworkIsolationTest.java | 11 +-
tests/smoke-tests/pom.xml | 42 ++++
.../broker.xml | 10 +-
.../management.xml | 20 ++
.../servers/dnsswitch-replicated-backup/broker.xml | 6 +-
.../broker.xml | 4 +-
.../broker.xml | 14 +-
.../management.xml | 20 ++
.../servers/dnsswitch-replicated-main/broker.xml | 4 +-
.../tests/smoke/dnsswitch/DNSSwitchTest.java | 240 ++++++++++++++++-----
.../tests/unit/core/util/RandomUtilTest.java | 39 ++++
30 files changed, 513 insertions(+), 129 deletions(-)
[activemq-artemis] 03/04: ARTEMIS-2868 Protect Topology Updates
from Split Brain
Posted by cl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
clebertsuconic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/activemq-artemis.git
commit 27cb9b37b1459a0ca19363fef22002b87be3f2b2
Author: Clebert Suconic <cl...@apache.org>
AuthorDate: Tue Aug 4 13:29:52 2020 -0400
ARTEMIS-2868 Protect Topology Updates from Split Brain
---
.../core/client/impl/ServerLocatorImpl.java | 4 ++
.../artemis/core/client/impl/Topology.java | 12 ++++
.../artemis/core/client/impl/TopologyManager.java | 22 +++++++
.../protocol/core/impl/CoreProtocolManager.java | 1 +
.../artemis/core/server/ActiveMQServerLogger.java | 6 ++
.../artemis/core/server/cluster/BackupManager.java | 9 +++
.../core/server/cluster/ClusterConnection.java | 7 ++
.../server/cluster/impl/ClusterConnectionImpl.java | 36 +++++++++-
.../core/server/impl/ActiveMQServerImpl.java | 11 +++-
.../core/server/impl/LiveOnlyActivation.java | 2 +-
.../server/impl/SharedNothingBackupActivation.java | 33 +++-------
.../server/impl/SharedNothingLiveActivation.java | 3 +-
.../server/impl/SharedStoreBackupActivation.java | 2 +-
.../server/impl/SharedStoreLiveActivation.java | 2 +-
.../tests/smoke/dnsswitch/DNSSwitchTest.java | 77 +++++++++++++++++++++-
15 files changed, 196 insertions(+), 31 deletions(-)
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ServerLocatorImpl.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ServerLocatorImpl.java
index d345e06..5a436ab 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ServerLocatorImpl.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/ServerLocatorImpl.java
@@ -445,6 +445,10 @@ public final class ServerLocatorImpl implements ServerLocatorInternal, Discovery
int pos = loadBalancingPolicy.select(initialConnectors.length);
+ if (initialConnectors.length == 0) {
+ return null;
+ }
+
return new Pair(initialConnectors[pos], null);
}
}
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/Topology.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/Topology.java
index 3b1c64e..432d49b 100644
--- a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/Topology.java
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/Topology.java
@@ -50,6 +50,8 @@ public final class Topology {
*/
private volatile Object owner;
+ private final TopologyManager manager;
+
/**
* topology describes the other cluster nodes that this server knows about:
*
@@ -80,6 +82,11 @@ public final class Topology {
}
this.executor = executor;
this.owner = owner;
+ if (owner instanceof TopologyManager) {
+ manager = (TopologyManager)owner;
+ } else {
+ manager = null;
+ }
if (logger.isTraceEnabled()) {
logger.trace("Topology@" + Integer.toHexString(System.identityHashCode(this)) + " CREATE", new Exception("trace"));
}
@@ -196,6 +203,11 @@ public final class Topology {
return false;
}
+ if (manager != null && !manager.updateMember(uniqueEventID, nodeId, memberInput)) {
+ logger.debugf("TopologyManager rejected the update towards %s", memberInput);
+ return false;
+ }
+
synchronized (this) {
TopologyMemberImpl currentMember = topology.get(nodeId);
diff --git a/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/TopologyManager.java b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/TopologyManager.java
new file mode 100644
index 0000000..611bf73
--- /dev/null
+++ b/artemis-core-client/src/main/java/org/apache/activemq/artemis/core/client/impl/TopologyManager.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.activemq.artemis.core.client.impl;
+
+public interface TopologyManager {
+ boolean updateMember(long uniqueEventID, String nodeId, TopologyMemberImpl memberInput);
+}
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/CoreProtocolManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/CoreProtocolManager.java
index 445faff..84baf25 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/CoreProtocolManager.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/CoreProtocolManager.java
@@ -323,6 +323,7 @@ public class CoreProtocolManager implements ProtocolManager<Interceptor> {
}
});
} catch (RejectedExecutionException ignored) {
+ logger.debug(ignored.getMessage(), ignored);
// this could happen during a shutdown and we don't care, if we lost a nodeDown during a shutdown
// what can we do anyways?
}
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java
index 0a92284..ac128c4 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java
@@ -1700,6 +1700,12 @@ public interface ActiveMQServerLogger extends BasicLogger {
@Message(id = 222292, value = "The metrics-plugin element is ignored because the metrics element is defined", format = Message.Format.MESSAGE_FORMAT)
void metricsPluginElementIgnored();
+ @LogMessage(level = Logger.Level.WARN) // I really want emphasis on this logger, so adding the stars
+ @Message(id = 222294, value = "\n**************************************************************************************************************************************************************************************************************************************************************\n" +
+ "There is a possible split brain on nodeID {0}, coming from connectors {1}. Topology update ignored.\n" +
+ "**************************************************************************************************************************************************************************************************************************************************************", format = Message.Format.MESSAGE_FORMAT)
+ void possibleSplitBrain(String nodeID, String connectionPairInformation);
+
@LogMessage(level = Logger.Level.ERROR)
@Message(id = 224000, value = "Failure in initialisation", format = Message.Format.MESSAGE_FORMAT)
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/BackupManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/BackupManager.java
index 25456b4..97dcab5 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/BackupManager.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/BackupManager.java
@@ -88,11 +88,15 @@ public class BackupManager implements ActiveMQComponent {
return;
//deploy the backup connectors using the cluster configuration
for (ClusterConnectionConfiguration config : configuration.getClusterConfigurations()) {
+ logger.debug("deploy backup config " + config);
deployBackupConnector(config);
}
//start each connector and if we are backup and shared store announce ourselves. NB with replication we don't do this
//as we wait for replication to start and be notified by the replication manager.
for (BackupConnector conn : backupConnectors) {
+ if (logger.isDebugEnabled()) {
+ logger.debugf("****** BackupManager connecting to %s", conn);
+ }
conn.start();
if (server.getHAPolicy().isBackup() && server.getHAPolicy().isSharedStore()) {
conn.informTopology();
@@ -192,6 +196,11 @@ public class BackupManager implements ActiveMQComponent {
private boolean announcingBackup;
private boolean backupAnnounced = false;
+ @Override
+ public String toString() {
+ return "BackupConnector{" + "name='" + name + '\'' + ", connector=" + connector + '}';
+ }
+
private BackupConnector(String name,
TransportConfiguration connector,
long retryInterval,
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterConnection.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterConnection.java
index 6171476..306d4da 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterConnection.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterConnection.java
@@ -48,6 +48,13 @@ public interface ClusterConnection extends ActiveMQComponent, ClusterTopologyLis
void removeClusterTopologyListener(ClusterTopologyListener listener);
/**
+ * This is needed on replication, however we don't need it on shared storage.
+ * */
+ void setSplitBrainDetection(boolean splitBrainDetection);
+
+ boolean isSplitBrainDetection();
+
+ /**
* Only used for tests?
*
* @return a Map of node ID and addresses
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/impl/ClusterConnectionImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/impl/ClusterConnectionImpl.java
index ce56d81..5abba5d 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/impl/ClusterConnectionImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/impl/ClusterConnectionImpl.java
@@ -46,6 +46,7 @@ import org.apache.activemq.artemis.core.client.impl.ClientSessionFactoryInternal
import org.apache.activemq.artemis.core.client.impl.ServerLocatorImpl;
import org.apache.activemq.artemis.core.client.impl.ServerLocatorInternal;
import org.apache.activemq.artemis.core.client.impl.Topology;
+import org.apache.activemq.artemis.core.client.impl.TopologyManager;
import org.apache.activemq.artemis.core.client.impl.TopologyMemberImpl;
import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl;
import org.apache.activemq.artemis.core.postoffice.Binding;
@@ -74,7 +75,7 @@ import org.apache.activemq.artemis.utils.FutureLatch;
import org.apache.activemq.artemis.utils.collections.TypedProperties;
import org.jboss.logging.Logger;
-public final class ClusterConnectionImpl implements ClusterConnection, AfterConnectInternalListener {
+public final class ClusterConnectionImpl implements ClusterConnection, AfterConnectInternalListener, TopologyManager {
private static final Logger logger = Logger.getLogger(ClusterConnectionImpl.class);
@@ -174,6 +175,8 @@ public final class ClusterConnectionImpl implements ClusterConnection, AfterConn
private final String storeAndForwardPrefix;
+ private boolean splitBrainDetection;
+
public ClusterConnectionImpl(final ClusterManager manager,
final TransportConfiguration[] staticTranspConfigs,
final TransportConfiguration connector,
@@ -508,6 +511,37 @@ public final class ClusterConnectionImpl implements ClusterConnection, AfterConn
}
}
+ /** This is the implementation of TopologyManager. It is used to reject eventual updates from a split brain server.
+ *
+ * @param uniqueEventID
+ * @param nodeId
+ * @param memberInput
+ * @return
+ */
+ @Override
+ public boolean updateMember(long uniqueEventID, String nodeId, TopologyMemberImpl memberInput) {
+ if (splitBrainDetection && nodeId.equals(nodeManager.getNodeId().toString())) {
+ TopologyMemberImpl member = topology.getMember(nodeId);
+ if (member != null) {
+ if (member.getLive() != null && memberInput.getLive() != null && !member.getLive().isSameParams(connector)) {
+ ActiveMQServerLogger.LOGGER.possibleSplitBrain(nodeId, memberInput.toString());
+ }
+ }
+ memberInput.setLive(connector);
+ }
+ return true;
+ }
+
+ @Override
+ public void setSplitBrainDetection(boolean splitBrainDetection) {
+ this.splitBrainDetection = splitBrainDetection;
+ }
+
+ @Override
+ public boolean isSplitBrainDetection() {
+ return splitBrainDetection;
+ }
+
@Override
public void onConnection(ClientSessionFactoryInternal sf) {
TopologyMember localMember = getLocalMember();
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java
index 25685e3..3384912 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ActiveMQServerImpl.java
@@ -141,6 +141,7 @@ import org.apache.activemq.artemis.core.server.ServerSession;
import org.apache.activemq.artemis.core.server.ServiceComponent;
import org.apache.activemq.artemis.core.server.ServiceRegistry;
import org.apache.activemq.artemis.core.server.cluster.BackupManager;
+import org.apache.activemq.artemis.core.server.cluster.ClusterConnection;
import org.apache.activemq.artemis.core.server.cluster.ClusterManager;
import org.apache.activemq.artemis.core.server.cluster.ha.HAPolicy;
import org.apache.activemq.artemis.core.server.federation.FederationManager;
@@ -3080,8 +3081,16 @@ public class ActiveMQServerImpl implements ActiveMQServer {
return fileStoreMonitor;
}
- public void completeActivation() throws Exception {
+ public void completeActivation(boolean replicated) throws Exception {
setState(ActiveMQServerImpl.SERVER_STATE.STARTED);
+ if (replicated) {
+ if (getClusterManager() != null) {
+ for (ClusterConnection clusterConnection : getClusterManager().getClusterConnections()) {
+ // we need to avoid split brain on topology for replication
+ clusterConnection.setSplitBrainDetection(true);
+ }
+ }
+ }
getRemotingService().startAcceptors();
activationLatch.countDown();
callActivationCompleteCallbacks();
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/LiveOnlyActivation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/LiveOnlyActivation.java
index cea644f..f8e2d5a 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/LiveOnlyActivation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/LiveOnlyActivation.java
@@ -75,7 +75,7 @@ public class LiveOnlyActivation extends Activation {
activeMQServer.initialisePart2(false);
- activeMQServer.completeActivation();
+ activeMQServer.completeActivation(false);
if (activeMQServer.getIdentity() != null) {
ActiveMQServerLogger.LOGGER.serverIsLive(activeMQServer.getIdentity());
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingBackupActivation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingBackupActivation.java
index 36c9c9f..38483d0 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingBackupActivation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingBackupActivation.java
@@ -126,9 +126,7 @@ public final class SharedNothingBackupActivation extends Activation {
return;
}
- logger.trace("Waiting for a synchronize now...");
synchronized (this) {
- logger.trace("Entered a synchronized");
if (closed)
return;
backupQuorum = new SharedNothingBackupQuorum(activeMQServer.getNodeManager(), activeMQServer.getScheduledPool(), networkHealthCheck, replicaPolicy.getQuorumSize(), replicaPolicy.getVoteRetries(), replicaPolicy.getVoteRetryWait(), replicaPolicy.getQuorumVoteWait(), attemptFailBack);
@@ -154,24 +152,16 @@ public final class SharedNothingBackupActivation extends Activation {
clusterController.addIncomingInterceptorForReplication(new ReplicationError(nodeLocator));
- // nodeManager.startBackup();
- if (logger.isTraceEnabled()) {
- logger.trace("Starting backup manager");
- }
+ logger.debug("Starting backup manager");
activeMQServer.getBackupManager().start();
- if (logger.isTraceEnabled()) {
- logger.trace("Set backup Quorum");
- }
+ logger.debug("Set backup Quorum");
replicationEndpoint.setBackupQuorum(backupQuorum);
replicationEndpoint.setExecutor(activeMQServer.getExecutorFactory().getExecutor());
EndpointConnector endpointConnector = new EndpointConnector();
- if (logger.isTraceEnabled()) {
- logger.trace("Starting Backup Server");
- }
-
+ logger.debug("Starting Backup Server");
ActiveMQServerLogger.LOGGER.backupServerStarted(activeMQServer.getVersion().getFullVersion(), activeMQServer.getNodeManager().getNodeId());
activeMQServer.setState(ActiveMQServerImpl.SERVER_STATE.STARTED);
@@ -180,11 +170,8 @@ public final class SharedNothingBackupActivation extends Activation {
SharedNothingBackupQuorum.BACKUP_ACTIVATION signal;
do {
-
if (closed) {
- if (logger.isTraceEnabled()) {
- logger.trace("Activation is closed, so giving up");
- }
+ logger.debug("Activation is closed, so giving up");
return;
}
@@ -195,15 +182,13 @@ public final class SharedNothingBackupActivation extends Activation {
nodeLocator.locateNode();
Pair<TransportConfiguration, TransportConfiguration> possibleLive = nodeLocator.getLiveConfiguration();
nodeID = nodeLocator.getNodeID();
-
- if (logger.isTraceEnabled()) {
- logger.trace("nodeID = " + nodeID);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Connecting towards a possible live, connection information=" + possibleLive + ", nodeID=" + nodeID);
}
+
//in a normal (non failback) scenario if we couldn't find our live server we should fail
if (!attemptFailBack) {
- if (logger.isTraceEnabled()) {
- logger.trace("attemptFailback=false, nodeID=" + nodeID);
- }
+ logger.debug("attemptFailback=false, nodeID=" + nodeID);
//this shouldn't happen
if (nodeID == null) {
@@ -348,7 +333,7 @@ public final class SharedNothingBackupActivation extends Activation {
logger.trace("completeActivation at the end");
- activeMQServer.completeActivation();
+ activeMQServer.completeActivation(true);
}
} catch (Exception e) {
if (logger.isTraceEnabled()) {
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingLiveActivation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingLiveActivation.java
index 5e3f225..7178cd8 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingLiveActivation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedNothingLiveActivation.java
@@ -98,6 +98,7 @@ public class SharedNothingLiveActivation extends LiveActivation {
@Override
public void run() {
try {
+ // Tell cluster connections to not accept split brains updates on the topology
if (replicatedPolicy.isCheckForLiveServer() && isNodeIdUsed()) {
//set for when we failback
if (logger.isTraceEnabled()) {
@@ -116,7 +117,7 @@ public class SharedNothingLiveActivation extends LiveActivation {
activeMQServer.initialisePart2(false);
- activeMQServer.completeActivation();
+ activeMQServer.completeActivation(true);
if (activeMQServer.getIdentity() != null) {
ActiveMQServerLogger.LOGGER.serverIsLive(activeMQServer.getIdentity());
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreBackupActivation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreBackupActivation.java
index c978ff6..463d97c 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreBackupActivation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreBackupActivation.java
@@ -89,7 +89,7 @@ public final class SharedStoreBackupActivation extends Activation {
activeMQServer.initialisePart2(scalingDown);
- activeMQServer.completeActivation();
+ activeMQServer.completeActivation(false);
if (scalingDown) {
ActiveMQServerLogger.LOGGER.backupServerScaledDown();
diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreLiveActivation.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreLiveActivation.java
index 5802525..6745ed9 100644
--- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreLiveActivation.java
+++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/SharedStoreLiveActivation.java
@@ -82,7 +82,7 @@ public final class SharedStoreLiveActivation extends LiveActivation {
activeMQServer.initialisePart2(false);
- activeMQServer.completeActivation();
+ activeMQServer.completeActivation(false);
ActiveMQServerLogger.LOGGER.serverIsLive();
} catch (Exception e) {
diff --git a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
index aa51d9b..2be1541 100644
--- a/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
+++ b/tests/smoke-tests/src/test/java/org/apache/activemq/artemis/tests/smoke/dnsswitch/DNSSwitchTest.java
@@ -650,8 +650,83 @@ public class DNSSwitchTest extends SmokeTestBase {
Assert.assertTrue(ok);
- //connectAndWaitBackup();
+ } finally {
+ if (serverBackup != null) {
+ serverBackup.destroyForcibly();
+ }
+ if (serverLive != null) {
+ serverLive.destroyForcibly();
+ }
+
+
+ }
+
+ }
+
+
+ @Test
+ public void testWithoutPing() throws Throwable {
+ spawnRun(serverLocation, "testWithoutPing", getServerLocation(SERVER_LIVE), getServerLocation(SERVER_BACKUP));
+ }
+
+ public static void testWithoutPing(String[] args) throws Throwable {
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
+
+ // notice there's no THIRD_IP anywhere
+ saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND");
+
+ Process serverLive = null;
+ Process serverBackup = null;
+ try {
+ serverLive = ServerUtil.startServer(args[1], "live", "tcp://FIRST:61616", 0);
+ ActiveMQServerControl liveControl = getServerControl(liveURI, liveNameBuilder, 20_000);
+
+ Wait.assertTrue(liveControl::isStarted);
+
+ // notice the first server does not know about this server at all
+ serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0);
+ ActiveMQServerControl backupControl = getServerControl(backupURI, backupNameBuilder, 20_000);
+
+ Wait.assertTrue(backupControl::isStarted);
+ Wait.assertTrue(backupControl::isReplicaSync);
+
+ logger.debug("shutdown the Network now");
+
+ // this will remove all the DNS information
+ // I need the pingers to stop responding.
+ // That will only happen if I stop both devices on Linux.
+ // On mac that works regardless
+ NetUtil.netDown(FIRST_IP, "lo:first", false);
+ NetUtil.netDown(SECOND_IP, "lo:second", false);
+ saveConf(hostsFile);
+
+ Wait.assertTrue(backupControl::isActive);
+
+ logger.debug("Starting the network");
+
+ NetUtil.netUp(FIRST_IP, "lo:first");
+ NetUtil.netUp(SECOND_IP, "lo:second");
+ saveConf(hostsFile, FIRST_IP, "FIRST", SECOND_IP, "SECOND");
+
+ // I must wait some time for the backup to have a chance to retry here
+ Thread.sleep(2000);
+
+ logger.debug("Going down now");
+
+ System.out.println("*******************************************************************************************************************************");
+ System.out.println("Forcing backup down and restarting it");
+ System.out.println("*******************************************************************************************************************************");
+
+ serverBackup.destroyForcibly();
+
+ cleanupData(SERVER_BACKUP);
+
+ serverBackup = ServerUtil.startServer(args[2], "backup", "tcp://SECOND:61716", 0);
+ backupControl = getServerControl(backupURI, backupNameBuilder, 20_000);
+ Wait.assertTrue(backupControl::isStarted);
+ Wait.assertTrue(backupControl::isReplicaSync);
} finally {
if (serverBackup != null) {
serverBackup.destroyForcibly();