You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ji...@apache.org on 2018/06/15 16:30:18 UTC

[geode] branch develop updated: GEODE-5306: add crashVM and forceDisconnect in ClusterStartupRule (#2048)

This is an automated email from the ASF dual-hosted git repository.

jinmeiliao pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/geode.git


The following commit(s) were added to refs/heads/develop by this push:
     new 816c180  GEODE-5306: add crashVM and forceDisconnect in ClusterStartupRule (#2048)
816c180 is described below

commit 816c1809f0a8f98cf43ec1d3b19aa70808250a22
Author: jinmeiliao <ji...@pivotal.io>
AuthorDate: Fri Jun 15 09:27:56 2018 -0700

    GEODE-5306: add crashVM and forceDisconnect in ClusterStartupRule (#2048)
    
    Co-authored-by: Helena Bales <hb...@pivotal.io>
---
 .../cli/JdbcClusterConfigDistributedTest.java      |  2 +-
 .../membership/gms/MembershipManagerHelper.java    | 14 ++--
 .../AlterAsyncEventQueueCommandDUnitTest.java      |  4 +-
 .../cli/commands/AlterCompressorDUnitTest.java     | 12 +--
 .../cli/commands/AlterDiskStoreDUnitTest.java      |  4 +-
 .../CreateJndiBindingCommandDUnitTest.java         |  2 +-
 .../commands/DeployCommandRedeployDUnitTest.java   |  2 +-
 .../DestroyJndiBindingCommandDUnitTest.java        |  2 +-
 .../commands/DestroyRegionCommandDUnitTest.java    |  2 +-
 .../cli/commands/DiskStoreCommandsDUnitTest.java   |  8 +-
 .../IndexCommandsShareConfigurationDUnitTest.java  |  2 +-
 .../commands/ShowMissingDiskStoresDUnitTest.java   |  4 +-
 .../ClusterConfigLocatorRestartDUnitTest.java      | 42 +++-------
 .../ImportClusterConfigDistributedTest.java        |  4 +-
 .../ImportOldClusterConfigDUnitTest.java           |  2 +-
 ...onUsingLegacySecurityWithFailoverDUnitTest.java |  8 +-
 .../geode/test/dunit/DistributedTestUtils.java     | 15 +---
 .../geode/test/dunit/rules/ClusterStartupRule.java | 89 +++++++++++++++++++---
 .../apache/geode/test/dunit/rules/MemberVM.java    | 16 +++-
 .../geode/test/junit/rules/LocatorStarterRule.java |  2 +
 .../geode/test/junit/rules/MemberStarterRule.java  | 10 +++
 .../apache/geode/test/junit/rules/VMProvider.java  | 30 +++++---
 ...rConfigurationIndexWithFromClauseDUnitTest.java |  6 +-
 23 files changed, 177 insertions(+), 105 deletions(-)

diff --git a/geode-connectors/src/test/java/org/apache/geode/connectors/jdbc/internal/cli/JdbcClusterConfigDistributedTest.java b/geode-connectors/src/test/java/org/apache/geode/connectors/jdbc/internal/cli/JdbcClusterConfigDistributedTest.java
index d16ef47..406922f 100644
--- a/geode-connectors/src/test/java/org/apache/geode/connectors/jdbc/internal/cli/JdbcClusterConfigDistributedTest.java
+++ b/geode-connectors/src/test/java/org/apache/geode/connectors/jdbc/internal/cli/JdbcClusterConfigDistributedTest.java
@@ -64,7 +64,7 @@ public class JdbcClusterConfigDistributedTest {
       validateRegionMapping(service.getMappingForRegion("regionName"));
     });
 
-    server.stopVM(false);
+    server.stopMember(false);
 
     server = cluster.startServerVM(1, locator.getPort());
     server.invoke(() -> {
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/membership/gms/MembershipManagerHelper.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/membership/gms/MembershipManagerHelper.java
index 645ae0b..f08c05d 100644
--- a/geode-core/src/test/java/org/apache/geode/distributed/internal/membership/gms/MembershipManagerHelper.java
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/membership/gms/MembershipManagerHelper.java
@@ -14,6 +14,10 @@
  */
 package org.apache.geode.distributed.internal.membership.gms;
 
+import java.util.concurrent.TimeUnit;
+
+import org.awaitility.Awaitility;
+
 import org.apache.geode.CancelException;
 import org.apache.geode.distributed.DistributedMember;
 import org.apache.geode.distributed.DistributedSystem;
@@ -125,14 +129,8 @@ public class MembershipManagerHelper {
     MembershipManagerHelper.playDead(msys);
     GMSMembershipManager mgr = ((GMSMembershipManager) getMembershipManager(msys));
     mgr.forceDisconnect("for testing");
-    while (msys.isConnected()) {
-      try {
-        Thread.sleep(1000);
-      } catch (InterruptedException e) {
-        Thread.currentThread().interrupt();
-        return;
-      }
-    }
+    // wait at most 10 seconds for system to be disconnected
+    Awaitility.await().pollInterval(1, TimeUnit.SECONDS).until(() -> !msys.isConnected());
     MembershipManagerHelper.inhibitForcedDisconnectLogging(false);
   }
 
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java
index 78d0276..8f8e12c 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterAsyncEventQueueCommandDUnitTest.java
@@ -83,8 +83,8 @@ public class AlterAsyncEventQueueCommandDUnitTest {
     });
 
     // restart locator and server without clearing the file system
-    lsRule.stopVM(1, false);
-    lsRule.stopVM(0, false);
+    lsRule.stopMember(1, false);
+    lsRule.stopMember(0, false);
 
     locator = lsRule.startLocatorVM(0);
     server1 = lsRule.startServerVM(1, "group1", locator.getPort());
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java
index 28c5a23..f82b0ca 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterCompressorDUnitTest.java
@@ -104,9 +104,9 @@ public class AlterCompressorDUnitTest {
         server2);
 
     // stop the VM
-    server1.stopVM(false);
-    server2.stopVM(false);
-    server3.stopVM(false);
+    server1.stopMember(false);
+    server2.stopMember(false);
+    server3.stopMember(false);
 
     // Alter disk-store & region to add compressor
     Arrays.asList(server1, server2).stream().forEach(server -> {
@@ -146,9 +146,9 @@ public class AlterCompressorDUnitTest {
     VMProvider.invokeInEveryMember(AlterCompressorDUnitTest::verifyRegionIsCompressed, server1);
 
     // stop the VM
-    server1.stopVM(false);
-    server2.stopVM(false);
-    server3.stopVM(false);
+    server1.stopMember(false);
+    server2.stopMember(false);
+    server3.stopMember(false);
 
     Arrays.asList(server1, server2).stream().forEach(server -> {
       String diskDir = server.getWorkingDir() + "/diskStore";
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterDiskStoreDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterDiskStoreDUnitTest.java
index 726f2b6..3f9015f 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterDiskStoreDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/AlterDiskStoreDUnitTest.java
@@ -96,7 +96,7 @@ public class AlterDiskStoreDUnitTest {
 
   @Test
   public void alterDiskStoreUpdatesValues() throws Exception {
-    startupRule.stopVM(1);
+    startupRule.stopMember(1);
 
     gfsh.connectAndVerify(locator.getJmxPort(), PortType.jmxManager);
     String commandOne = commandToSetManyVariables();
@@ -118,7 +118,7 @@ public class AlterDiskStoreDUnitTest {
     });
 
     // Halt the member and "alter" the disk store with the --remove option
-    startupRule.stopVM(1);
+    startupRule.stopMember(1);
     String cmd = commandWithRemoveAndNoOtherOption();
     gfsh.executeAndAssertThat(cmd).statusIsSuccess().containsOutput("The region " + regionName
         + " was successfully removed from the disk store " + diskStoreName);
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateJndiBindingCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateJndiBindingCommandDUnitTest.java
index 8d159d6..992d3d1 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateJndiBindingCommandDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/CreateJndiBindingCommandDUnitTest.java
@@ -93,7 +93,7 @@ public class CreateJndiBindingCommandDUnitTest {
         () -> assertThat(JNDIInvoker.getNoOfAvailableDataSources()).isEqualTo(1));
 
     // bounce server1
-    server1.stopVM(false);
+    server1.stopMember(false);
     server1 = cluster.startServerVM(1, locator.getPort());
 
     // verify it has recreated the datasource from cluster config
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DeployCommandRedeployDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DeployCommandRedeployDUnitTest.java
index 52f66a4..455ce50 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DeployCommandRedeployDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DeployCommandRedeployDUnitTest.java
@@ -137,7 +137,7 @@ public class DeployCommandRedeployDUnitTest {
     server.invoke(() -> assertThatCanLoad(JAR_NAME_A, FUNCTION_A));
     server.invoke(() -> assertThatFunctionHasVersion(FUNCTION_A, VERSION2));
 
-    server.stopVM(false);
+    server.stopMember(false);
 
     lsRule.startServerVM(1, locator.getPort());
 
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyJndiBindingCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyJndiBindingCommandDUnitTest.java
index 247d6ea..b44cfd2 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyJndiBindingCommandDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyJndiBindingCommandDUnitTest.java
@@ -97,7 +97,7 @@ public class DestroyJndiBindingCommandDUnitTest {
         () -> assertThat(JNDIInvoker.getNoOfAvailableDataSources()).isEqualTo(0));
 
     // bounce server1 and assert that there is still no datasource received from cluster config
-    server1.stopVM(false);
+    server1.stopMember(false);
     server1 = cluster.startServerVM(1, locator.getPort());
 
     // verify no datasource from cluster config
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java
index 75826fe..c0cb3de 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DestroyRegionCommandDUnitTest.java
@@ -170,7 +170,7 @@ public class DestroyRegionCommandDUnitTest {
     });
 
     // restart one server to make sure the region does not exist anymore
-    lsRule.stopVM(1);
+    lsRule.stopMember(1);
     lsRule.startServerVM(1, locator.getPort());
 
     // make sure region does not exist
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java
index f58bade..2238420 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/DiskStoreCommandsDUnitTest.java
@@ -100,7 +100,7 @@ public class DiskStoreCommandsDUnitTest {
     gfsh.executeAndAssertThat("show missing-disk-stores")
         .containsOutput("No missing disk store found");
 
-    server1.stopVM(false);
+    server1.stopMember(false);
 
     server2.invoke(() -> {
       Cache cache = ClusterStartupRule.getCache();
@@ -108,7 +108,7 @@ public class DiskStoreCommandsDUnitTest {
       r.put("A", "C");
     });
 
-    server2.stopVM(false);
+    server2.stopMember(false);
 
     int locatorPort = locator.getPort();
     server1.invokeAsync(() -> {
@@ -156,7 +156,7 @@ public class DiskStoreCommandsDUnitTest {
     gfsh.executeAndAssertThat("show missing-disk-stores")
         .containsOutput("No missing disk store found");
 
-    server1.stopVM(false);
+    server1.stopMember(false);
 
     String diskDirs = new File(server1.getWorkingDir(), DISKSTORE).getAbsolutePath();
     gfsh.executeAndAssertThat(
@@ -182,7 +182,7 @@ public class DiskStoreCommandsDUnitTest {
     gfsh.executeAndAssertThat("show missing-disk-stores")
         .containsOutput("No missing disk store found");
 
-    server1.stopVM(false);
+    server1.stopMember(false);
 
     String diskDirs = new File(server1.getWorkingDir(), DISKSTORE).getAbsolutePath();
     File exportDir = tempDir.newFolder();
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/IndexCommandsShareConfigurationDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/IndexCommandsShareConfigurationDUnitTest.java
index 2ba5100..b480b25 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/IndexCommandsShareConfigurationDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/IndexCommandsShareConfigurationDUnitTest.java
@@ -106,7 +106,7 @@ public class IndexCommandsShareConfigurationDUnitTest {
     });
 
     // Restart the data member cache to make sure that the index is destroyed.
-    startupRule.stopVM(1);
+    startupRule.stopMember(1);
     serverVM = startupRule.startServerVM(1, groupName, locator.getPort());
 
     serverVM.invoke(() -> {
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ShowMissingDiskStoresDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ShowMissingDiskStoresDUnitTest.java
index c57c6e6..b70c288 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ShowMissingDiskStoresDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/cli/commands/ShowMissingDiskStoresDUnitTest.java
@@ -98,8 +98,8 @@ public class ShowMissingDiskStoresDUnitTest {
     putUsingGfsh(gfshConnector, testRegionName, 2, "B");
     putUsingGfsh(gfshConnector, testRegionName, 3, "C");
 
-    lsRule.stopVM(1);
-    lsRule.stopVM(2);
+    lsRule.stopMember(1);
+    lsRule.stopMember(2);
 
     AsyncInvocation restart1 = restartServerAsync(server1);
     checkAsyncResults(restart1, gfshConnector, 5);
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigLocatorRestartDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigLocatorRestartDUnitTest.java
index 113521c..e258e07 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigLocatorRestartDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigLocatorRestartDUnitTest.java
@@ -15,7 +15,6 @@
 
 package org.apache.geode.management.internal.configuration;
 
-import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
 import org.awaitility.Awaitility;
@@ -23,11 +22,8 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-import org.apache.geode.distributed.ConfigurationProperties;
 import org.apache.geode.distributed.internal.InternalDistributedSystem;
 import org.apache.geode.distributed.internal.InternalLocator;
-import org.apache.geode.distributed.internal.membership.gms.MembershipManagerHelper;
-import org.apache.geode.test.dunit.Host;
 import org.apache.geode.test.dunit.IgnoredException;
 import org.apache.geode.test.dunit.rules.ClusterStartupRule;
 import org.apache.geode.test.dunit.rules.MemberVM;
@@ -61,19 +57,15 @@ public class ClusterConfigLocatorRestartDUnitTest {
   public void serverRestartsAfterLocatorReconnects() throws Exception {
     IgnoredException.addIgnoredException("org.apache.geode.ForcedDisconnectException: for testing");
 
-    Properties props = new Properties();
-    props.setProperty(ConfigurationProperties.MAX_WAIT_TIME_RECONNECT, "5000");
-    MemberVM locator0 = rule.startLocatorVM(0, props);
+    MemberVM locator0 = rule.startLocatorVM(0);
 
-    rule.startServerVM(1, props, locator0.getPort());
-    MemberVM server2 = rule.startServerVM(2, props, locator0.getPort());
+    rule.startServerVM(1, locator0.getPort());
+    MemberVM server2 = rule.startServerVM(2, locator0.getPort());
 
     addDisconnectListener(locator0);
 
-    server2.invokeAsync(() -> MembershipManagerHelper
-        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));
-    locator0.invokeAsync(() -> MembershipManagerHelper
-        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));
+    server2.forceDisconnectMember();
+    locator0.forceDisconnectMember();
 
     waitForLocatorToReconnect(locator0);
 
@@ -92,29 +84,17 @@ public class ClusterConfigLocatorRestartDUnitTest {
     IgnoredException.addIgnoredException("This node is no longer in the membership view");
     IgnoredException.addIgnoredException("org.apache.geode.ForcedDisconnectException: for testing");
 
-    // Otherwise we get a graceful shutdown...
-    Host.getHost(0).getVM(0).invoke(() -> {
-      if (InternalDistributedSystem.shutdownHook != null) {
-        Runtime.getRuntime().removeShutdownHook(InternalDistributedSystem.shutdownHook);
-      }
-    });
 
-    Properties props = new Properties();
-    props.setProperty(ConfigurationProperties.MAX_WAIT_TIME_RECONNECT, "5000");
-    MemberVM locator0 = rule.startLocatorVM(0, props);
-    MemberVM locator1 = rule.startLocatorVM(1, props, locator0.getPort());
+    MemberVM locator0 = rule.startLocatorVM(0);
+    MemberVM locator1 = rule.startLocatorVM(1, locator0.getPort());
 
-    MemberVM server2 = rule.startServerVM(2, props, locator0.getPort(), locator1.getPort());
-    MemberVM server3 = rule.startServerVM(3, props, locator0.getPort(), locator1.getPort());
+    MemberVM server2 = rule.startServerVM(2, locator0.getPort(), locator1.getPort());
+    MemberVM server3 = rule.startServerVM(3, locator0.getPort(), locator1.getPort());
 
     // Shut down hard
-    locator0.invokeAsync(() -> System.exit(1));
-
-    // Recover the VM so that subsequent rule cleanup works
-    locator0.getVM().bounce();
+    rule.crashVM(0);
 
-    server3.invokeAsync(() -> MembershipManagerHelper
-        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance()));
+    server3.forceDisconnectMember();
 
     rule.startServerVM(4, locator1.getPort(), locator0.getPort());
 
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportClusterConfigDistributedTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportClusterConfigDistributedTest.java
index 3e68c2a..559e269 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportClusterConfigDistributedTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportClusterConfigDistributedTest.java
@@ -75,8 +75,8 @@ public class ImportClusterConfigDistributedTest {
         .statusIsSuccess();
 
     gfsh.disconnect();
-    locator.stopVM(true);
-    server.stopVM(true);
+    locator.stopMember(true);
+    server.stopMember(true);
 
     assertThat(this.exportedClusterConfig).exists();
     assertThat(this.exportedClusterConfig.length()).isGreaterThan(100);
diff --git a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportOldClusterConfigDUnitTest.java b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportOldClusterConfigDUnitTest.java
index 6ab3153..5e63ee4 100644
--- a/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportOldClusterConfigDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/management/internal/configuration/ImportOldClusterConfigDUnitTest.java
@@ -80,7 +80,7 @@ public class ImportOldClusterConfigDUnitTest {
     server.invoke(ImportOldClusterConfigDUnitTest::regionOneExists);
     server.invoke(ImportOldClusterConfigDUnitTest::regionTwoExists);
 
-    lsRule.stopVM(1);
+    lsRule.stopMember(1);
 
     server = lsRule.startServerVM(1, locator.getPort());
     server.invoke(ImportOldClusterConfigDUnitTest::regionOneExists);
diff --git a/geode-core/src/test/java/org/apache/geode/security/ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest.java b/geode-core/src/test/java/org/apache/geode/security/ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest.java
index 354b476..86ceaaf 100644
--- a/geode-core/src/test/java/org/apache/geode/security/ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/security/ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest.java
@@ -192,7 +192,7 @@ public class ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest {
     MemberVM server_to_fail = determinePrimaryServer(client);
 
     // Bring down primary server
-    server_to_fail.stopVM(true);
+    server_to_fail.stopMember(true);
 
     // Confirm failover
     MemberVM secondaryServer = (server1.getPort() == server_to_fail.getPort()) ? server2 : server1;
@@ -256,7 +256,7 @@ public class ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest {
     MemberVM server_to_fail = determinePrimaryServer(client);
 
     // Bring down primary server
-    server_to_fail.stopVM(true);
+    server_to_fail.stopMember(true);
 
     // Confirm failover
     MemberVM secondaryServer = (server1.getPort() == server_to_fail.getPort()) ? server2 : server1;
@@ -302,7 +302,7 @@ public class ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest {
     MemberVM server_to_fail = determinePrimaryServer(client);
 
     // Bring down primary server
-    server_to_fail.stopVM(true);
+    server_to_fail.stopMember(true);
 
     // Confirm failover
     MemberVM secondaryServer = (server1.getPort() == server_to_fail.getPort()) ? server2 : server1;
@@ -361,7 +361,7 @@ public class ClientDataAuthorizationUsingLegacySecurityWithFailoverDUnitTest {
     MemberVM server_to_fail = determinePrimaryServer(client);
 
     // Bring down primary server
-    server_to_fail.stopVM(true);
+    server_to_fail.stopMember(true);
 
     // Confirm failover
     MemberVM secondaryServer = (server1.getPort() == server_to_fail.getPort()) ? server2 : server1;
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/DistributedTestUtils.java b/geode-core/src/test/java/org/apache/geode/test/dunit/DistributedTestUtils.java
index 2892783..8010ecf 100755
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/DistributedTestUtils.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/DistributedTestUtils.java
@@ -35,13 +35,13 @@ import org.apache.geode.internal.InternalInstantiator;
  * or artifacts generated by a DistributedTest.
  *
  * <p>
- * These methods can be used directly: {@code DistributedTestUtils.crashDistributedSystem(...)},
+ * These methods can be used directly: {@code DistributedTestUtils.forceDisconnectMember(...)},
  * however, they are intended to be referenced through static import:
  *
  * <pre>
  * import static org.apache.geode.test.dunit.DistributedTestUtils.*;
  *    ...
- *    crashDistributedSystem(...);
+ *    forceDisconnectMember(...);
  * </pre>
  *
  * Extracted from DistributedTestCase.
@@ -82,17 +82,6 @@ public class DistributedTestUtils {
    */
   public static void crashDistributedSystem(final DistributedSystem system) {
     MembershipManagerHelper.crashDistributedSystem(system);
-    MembershipManagerHelper.inhibitForcedDisconnectLogging(false);
-    WaitCriterion wc = new WaitCriterion() {
-      public boolean done() {
-        return !system.isConnected();
-      }
-
-      public String description() {
-        return "Waiting for distributed system to finish disconnecting: " + system;
-      }
-    };
-    Wait.waitForCriterion(wc, 10000, 1000, true);
   }
 
   /**
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/ClusterStartupRule.java b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/ClusterStartupRule.java
index 9224c2f..73ab466 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/ClusterStartupRule.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/ClusterStartupRule.java
@@ -22,23 +22,29 @@ import static org.apache.geode.test.dunit.standalone.DUnitLauncher.NUM_VMS;
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.FileUtils;
+import org.awaitility.Awaitility;
 import org.junit.rules.ExternalResource;
 
 import org.apache.geode.cache.client.ClientCache;
 import org.apache.geode.cache.client.ClientCacheFactory;
 import org.apache.geode.cache.server.CacheServer;
+import org.apache.geode.distributed.internal.InternalDistributedSystem;
 import org.apache.geode.distributed.internal.InternalLocator;
 import org.apache.geode.internal.cache.InternalCache;
 import org.apache.geode.security.templates.UserPasswordAuthInit;
 import org.apache.geode.test.dunit.Host;
+import org.apache.geode.test.dunit.RMIException;
 import org.apache.geode.test.dunit.VM;
 import org.apache.geode.test.dunit.standalone.DUnitLauncher;
 import org.apache.geode.test.dunit.standalone.VersionManager;
@@ -52,10 +58,11 @@ import org.apache.geode.test.junit.rules.ServerStarterRule;
 import org.apache.geode.test.junit.rules.VMProvider;
 
 /**
- * A rule to help you start locators and servers inside of a
+ * A rule to help you start locators and servers or clients inside of a
  * <a href="https://cwiki.apache.org/confluence/display/GEODE/Distributed-Unit-Tests">DUnit
  * test</a>. This rule will start Servers and Locators inside of the four remote {@link VM}s created
- * by the DUnit framework.
+ * by the DUnit framework. Using this rule will eliminate the need to extends
+ * JUnit4DistributedTestCase when writing a Dunit test
  *
  * <p>
  * If you use this Rule in any test that uses more than the default of 4 VMs in DUnit, then
@@ -73,10 +80,16 @@ public class ClusterStartupRule extends ExternalResource implements Serializable
   }
 
   public static InternalLocator getLocator() {
+    if (memberStarter == null || !(memberStarter instanceof LocatorStarterRule)) {
+      return null;
+    }
     return ((LocatorStarterRule) memberStarter).getLocator();
   }
 
   public static CacheServer getServer() {
+    if (memberStarter == null || !(memberStarter instanceof ServerStarterRule)) {
+      return null;
+    }
     return ((ServerStarterRule) memberStarter).getServer();
   }
 
@@ -126,9 +139,15 @@ public class ClusterStartupRule extends ExternalResource implements Serializable
     } finally {
       MemberStarterRule.disconnectDSIfAny();
 
-      // stop all the clientsVM before stop all the memberVM
-      occupiedVMs.values().forEach(x -> x.stopVMIfNotLocator(true));
-      occupiedVMs.values().forEach(x -> x.stopVM(true));
+      // stop all the members in the order of clients, servers and locators
+      List<VMProvider> vms = new ArrayList<>();
+      vms.addAll(
+          occupiedVMs.values().stream().filter(x -> x.isClient()).collect(Collectors.toSet()));
+      vms.addAll(
+          occupiedVMs.values().stream().filter(x -> x.isServer()).collect(Collectors.toSet()));
+      vms.addAll(
+          occupiedVMs.values().stream().filter(x -> x.isLocator()).collect(Collectors.toSet()));
+      vms.forEach(x -> x.stopMember(true));
 
       // delete any file under root dir
       Arrays.stream(getWorkingDirRoot().listFiles()).filter(File::isFile)
@@ -304,17 +323,69 @@ public class ClusterStartupRule extends ExternalResource implements Serializable
     return getHost(0).getVM(index);
   }
 
-  public void stopVM(int index) {
-    stopVM(index, true);
+  /**
+   * gracefully stop the member inside this vm
+   *
+   * @param index vm index
+   */
+  public void stopMember(int index) {
+    stopMember(index, true);
   }
 
-  public void stopVM(int index, boolean cleanWorkingDir) {
+  /**
+   * gracefully stop the member inside this vm
+   */
+  public void stopMember(int index, boolean cleanWorkingDir) {
     VMProvider member = occupiedVMs.get(index);
 
     if (member == null)
       return;
 
-    member.stopVM(cleanWorkingDir);
+    member.stopMember(cleanWorkingDir);
+  }
+
+  /**
+   * this forces a disconnect of the distributed system of the member.
+   * The member will automatically try to reconnect after 5 seconds.
+   *
+   * will throw a ClassCastException if this method is called on a client VM.
+   */
+  public void forceDisconnectMember(int index) {
+    MemberVM member = getMember(index);
+    if (member == null)
+      return;
+    member.forceDisconnectMember();
+  }
+
+  /**
+   * this crashes the VM hosting the member/client. It removes the VM from the occupied VM list
+   * so that we can ignore it at cleanup.
+   */
+  public void crashVM(int index) {
+    VMProvider member = occupiedVMs.remove(index);
+    member.invokeAsync(() -> {
+      if (InternalDistributedSystem.shutdownHook != null) {
+        Runtime.getRuntime().removeShutdownHook(InternalDistributedSystem.shutdownHook);
+      }
+      System.exit(1);
+    });
+
+    // wait till member is not reachable anymore.
+    Awaitility.await().until(() -> {
+      try {
+        member.invoke(() -> {
+        });
+      } catch (RMIException e) {
+        return true;
+      }
+      return false;
+    });
+
+    // delete the lingering files under this vm
+    Arrays.stream(member.getVM().getWorkingDirectory().listFiles())
+        .forEach(FileUtils::deleteQuietly);
+
+    member.getVM().bounce();
   }
 
   public File getWorkingDirRoot() {
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java
index c9e3fa0..7f2c095 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/rules/MemberVM.java
@@ -78,9 +78,12 @@ public class MemberVM extends VMProvider implements Member {
     return ((Server) member).getEmbeddedLocatorPort();
   }
 
+  /**
+   * this gracefully shutdown the member inside this vm
+   */
   @Override
-  public void stopVM(boolean cleanWorkingDir) {
-    super.stopVM(cleanWorkingDir);
+  public void stopMember(boolean cleanWorkingDir) {
+    super.stopMember(cleanWorkingDir);
 
     if (!cleanWorkingDir) {
       return;
@@ -92,6 +95,14 @@ public class MemberVM extends VMProvider implements Member {
   }
 
   /**
+   * this disconnects the distributed system of the member. The member will automatically try to
+   * reconnect after 5 seconds.
+   */
+  public void forceDisconnectMember() {
+    vm.invoke(() -> ClusterStartupRule.memberStarter.forceDisconnectMember());
+  }
+
+  /**
    * this should called on a locatorVM or a serverVM with jmxManager enabled
    */
   public void waitTillRegionsAreReadyOnServers(String regionPath, int serverCount) {
@@ -113,4 +124,5 @@ public class MemberVM extends VMProvider implements Member {
     vm.invoke(() -> ClusterStartupRule.memberStarter
         .waitTilGatewaySendersAreReady(expectedGatewayObjectCount));
   }
+
 }
diff --git a/geode-core/src/test/java/org/apache/geode/test/junit/rules/LocatorStarterRule.java b/geode-core/src/test/java/org/apache/geode/test/junit/rules/LocatorStarterRule.java
index cc0d45c..f33e0eb 100644
--- a/geode-core/src/test/java/org/apache/geode/test/junit/rules/LocatorStarterRule.java
+++ b/geode-core/src/test/java/org/apache/geode/test/junit/rules/LocatorStarterRule.java
@@ -72,6 +72,8 @@ public class LocatorStarterRule extends MemberStarterRule<LocatorStarterRule> im
     }
   }
 
+
+
   public void startLocator() {
     try {
       // this will start a jmx manager and admin rest service by default
diff --git a/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java b/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java
index 681b905..bd015d2 100644
--- a/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java
+++ b/geode-core/src/test/java/org/apache/geode/test/junit/rules/MemberStarterRule.java
@@ -21,6 +21,7 @@ import static org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER_P
 import static org.apache.geode.distributed.ConfigurationProperties.JMX_MANAGER_START;
 import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
 import static org.apache.geode.distributed.ConfigurationProperties.LOG_FILE;
+import static org.apache.geode.distributed.ConfigurationProperties.MAX_WAIT_TIME_RECONNECT;
 import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
 import static org.apache.geode.distributed.ConfigurationProperties.NAME;
 import static org.apache.geode.distributed.ConfigurationProperties.SECURITY_MANAGER;
@@ -39,6 +40,7 @@ import org.junit.rules.TemporaryFolder;
 
 import org.apache.geode.distributed.DistributedSystem;
 import org.apache.geode.distributed.internal.InternalDistributedSystem;
+import org.apache.geode.distributed.internal.membership.gms.MembershipManagerHelper;
 import org.apache.geode.internal.AvailablePortHelper;
 import org.apache.geode.internal.cache.InternalCache;
 import org.apache.geode.internal.net.SocketCreatorFactory;
@@ -75,6 +77,9 @@ public abstract class MemberStarterRule<T> extends SerializableExternalResource
     // initial values
     properties.setProperty(MCAST_PORT, "0");
     properties.setProperty(LOCATORS, "");
+    // set the reconnect wait time to 5 seconds in case some tests needs to reconnect in a timely
+    // manner.
+    properties.setProperty(MAX_WAIT_TIME_RECONNECT, "5000");
   }
 
   @Override
@@ -282,6 +287,11 @@ public abstract class MemberStarterRule<T> extends SerializableExternalResource
 
   abstract void stopMember();
 
+  public void forceDisconnectMember() {
+    MembershipManagerHelper
+        .crashDistributedSystem(InternalDistributedSystem.getConnectedInstance());
+  }
+
   @Override
   public File getWorkingDir() {
     return workingDir;
diff --git a/geode-core/src/test/java/org/apache/geode/test/junit/rules/VMProvider.java b/geode-core/src/test/java/org/apache/geode/test/junit/rules/VMProvider.java
index dd7ca32..f558b98 100644
--- a/geode-core/src/test/java/org/apache/geode/test/junit/rules/VMProvider.java
+++ b/geode-core/src/test/java/org/apache/geode/test/junit/rules/VMProvider.java
@@ -30,22 +30,31 @@ public abstract class VMProvider {
 
   public abstract VM getVM();
 
-  public void stopVMIfNotLocator(boolean cleanWorkingDir) {
-    getVM().invoke(() -> {
-      if (!org.apache.geode.distributed.Locator.hasLocator()) {
-        ClusterStartupRule.stopElementInsideVM();
-        MemberStarterRule.disconnectDSIfAny();
-      }
-    });
-  }
-
-  public void stopVM(boolean cleanWorkingDir) {
+  public void stopMember(boolean cleanWorkingDir) {
     getVM().invoke(() -> {
       ClusterStartupRule.stopElementInsideVM();
       MemberStarterRule.disconnectDSIfAny();
     });
   };
 
+  public boolean isClient() {
+    return getVM().invoke(() -> {
+      return ClusterStartupRule.clientCacheRule != null;
+    });
+  }
+
+  public boolean isLocator() {
+    return getVM().invoke(() -> {
+      return ClusterStartupRule.getLocator() != null;
+    });
+  }
+
+  public boolean isServer() {
+    return getVM().invoke(() -> {
+      return ClusterStartupRule.getServer() != null;
+    });
+  }
+
   public void invoke(final SerializableRunnableIF runnable) {
     getVM().invoke(runnable);
   }
@@ -61,4 +70,5 @@ public abstract class VMProvider {
   public AsyncInvocation invokeAsync(String name, final SerializableRunnableIF runnable) {
     return getVM().invokeAsync(name, runnable);
   }
+
 }
diff --git a/geode-wan/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigurationIndexWithFromClauseDUnitTest.java b/geode-wan/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigurationIndexWithFromClauseDUnitTest.java
index 14c1bc8..28575e5 100644
--- a/geode-wan/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigurationIndexWithFromClauseDUnitTest.java
+++ b/geode-wan/src/test/java/org/apache/geode/management/internal/configuration/ClusterConfigurationIndexWithFromClauseDUnitTest.java
@@ -73,7 +73,7 @@ public class ClusterConfigurationIndexWithFromClauseDUnitTest {
     String serverName = vm1.getName();
     CommandStringBuilder csb = new CommandStringBuilder(CliStrings.LIST_MEMBER);
     gfshCommandRule.executeAndAssertThat(csb.toString()).statusIsSuccess();
-    lsRule.stopVM(1);
+    lsRule.stopMember(1);
     lsRule.startServerVM(1, locator.getPort());
     verifyIndexRecreated(INDEX_NAME);
   }
@@ -91,7 +91,7 @@ public class ClusterConfigurationIndexWithFromClauseDUnitTest {
     String serverName = vm1.getName();
     CommandStringBuilder csb = new CommandStringBuilder(CliStrings.CREATE_DEFINED_INDEXES);
     gfshCommandRule.executeAndAssertThat(csb.toString()).statusIsSuccess();
-    lsRule.stopVM(1);
+    lsRule.stopMember(1);
     lsRule.startServerVM(1, locator.getPort());
     verifyIndexRecreated(INDEX_NAME);
 
@@ -110,7 +110,7 @@ public class ClusterConfigurationIndexWithFromClauseDUnitTest {
     String serverName = vm1.getName();
     CommandStringBuilder csb = new CommandStringBuilder(CliStrings.CREATE_DEFINED_INDEXES);
     gfshCommandRule.executeAndAssertThat(csb.toString()).statusIsSuccess();
-    lsRule.stopVM(1);
+    lsRule.stopMember(1);
     lsRule.startServerVM(1, locator.getPort());
     verifyIndexRecreated(INDEX_NAME);
 

-- 
To stop receiving notification emails like this one, please contact
jinmeiliao@apache.org.