You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2017/01/03 22:30:25 UTC

[2/2] geode git commit: GEODE-1580 ci failure: GemFireDeadlockDetectorDUnitTest.testDistributedDeadlockWithDLock

GEODE-1580 ci failure: GemFireDeadlockDetectorDUnitTest.testDistributedDeadlockWithDLock

Added a dunit blackboard to coordinate creation of a deadlock with the
DistributedLockService.  Removed pauses and added use of Awaitility to
wait for conditions.


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/43f376f2
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/43f376f2
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/43f376f2

Branch: refs/heads/develop
Commit: 43f376f2a25cc83e9eb3de33ad97f86c4457cc4e
Parents: 3200447
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Tue Jan 3 14:25:29 2017 -0800
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Tue Jan 3 14:30:11 2017 -0800

----------------------------------------------------------------------
 .../GemFireDeadlockDetectorDUnitTest.java       |  77 +++++++-----
 .../geode/test/dunit/DUnitBlackboard.java       | 112 +++++++++++++++++
 .../internal/DUnitBlackboardDUnitTest.java      |  98 +++++++++++++++
 .../test/dunit/internal/InternalBlackboard.java |  72 +++++++++++
 .../dunit/internal/InternalBlackboardImpl.java  | 122 +++++++++++++++++++
 .../internal/JUnit4DistributedTestCase.java     |  54 +++++---
 .../geode/test/dunit/standalone/ChildVM.java    |   2 +
 .../test/dunit/standalone/DUnitLauncher.java    |   4 +
 .../test/dunit/standalone/ProcessManager.java   |   3 +-
 9 files changed, 495 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java
index 17ec800..9c0f8f0 100644
--- a/geode-core/src/test/java/org/apache/geode/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/deadlock/GemFireDeadlockDetectorDUnitTest.java
@@ -14,11 +14,13 @@
  */
 package org.apache.geode.distributed.internal.deadlock;
 
+import org.apache.geode.test.dunit.ThreadUtils;
 import org.junit.experimental.categories.Category;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
 
+import com.jayway.awaitility.Awaitility;
 import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase;
 import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
 import org.apache.geode.test.junit.categories.DistributedTest;
@@ -28,6 +30,7 @@ import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
@@ -153,56 +156,72 @@ public class GemFireDeadlockDetectorDUnitTest extends JUnit4CacheTestCase {
     });
   }
 
-  @Category(FlakyTest.class) // GEODE-1580 uses asyncs with pauses
   @Test
   public void testDistributedDeadlockWithDLock() throws Throwable {
     Host host = Host.getHost(0);
     VM vm0 = host.getVM(0);
     VM vm1 = host.getVM(1);
+    getBlackboard().initBlackboard();
 
+    getSystem();
     AsyncInvocation async1 = lockTheDLocks(vm0, "one", "two");
     AsyncInvocation async2 = lockTheDLocks(vm1, "two", "one");
-    getSystem();
-    GemFireDeadlockDetector detect = new GemFireDeadlockDetector();
 
-    LinkedList<Dependency> deadlock = null;
-    for (int i = 0; i < 60; i++) {
-      deadlock = detect.find().findCycle();
-      if (deadlock != null) {
-        break;
-      }
-      Thread.sleep(1000);
-    }
+    Awaitility.await("waiting for locks to be acquired").atMost(60, TimeUnit.SECONDS)
+        .until(Awaitility.matches(() ->
+            assertTrue(getBlackboard().isGateSignaled("one"))));
+
+    Awaitility.await("waiting for locks to be acquired").atMost(60, TimeUnit.SECONDS)
+        .until(Awaitility.matches(() ->
+            assertTrue(getBlackboard().isGateSignaled("two"))));
 
+    GemFireDeadlockDetector detect = new GemFireDeadlockDetector();
+    LinkedList<Dependency> deadlock = detect.find().findCycle();
+    
     assertTrue(deadlock != null);
-    LogWriterUtils.getLogWriter().info("Deadlock=" + DeadlockDetector.prettyFormat(deadlock));
+
+    System.out.println("Deadlock=" + DeadlockDetector.prettyFormat(deadlock));
+
     assertEquals(4, deadlock.size());
+
     disconnectAllFromDS();
-    async1.getResult(30000);
-    async2.getResult(30000);
+    try {
+      waitForAsyncInvocation(async1, 45, TimeUnit.SECONDS);
+    } finally {
+      waitForAsyncInvocation(async2, 45, TimeUnit.SECONDS);
+    }
+  }
+
+  private void waitForAsyncInvocation(AsyncInvocation async1, int howLong, TimeUnit units)
+      throws java.util.concurrent.ExecutionException, InterruptedException {
+    try {
+      async1.get(howLong, units);
+    } catch (TimeoutException e) {
+      fail("test is leaving behind an async invocation thread");
+    }
   }
 
   private AsyncInvocation lockTheDLocks(VM vm, final String first, final String second) {
     return vm.invokeAsync(new SerializableRunnable() {
 
       public void run() {
-        getCache();
-        DistributedLockService dls = DistributedLockService.create("deadlock_test", getSystem());
-        dls.lock(first, 10 * 1000, -1);
-
         try {
-          Thread.sleep(1000);
-        } catch (InterruptedException e) {
-          e.printStackTrace();
-        }
-        try {
-          dls.lock(second, 10 * 1000, -1);
-        } catch (LockServiceDestroyedException expected) {
-          // this is ok, the test is terminating
-        } catch (DistributedSystemDisconnectedException expected) {
-          // this is ok, the test is terminating
+          getCache();
+          DistributedLockService dls = DistributedLockService.create("deadlock_test", getSystem());
+          dls.lock(first, 10 * 1000, -1);
+          getBlackboard().signalGate(first);
+          getBlackboard().waitForGate(second, 30, TimeUnit.SECONDS);
+          // this will block since the other DUnit VM will have locked the second key
+          try {
+            dls.lock(second, 10 * 1000, -1);
+          } catch (LockServiceDestroyedException expected) {
+            // this is ok, the test is terminating
+          } catch (DistributedSystemDisconnectedException expected) {
+            // this is ok, the test is terminating
+          }
+        } catch (Exception e) {
+          throw new RuntimeException("test failed", e);
         }
-
       }
     });
 

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/DUnitBlackboard.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/DUnitBlackboard.java b/geode-core/src/test/java/org/apache/geode/test/dunit/DUnitBlackboard.java
new file mode 100755
index 0000000..4bbb19e
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/DUnitBlackboard.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.test.dunit;
+
+import java.io.Serializable;
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.geode.test.dunit.internal.InternalBlackboard;
+import org.apache.geode.test.dunit.internal.InternalBlackboardImpl;
+
+/**
+ * DUnitBlackboard provides mailboxes and synchronization gateways for distributed unit tests.
+ * <p>
+ * Tests may use the blackboard to pass objects and status between JVMs with mailboxes instead of
+ * using static variables in classes. The caveat being that the objects will be serialized using
+ * Java serialization.
+ * <p>
+ * Gates may be used to synchronize operations between unit test JVMs. Combined with Awaitility
+ * these can be used to test for conditions being met, actions having happened, etc.
+ * <p>
+ * Look for references to the given methods in your IDE for examples.
+ */
+public class DUnitBlackboard {
+  InternalBlackboard blackboard;
+  
+  public DUnitBlackboard() {
+    blackboard = InternalBlackboardImpl.getInstance();
+  }
+  
+  /**
+   * resets the blackboard
+   */
+  public void initBlackboard() {
+    try {
+      blackboard.initBlackboard();
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+  }
+
+  /**
+   * signals a boolean gate
+   */
+  public void signalGate(String gateName) {
+    try {
+      blackboard.signalGate(gateName);
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+  }
+
+  /**
+   * wait for a gate to be signaled
+   */
+  public void waitForGate(String gateName, long timeout, TimeUnit units)
+      throws TimeoutException, InterruptedException {
+    try {
+      blackboard.waitForGate(gateName, timeout, units);
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+    
+  }
+
+  /**
+   * test to see if a gate has been signeled
+   */
+  public boolean isGateSignaled(String gateName) {
+    try {
+      return blackboard.isGateSignaled(gateName);
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+  }
+  
+  /**
+   * put an object into a mailbox slot. The object must be java-serializable
+   */
+  public void setMailbox(String boxName, Object value) {
+    try {
+      blackboard.setMailbox(boxName, value);
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+  }
+
+  /**
+   * retrieve an object from a mailbox slot
+   */
+  public Object getMailbox(String boxName) {
+    try {
+      return blackboard.getMailbox(boxName);
+    } catch (RemoteException e) {
+      throw new RuntimeException("remote call failed", e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/internal/DUnitBlackboardDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/internal/DUnitBlackboardDUnitTest.java b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/DUnitBlackboardDUnitTest.java
new file mode 100755
index 0000000..a6fa042
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/DUnitBlackboardDUnitTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.test.dunit.internal;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.geode.test.dunit.Host;
+import org.apache.geode.test.dunit.VM;
+import org.apache.geode.test.junit.categories.DistributedTest;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+@Category(DistributedTest.class)
+public class DUnitBlackboardDUnitTest extends JUnit4DistributedTestCase {
+  @Test
+  public void canPassDataBetweenVMs() throws Exception {
+    final String MBOX = "myMailbox";
+    VM vm0 = Host.getHost(0).getVM(0);
+    VM vm1 = Host.getHost(0).getVM(1);
+    
+    vm0.invoke("put data in mailbox", () ->
+      getBlackboard().setMailbox(MBOX, "testing"));
+    
+    String result = (String)vm1.invoke("get data from mailbox", () -> {
+      return getBlackboard().getMailbox(MBOX); });
+    
+    assertEquals("testing", result);
+  }
+  @Test
+  public void canSignalAnotherVM() throws Exception {
+    final String GATE = "myGate";
+    VM vm0 = Host.getHost(0).getVM(0);
+    VM vm1 = Host.getHost(0).getVM(1);
+    
+    vm1.invoke("wait on gate not yet signalled", () -> {
+      assertFalse(getBlackboard().isGateSignaled(GATE));
+      try {
+        getBlackboard().waitForGate(GATE, 1, TimeUnit.SECONDS);
+      } catch (TimeoutException e) {
+        // expected
+        return;
+      } catch (InterruptedException e) {
+        fail("unexpected interrupt");
+      }
+      fail("unexpected success");
+    });
+    
+    vm0.invoke("signal gate", () ->
+        getBlackboard().signalGate(GATE));
+
+    vm1.invoke("wait on gate not yet signalled", () -> {
+      try {
+        getBlackboard().waitForGate(GATE, 1, TimeUnit.SECONDS);
+      } catch (TimeoutException e) {
+        fail("unexpected timeout");
+      } catch (InterruptedException e) {
+        fail("unexpected interrupt");
+      }
+      // success expected
+    });
+  }
+  
+  @Test
+  public void initBlackboardClearsEverything() throws Exception {
+    for (int i = 0; i < 100; i++) {
+      getBlackboard().setMailbox("MBOX" + i, "value" + i);
+      assertEquals("value" + i, getBlackboard().getMailbox("MBOX" + i));
+      getBlackboard().signalGate("GATE" + i);
+      assertTrue(getBlackboard().isGateSignaled("GATE" + i));
+    }
+    Host.getHost(0).getVM(1).invoke("clear blackboard", () ->
+        getBlackboard().initBlackboard());
+
+    for (int i = 0; i < 100; i++) {
+      assertNull(getBlackboard().getMailbox("MBOX" + i));
+      assertFalse(getBlackboard().isGateSignaled("GATE" + i));
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboard.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboard.java b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboard.java
new file mode 100755
index 0000000..63f833b
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboard.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.test.dunit.internal;
+
+import java.io.Serializable;
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * DUnitBlackboard provides mailboxes and synchronization gateways for distributed unit tests.
+ * <p>
+ * Tests may use the blackboard to pass objects and status between JVMs with mailboxes instead of
+ * using static variables in classes. The caveat being that the objects will be serialized using
+ * Java serialization.
+ * <p>
+ * Gates may be used to synchronize operations between unit test JVMs. Combined with Awaitility
+ * these can be used to test for conditions being met, actions having happened, etc.
+ * <p>
+ * Look for references to the given methods in your IDE for examples.
+ */
+public interface InternalBlackboard extends Remote, Serializable {
+  /**
+   * resets the blackboard
+   */
+  void initBlackboard() throws RemoteException;
+
+  /**
+   * signals a boolean gate
+   */
+  void signalGate(String gateName) throws RemoteException;
+
+  /**
+   * wait for a gate to be signaled
+   */
+  void waitForGate(String gateName, long timeout, TimeUnit units)
+      throws RemoteException, TimeoutException, InterruptedException;
+
+  /**
+   * test to see if a gate has been signeled
+   */
+  boolean isGateSignaled(String gateName) throws RemoteException;
+
+  /**
+   * put an object into a mailbox slot. The object must be java-serializable
+   */
+  void setMailbox(String boxName, Object value) throws RemoteException;
+
+  /**
+   * retrieve an object from a mailbox slot
+   */
+  Object getMailbox(String boxName) throws RemoteException;
+
+  /**
+   * ping the blackboard to make sure it's there
+   */
+  void ping() throws RemoteException;
+
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboardImpl.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboardImpl.java b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboardImpl.java
new file mode 100755
index 0000000..e7657ed
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/InternalBlackboardImpl.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.test.dunit.internal;
+
+import org.apache.geode.test.dunit.standalone.DUnitLauncher;
+
+import java.rmi.Naming;
+import java.rmi.NotBoundException;
+import java.rmi.RemoteException;
+import java.rmi.server.UnicastRemoteObject;
+import java.sql.Time;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+
+public class InternalBlackboardImpl extends UnicastRemoteObject implements InternalBlackboard {
+  public static InternalBlackboard blackboard;
+
+  private Map<String, Boolean> gates = new ConcurrentHashMap<>();
+
+  private Map<String, Object> mailboxes = new ConcurrentHashMap();
+
+
+  /**
+   * Zero-arg constructor for remote method invocations.
+   */
+  public InternalBlackboardImpl() throws RemoteException {
+    super();
+  }
+
+  /**
+   * Creates a singleton event listeners blackboard.
+   */
+  public static InternalBlackboard getInstance() {
+    if (blackboard == null) {
+      try {
+        initialize();
+      } catch (Exception e) {
+        throw new RuntimeException("failed to initialize blackboard", e);
+      }
+    }
+    return blackboard;
+  }
+
+  private static synchronized void initialize() throws Exception {
+    if (blackboard == null) {
+      System.out.println(
+          DUnitLauncher.RMI_PORT_PARAM + "=" + System.getProperty(DUnitLauncher.RMI_PORT_PARAM));
+      int namingPort = Integer.getInteger(DUnitLauncher.RMI_PORT_PARAM).intValue();
+      String name = "//localhost:" + namingPort + "/" + "InternalBlackboard";
+      try {
+        blackboard = (InternalBlackboard) Naming.lookup(name);
+      } catch (NotBoundException e) {
+        // create the master blackboard in this VM
+        blackboard = new InternalBlackboardImpl();
+        Naming.bind(name, blackboard);
+      }
+    }
+  }
+
+  @Override
+  public void initBlackboard() throws RemoteException {
+    this.gates.clear();
+    this.mailboxes.clear();
+  }
+
+  @Override
+  public void signalGate(final String gateName) throws RemoteException {
+    gates.put(gateName, Boolean.TRUE);
+  }
+
+  @Override
+  public void waitForGate(final String gateName, final long timeout, final TimeUnit units)
+      throws RemoteException, TimeoutException, InterruptedException {
+    long giveupTime = System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(timeout, units);
+    while (System.currentTimeMillis() < giveupTime) {
+      Boolean gate = gates.get(gateName);
+      if (gate != null && gate) {
+        return;
+      }
+      Thread.sleep(50L);
+    }
+    throw new TimeoutException();
+  }
+
+  @Override
+  public boolean isGateSignaled(final String gateName) {
+    Boolean gate = gates.get(gateName);
+    return (gate != null && gate);
+  }
+
+  @Override
+  public void setMailbox(String boxName, Object value) {
+    mailboxes.put(boxName, value);
+  }
+
+  @Override
+  public Object getMailbox(String boxName) {
+    return mailboxes.get(boxName);
+  }
+
+  @Override
+  public void ping() throws RemoteException {
+    // no-op
+  }
+
+
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/internal/JUnit4DistributedTestCase.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/internal/JUnit4DistributedTestCase.java b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/JUnit4DistributedTestCase.java
index ebbb2fd..344a0e1 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/internal/JUnit4DistributedTestCase.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/internal/JUnit4DistributedTestCase.java
@@ -14,22 +14,11 @@
  */
 package org.apache.geode.test.dunit.internal;
 
-import static org.apache.geode.distributed.ConfigurationProperties.*;
-import static org.junit.Assert.*;
-
-import java.io.Serializable;
-import java.text.DecimalFormat;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-import org.apache.logging.log4j.Logger;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Rule;
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.LOG_FILE;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+import static org.apache.geode.distributed.ConfigurationProperties.STATISTIC_ARCHIVE_FILE;
+import static org.junit.Assert.assertNotNull;
 
 import org.apache.geode.admin.internal.AdminDistributedSystemImpl;
 import org.apache.geode.cache.Cache;
@@ -44,8 +33,6 @@ import org.apache.geode.distributed.DistributedSystem;
 import org.apache.geode.distributed.internal.DistributionConfig;
 import org.apache.geode.distributed.internal.DistributionMessageObserver;
 import org.apache.geode.distributed.internal.InternalDistributedSystem;
-import org.apache.geode.internal.net.SSLConfigurationFactory;
-import org.apache.geode.internal.net.SocketCreator;
 import org.apache.geode.internal.admin.ClientStatsManager;
 import org.apache.geode.internal.cache.DiskStoreObserver;
 import org.apache.geode.internal.cache.GemFireCacheImpl;
@@ -58,8 +45,10 @@ import org.apache.geode.internal.cache.tier.sockets.ClientProxyMembershipID;
 import org.apache.geode.internal.cache.tier.sockets.Message;
 import org.apache.geode.internal.cache.xmlcache.CacheCreation;
 import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.internal.net.SocketCreator;
 import org.apache.geode.internal.net.SocketCreatorFactory;
 import org.apache.geode.management.internal.cli.LogWrapper;
+import org.apache.geode.test.dunit.DUnitBlackboard;
 import org.apache.geode.test.dunit.DistributedTestUtils;
 import org.apache.geode.test.dunit.Host;
 import org.apache.geode.test.dunit.IgnoredException;
@@ -67,6 +56,19 @@ import org.apache.geode.test.dunit.Invoke;
 import org.apache.geode.test.dunit.LogWriterUtils;
 import org.apache.geode.test.dunit.standalone.DUnitLauncher;
 import org.apache.geode.test.junit.rules.serializable.SerializableTestName;
+import org.apache.logging.log4j.Logger;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+
+import java.io.Serializable;
+import java.text.DecimalFormat;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
 
 /**
  * This class is the base class for all distributed tests using JUnit 4.
@@ -113,11 +115,27 @@ public abstract class JUnit4DistributedTestCase implements DistributedTestFixtur
   @Rule
   public SerializableTestName testName = new SerializableTestName();
 
+  private static DUnitBlackboard blackboard;
+
+  /**
+   * Returns a DUnitBlackboard that can be used to pass data between VMs and
+   * synchronize actions.
+   * 
+   * @return the blackboard
+   */
+  public DUnitBlackboard getBlackboard() {
+    return blackboard;
+  }
+
   @BeforeClass
   public static final void initializeDistributedTestCase() {
     DUnitLauncher.launchIfNeeded();
   }
 
+  public static final void initializeBlackboard() {
+    blackboard = new DUnitBlackboard();
+  }
+
   public final String getName() {
     if (this.distributedTestFixture != this) {
       return this.distributedTestFixture.getName();

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ChildVM.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ChildVM.java b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ChildVM.java
index 7a9d7f0..b85d0b5 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ChildVM.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ChildVM.java
@@ -18,6 +18,7 @@ import java.rmi.Naming;
 
 import org.apache.geode.internal.OSProcess;
 import org.apache.geode.internal.logging.LogService;
+import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
 import org.apache.geode.test.dunit.standalone.DUnitLauncher.MasterRemote;
 
 import org.apache.logging.log4j.Logger;
@@ -50,6 +51,7 @@ public class ChildVM {
       DUnitLauncher.locatorPort = holder.getLocatorPort();
       final RemoteDUnitVM dunitVM = new RemoteDUnitVM();
       Naming.rebind("//localhost:" + namingPort + "/vm" + vmNum, dunitVM);
+      JUnit4DistributedTestCase.initializeBlackboard();
       holder.signalVMReady();
       // This loop is here so this VM will die even if the master is mean killed.
       while (!stopMainLoop) {

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/DUnitLauncher.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/DUnitLauncher.java b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/DUnitLauncher.java
index 5a80a64..8587ea5 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/DUnitLauncher.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/DUnitLauncher.java
@@ -27,6 +27,8 @@ import org.apache.geode.test.dunit.Host;
 import org.apache.geode.test.dunit.SerializableCallable;
 import org.apache.geode.test.dunit.VM;
 import hydra.MethExecutorResult;
+
+import org.apache.geode.test.dunit.internal.JUnit4DistributedTestCase;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.LoggerContext;
@@ -145,6 +147,8 @@ public class DUnitLauncher {
     Registry registry = LocateRegistry.createRegistry(namingPort);
     System.setProperty(RMI_PORT_PARAM, "" + namingPort);
 
+    JUnit4DistributedTestCase.initializeBlackboard();
+
     final ProcessManager processManager = new ProcessManager(namingPort, registry);
     master = new Master(registry, processManager);
     registry.bind(MASTER_PARAM, master);

http://git-wip-us.apache.org/repos/asf/geode/blob/43f376f2/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ProcessManager.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ProcessManager.java b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ProcessManager.java
index 7852db5..c3c33d0 100644
--- a/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ProcessManager.java
+++ b/geode-core/src/test/java/org/apache/geode/test/dunit/standalone/ProcessManager.java
@@ -191,8 +191,7 @@ public class ProcessManager {
     cmds.add("-Xmx512m");
     cmds.add("-D" + DistributionConfig.GEMFIRE_PREFIX + "DEFAULT_MAX_OPLOG_SIZE=10");
     cmds.add("-D" + DistributionConfig.GEMFIRE_PREFIX + "disallowMcastDefaults=true");
-    cmds.add("-D" + DistributionConfig.GEMFIRE_PREFIX
-        + DistributionConfig.RESTRICT_MEMBERSHIP_PORT_RANGE + "=true");
+    cmds.add("-D" + DistributionConfig.RESTRICT_MEMBERSHIP_PORT_RANGE + "=true");
     cmds.add("-ea");
     cmds.add("-XX:MetaspaceSize=512m");
     cmds.add("-XX:+PrintGC");