You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ud...@apache.org on 2016/02/25 00:40:56 UTC

[1/7] incubator-geode git commit: GEODE-870: Adding GMSJoinLeaveTestHelper back after gemfire/geode merge

Repository: incubator-geode
Updated Branches:
  refs/heads/feature/GEODE-870 [created] 3fdee8f51


GEODE-870: Adding GMSJoinLeaveTestHelper back after gemfire/geode merge


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/3fdee8f5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/3fdee8f5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/3fdee8f5

Branch: refs/heads/feature/GEODE-870
Commit: 3fdee8f5115fc2f6167dd5fe404e0944e58add31
Parents: 2bff98b
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Tue Feb 23 08:20:45 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../gms/membership/GMSJoinLeaveTestHelper.java  | 67 --------------------
 .../gms/membership/GMSJoinLeaveTestHelper.java  | 67 ++++++++++++++++++++
 2 files changed, 67 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/3fdee8f5/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
deleted file mode 100644
index 493c625..0000000
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
-
-import com.gemstone.gemfire.distributed.Locator;
-import com.gemstone.gemfire.distributed.internal.DM;
-import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
-import com.gemstone.gemfire.distributed.internal.membership.gms.Services;
-import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager;
-
-public class GMSJoinLeaveTestHelper {
-
-  public static void becomeCoordinatorForTest(GMSJoinLeave gmsJoinLeave) {
-    synchronized (gmsJoinLeave.getViewInstallationLock()) {
-      gmsJoinLeave.becomeCoordinator();
-    }
-  }
-
-  public static boolean isViewCreator() {
-    GMSJoinLeave gmsJoinLeave = getGmsJoinLeave();
-    if (gmsJoinLeave != null) {
-      GMSJoinLeave.ViewCreator viewCreator = gmsJoinLeave.getViewCreator();
-      if (viewCreator != null && !viewCreator.isShutdown()) {
-        return true;
-      } else {
-        return false;
-      }
-    }
-    throw new RuntimeException("This should not have happened. There should be a JoinLeave for every DS");
-  }
-
-  private static GMSJoinLeave getGmsJoinLeave() {
-    InternalDistributedSystem distributedSystem = getInternalDistributedSystem();
-    DM dm = distributedSystem.getDM();
-    GMSMembershipManager membershipManager = (GMSMembershipManager) dm.getMembershipManager();
-    Services services = membershipManager.getServices();
-    return (GMSJoinLeave) services.getJoinLeave();
-  }
-
-  public static Integer getViewId() {
-    return getGmsJoinLeave().getView().getViewId();
-  }
-
-  private static InternalDistributedSystem getInternalDistributedSystem() {
-    InternalDistributedSystem distributedSystem = InternalDistributedSystem.getAnyInstance();
-    if (distributedSystem == null) {
-      Locator locator = Locator.getLocator();
-      return (InternalDistributedSystem) locator.getDistributedSystem();
-    } else {
-      return distributedSystem;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/3fdee8f5/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
new file mode 100644
index 0000000..493c625
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
+
+import com.gemstone.gemfire.distributed.Locator;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.distributed.internal.membership.gms.Services;
+import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager;
+
+public class GMSJoinLeaveTestHelper {
+
+  public static void becomeCoordinatorForTest(GMSJoinLeave gmsJoinLeave) {
+    synchronized (gmsJoinLeave.getViewInstallationLock()) {
+      gmsJoinLeave.becomeCoordinator();
+    }
+  }
+
+  public static boolean isViewCreator() {
+    GMSJoinLeave gmsJoinLeave = getGmsJoinLeave();
+    if (gmsJoinLeave != null) {
+      GMSJoinLeave.ViewCreator viewCreator = gmsJoinLeave.getViewCreator();
+      if (viewCreator != null && !viewCreator.isShutdown()) {
+        return true;
+      } else {
+        return false;
+      }
+    }
+    throw new RuntimeException("This should not have happened. There should be a JoinLeave for every DS");
+  }
+
+  private static GMSJoinLeave getGmsJoinLeave() {
+    InternalDistributedSystem distributedSystem = getInternalDistributedSystem();
+    DM dm = distributedSystem.getDM();
+    GMSMembershipManager membershipManager = (GMSMembershipManager) dm.getMembershipManager();
+    Services services = membershipManager.getServices();
+    return (GMSJoinLeave) services.getJoinLeave();
+  }
+
+  public static Integer getViewId() {
+    return getGmsJoinLeave().getView().getViewId();
+  }
+
+  private static InternalDistributedSystem getInternalDistributedSystem() {
+    InternalDistributedSystem distributedSystem = InternalDistributedSystem.getAnyInstance();
+    if (distributedSystem == null) {
+      Locator locator = Locator.getLocator();
+      return (InternalDistributedSystem) locator.getDistributedSystem();
+    } else {
+      return distributedSystem;
+    }
+  }
+}


[6/7] incubator-geode git commit: GEODE-870: test case fixes.

Posted by ud...@apache.org.
GEODE-870: test case fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/2bff98b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/2bff98b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/2bff98b1

Branch: refs/heads/feature/GEODE-870
Commit: 2bff98b1c0699be841d3383c1f98e81151f7d870
Parents: 83e4d24
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Wed Feb 17 14:51:19 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../membership/gms/membership/GMSJoinLeave.java | 212 ++++++++++++-------
 .../cache30/CacheSerializableRunnable.java      |  16 +-
 .../test/dunit/SerializableRunnable.java        |   2 +-
 3 files changed, 156 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2bff98b1/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index 8ba4952..dcb2721 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -55,104 +55,166 @@ import static com.gemstone.gemfire.internal.DataSerializableFixedID.*;
  * that Geode formerly used for this purpose.
  */
 public class GMSJoinLeave implements JoinLeave, MessageHandler {
-  
+
   public static final String BYPASS_DISCOVERY_PROPERTY = "gemfire.bypass-discovery";
 
-  /** amount of time to wait for responses to FindCoordinatorRequests */
+  /**
+   * amount of time to wait for responses to FindCoordinatorRequests
+   */
   private static final int DISCOVERY_TIMEOUT = Integer.getInteger("gemfire.discovery-timeout", 3000);
 
-  /** amount of time to sleep before trying to join after a failed attempt */
+  /**
+   * amount of time to sleep before trying to join after a failed attempt
+   */
   private static final int JOIN_RETRY_SLEEP = Integer.getInteger("gemfire.join-retry-sleep", 1000);
 
-  /** time to wait for a broadcast message to be transmitted by jgroups */
+  /**
+   * time to wait for a broadcast message to be transmitted by jgroups
+   */
   private static final long BROADCAST_MESSAGE_SLEEP_TIME = Long.getLong("gemfire.broadcast-message-sleep-time", 1000);
 
-  /** if the locators don't know who the coordinator is we send find-coord requests to this many nodes */
+  /**
+   * if the locators don't know who the coordinator is we send find-coord requests to this many nodes
+   */
   private static final int MAX_DISCOVERY_NODES = Integer.getInteger("gemfire.max-discovery-nodes", 30);
 
-  /** interval for broadcasting the current view to members in case they didn't get it the first time */
+  /**
+   * interval for broadcasting the current view to members in case they didn't get it the first time
+   */
   private static final long VIEW_BROADCAST_INTERVAL = Long.getLong("gemfire.view-broadcast-interval", 60000);
 
-  /** membership logger */
+  /**
+   * membership logger
+   */
   private static final Logger logger = Services.getLogger();
 
-  /** the view ID where I entered into membership */
+  /**
+   * the view ID where I entered into membership
+   */
   private int birthViewId;
 
-  /** my address */
+  /**
+   * my address
+   */
   private InternalDistributedMember localAddress;
 
   private Services services;
 
-  /** have I connected to the distributed system? */
+  /**
+   * have I connected to the distributed system?
+   */
   private volatile boolean isJoined;
 
-  /** guarded by viewInstallationLock */
+  /**
+   * guarded by viewInstallationLock
+   */
   private boolean isCoordinator;
 
-  /** a synch object that guards view installation */
+  /**
+   * a synch object that guards view installation
+   */
   private final Object viewInstallationLock = new Object();
-  
-  /** the currently installed view.  Guarded by viewInstallationLock */
+
+  /**
+   * the currently installed view.  Guarded by viewInstallationLock
+   */
   private volatile NetView currentView;
 
-  /** the previous view **/
+  /**
+   * the previous view
+   **/
   private volatile NetView previousView;
 
-  /** members who we have been declared dead in the current view */
+  /**
+   * members who we have been declared dead in the current view
+   */
   private final Set<InternalDistributedMember> removedMembers = new HashSet<>();
 
-  /** members who we've received a leave message from **/
+  /**
+   * members who we've received a leave message from
+   **/
   private final Set<InternalDistributedMember> leftMembers = new HashSet<>();
 
-  /** a new view being installed */
+  /**
+   * a new view being installed
+   */
   private NetView preparedView;
 
-  /** the last view that conflicted with view preparation */
+  /**
+   * the last view that conflicted with view preparation
+   */
   private NetView lastConflictingView;
 
   private List<InetSocketAddress> locators;
 
-  /** a list of join/leave/crashes */
+  /**
+   * a list of join/leave/crashes
+   */
   private final List<DistributionMessage> viewRequests = new LinkedList<DistributionMessage>();
 
-  /** the established request collection jitter.  This can be overridden for testing with delayViewCreationForTest */
+  /**
+   * the established request collection jitter.  This can be overridden for testing with delayViewCreationForTest
+   */
   long requestCollectionInterval = MEMBER_REQUEST_COLLECTION_INTERVAL;
 
-  /** collects the response to a join request */
+  /**
+   * collects the response to a join request
+   */
   private JoinResponseMessage[] joinResponse = new JoinResponseMessage[1];
 
-  /** collects responses to new views */
+  /**
+   * collects responses to new views
+   */
   private ViewReplyProcessor viewProcessor = new ViewReplyProcessor(false);
 
-  /** collects responses to view preparation messages */
+  /**
+   * collects responses to view preparation messages
+   */
   private ViewReplyProcessor prepareProcessor = new ViewReplyProcessor(true);
 
-  /** whether quorum checks can cause a forced-disconnect */
+  /**
+   * whether quorum checks can cause a forced-disconnect
+   */
   private boolean quorumRequired = false;
 
-  /** timeout in receiving view acknowledgement */
+  /**
+   * timeout in receiving view acknowledgement
+   */
   private int viewAckTimeout;
 
-  /** background thread that creates new membership views */
+  /**
+   * background thread that creates new membership views
+   */
   private ViewCreator viewCreator;
 
-  /** am I shutting down? */
+  /**
+   * am I shutting down?
+   */
   private volatile boolean isStopping;
 
-  /** state of collected artifacts during discovery */
+  /**
+   * state of collected artifacts during discovery
+   */
   final SearchState searchState = new SearchState();
 
-  /** a collection used to detect unit testing */
+  /**
+   * a collection used to detect unit testing
+   */
   Set<String> unitTesting = new HashSet<>();
-  
-  /** a test hook to make this member unresponsive */
+
+  /**
+   * a test hook to make this member unresponsive
+   */
   private volatile boolean playingDead;
-  
-  /** the view where quorum was most recently lost */
+
+  /**
+   * the view where quorum was most recently lost
+   */
   NetView quorumLostView;
 
-  /** a flag to mark a coordinator's viewCreator for shutdown */
+  /**
+   * a flag to mark a coordinator's viewCreator for shutdown
+   */
   private boolean markViewCreatorForShutdown = false;
 
   static class SearchState {
@@ -182,13 +244,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   /**
    * attempt to join the distributed system
    * loop
-   *   send a join request to a locator & get a response
-   *
+   * send a join request to a locator & get a response
+   * <p>
    * If the response indicates there's no coordinator it
    * will contain a set of members that have recently contacted
    * it.  The "oldest" member is selected as the coordinator
    * based on ID sort order.
-   * 
+   *
    * @return true if successful, false if not
    */
   public boolean join() {
@@ -372,12 +434,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         || !currentView.contains(mbr)) {
       return true;
     }
-    synchronized(removedMembers) {
+    synchronized (removedMembers) {
       if (removedMembers.contains(mbr)) {
         return true;
       }
     }
-    synchronized(leftMembers) {
+    synchronized (leftMembers) {
       if (leftMembers.contains(mbr)) {
         return true;
       }
@@ -419,13 +481,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       return;
     }
 
-//      Remove JoinResponseMessage to fix GEODE-870
-//        if (!this.localAddress.getNetMember().preferredForCoordinator() &&
-//            incomingRequest.getMemberID().getNetMember().preferredForCoordinator()){
-//          JoinResponseMessage joinResponseMessage = new JoinResponseMessage(incomingRequest.getMemberID(), currentView, true);
-//          services.getMessenger().send(joinResponseMessage);
-//          return;
-//        }
+    //      Remove JoinResponseMessage to fix GEODE-870
+//    if (!this.localAddress.getNetMember().preferredForCoordinator() &&
+//        incomingRequest.getMemberID().getNetMember().preferredForCoordinator()) {
+//      JoinResponseMessage joinResponseMessage = new JoinResponseMessage(incomingRequest.getMemberID(), currentView, true);
+//      services.getMessenger().send(joinResponseMessage);
+//      return;
+//    }
     recordViewRequest(incomingRequest);
   }
 
@@ -449,8 +511,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     InternalDistributedMember mbr = incomingRequest.getMemberID();
 
     if (logger.isDebugEnabled()) {
-      logger.debug("JoinLeave.processLeaveRequest invoked.  isCoordinator="+isCoordinator+ "; isStopping="+isStopping
-          +"; cancelInProgress="+ services.getCancelCriterion().isCancelInProgress());
+      logger.debug("JoinLeave.processLeaveRequest invoked.  isCoordinator=" + isCoordinator + "; isStopping=" + isStopping
+          + "; cancelInProgress=" + services.getCancelCriterion().isCancelInProgress());
     }
 
     if (!v.contains(mbr) && mbr.getVmViewId() < v.getViewId()) {
@@ -472,12 +534,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         check.removeAll(removedMembers);
         check.addCrashedMembers(removedMembers);
       }
-      synchronized(leftMembers) {
+      synchronized (leftMembers) {
         leftMembers.add(mbr);
         check.removeAll(leftMembers);
       }
       if (check.getCoordinator().equals(localAddress)) {
-        synchronized(viewInstallationLock) {
+        synchronized (viewInstallationLock) {
           becomeCoordinator(incomingRequest.getMemberID());
         }
       }
@@ -516,8 +578,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     if (!fromMe) {
       logger.info("Membership received a request to remove " + mbr
-        + " from " + incomingRequest.getSender()
-        + " reason="+incomingRequest.getReason());
+          + " from " + incomingRequest.getSender()
+          + " reason=" + incomingRequest.getReason());
     }
 
     if (mbr.equals(this.localAddress)) {
@@ -754,7 +816,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     if (preparing) {
       logger.debug("waiting for view responses");
 
-      Set<InternalDistributedMember> failedToRespond = rp.waitForResponses();
+      Set<InternalDistributedMember> failedToRespond = viewReplyProcessor.waitForResponses();
 
       logger.info("finished waiting for responses to view preparation");
 
@@ -775,7 +837,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     return true;
   }
 
-  private void processViewMessage(final InstallViewMessage installViewMessage) {
+  private void processViewMessage(final InstallViewMessage m) {
 
     NetView view = m.getView();
 
@@ -789,7 +851,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     if (!this.isJoined) {
       // if we're still waiting for a join response and we're in this view we
       // should install the view so join() can finish its work
-      for (InternalDistributedMember mbr: view.getMembers()) {
+      for (InternalDistributedMember mbr : view.getMembers()) {
         if (localAddress.compareTo(mbr) == 0) {
           viewContainsMyUnjoinedAddress = true;
           break;
@@ -940,7 +1002,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     }
     InternalDistributedMember coord = null;
     boolean coordIsNoob = true;
-    for (; it.hasNext();) {
+    for (; it.hasNext(); ) {
       InternalDistributedMember mbr = it.next();
       if (!state.alreadyTried.contains(mbr)) {
         boolean mbrIsNoob = (mbr.getVmViewId() < 0);
@@ -1603,13 +1665,17 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       }
     }
 
-    /** call with synchronized(this) */
+    /**
+     * call with synchronized(this)
+     */
     private void stopWaitingFor(InternalDistributedMember mbr) {
       notRepliedYet.remove(mbr);
       checkIfDone();
     }
 
-    /** call with synchronized(this) */
+    /**
+     * call with synchronized(this)
+     */
     private void checkIfDone() {
       if (notRepliedYet.isEmpty() || (pendingRemovals != null && pendingRemovals.containsAll(notRepliedYet))) {
         logger.debug("All anticipated view responses received - notifying waiting thread");
@@ -1828,7 +1894,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       sendInitialView();
       long okayToCreateView = System.currentTimeMillis() + requestCollectionInterval;
       try {
-        for (;;) {
+        for (; ; ) {
           synchronized (viewRequests) {
             if (shutdown) {
               return;
@@ -2168,8 +2234,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
      *
      * @param mbrs
      */
-    private void removeHealthyMembers(final Collection<InternalDistributedMember> mbrs) throws InterruptedException {
-      List<Callable<InternalDistributedMember>> checkers = new ArrayList<Callable<InternalDistributedMember>>(mbrs.size());
+    private void removeHealthyMembers(final Set<InternalDistributedMember> suspects) throws InterruptedException {
+      List<Callable<InternalDistributedMember>> checkers = new ArrayList<Callable<InternalDistributedMember>>(suspects.size());
 
       Set<InternalDistributedMember> newRemovals = new HashSet<>();
       Set<InternalDistributedMember> newLeaves = new HashSet<>();
@@ -2197,6 +2263,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
             }
             return mbr;
           }
+
           @Override
           public String toString() {
             return mbr.toString();
@@ -2228,7 +2295,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         // now wait for the tasks to do their work
         long waitTime = giveUpTime - System.currentTimeMillis();
         synchronized (viewRequests) {
-          while ( waitTime > 0 ) {
+          while (waitTime > 0) {
             logger.debug("removeHealthyMembers: mbrs" + suspects.size());
 
             filterMembers(suspects, newRemovals, REMOVE_MEMBER_REQUEST);
@@ -2237,7 +2304,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
             suspects.removeAll(newLeaves);
 
-            if(suspects.isEmpty() || newRemovals.containsAll(suspects)) {
+            if (suspects.isEmpty() || newRemovals.containsAll(suspects)) {
               break;
             }
 
@@ -2252,19 +2319,20 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     /**
      * This gets pending requests and returns the IDs of any that are in the given collection
-     * @param mbrs collection of IDs to search for
+     *
+     * @param mbrs            collection of IDs to search for
      * @param matchingMembers collection to store matching IDs in
-     * @param requestType leave/remove/join
+     * @param requestType     leave/remove/join
      */
     protected void filterMembers(Collection<InternalDistributedMember> mbrs, Set<InternalDistributedMember> matchingMembers, short requestType) {
       Set<InternalDistributedMember> requests = getPendingRequestIDs(requestType);
 
-      if(!requests.isEmpty()) {
+      if (!requests.isEmpty()) {
         logger.debug("filterMembers: processing " + requests.size() + " requests for type " + requestType);
         Iterator<InternalDistributedMember> itr = requests.iterator();
-        while(itr.hasNext()) {
+        while (itr.hasNext()) {
           InternalDistributedMember memberID = itr.next();
-          if(mbrs.contains(memberID)) {
+          if (mbrs.contains(memberID)) {
             testFlagForRemovalRequest = true;
             matchingMembers.add(memberID);
           }
@@ -2272,8 +2340,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       }
     }
 
-    private <T> List<Future<T>> submitAll ( ExecutorService executor, Collection<? extends Callable<T> > tasks ) {
-      List<Future<T>> result = new ArrayList<Future<T>>( tasks.size() );
+    private <T> List<Future<T>> submitAll(ExecutorService executor, Collection<? extends Callable<T>> tasks) {
+      List<Future<T>> result = new ArrayList<Future<T>>(tasks.size());
 
       for (Callable<T> task : tasks) {
         result.add(executor.submit(task));
@@ -2288,7 +2356,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   }
 
   boolean checkIfAvailable(InternalDistributedMember fmbr) {
- // return the member id if it fails health checks
+    // return the member id if it fails health checks
     logger.info("checking state of member " + fmbr);
     if (services.getHealthMonitor().checkIfAvailable(fmbr, "Member failed to acknowledge a membership view", false)) {
       logger.info("member " + fmbr + " passed availability check");

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2bff98b1/geode-core/src/test/java/com/gemstone/gemfire/cache30/CacheSerializableRunnable.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/cache30/CacheSerializableRunnable.java b/geode-core/src/test/java/com/gemstone/gemfire/cache30/CacheSerializableRunnable.java
index 8b80def..3745928 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/cache30/CacheSerializableRunnable.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/cache30/CacheSerializableRunnable.java
@@ -48,13 +48,25 @@ public abstract class CacheSerializableRunnable
   }
 
   /**
+   * Creates a new <code>CacheSerializableRunnable</code> with the
+   * given name
+   */
+  public CacheSerializableRunnable(String name,Object[] args) {
+    super(name);
+    this.args = args;
+  }
+
+  /**
    * Invokes the {@link #run2} method and will wrap any {@link
    * CacheException} thrown by <code>run2</code> in a {@link
    * CacheSerializableRunnableException}. 
    */
   public final void run() {
     try {
-      run2();
+      if(args == null){
+      run2();}else{
+        run3();
+      }
 
     } catch (CacheException ex) {
       String s = "While invoking \"" + this + "\"";
@@ -96,6 +108,8 @@ public abstract class CacheSerializableRunnable
    */
   public abstract void run2() throws CacheException;
 
+  public void run3() throws CacheException{}
+
   /////////////////////////  Inner Classes  /////////////////////////
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2bff98b1/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
index 4caf815..2e24cfd 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
@@ -51,7 +51,7 @@ public abstract class SerializableRunnable implements SerializableRunnableIF {
   private static final long serialVersionUID = 7584289978241650456L;
 
   private String name;
-  private Object[] args;
+  protected Object[] args;
 
   public SerializableRunnable() {
     this.name = null;


[7/7] incubator-geode git commit: GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination

Posted by ud...@apache.org.
GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/9e241d5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/9e241d5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/9e241d5d

Branch: refs/heads/feature/GEODE-870
Commit: 9e241d5d08be723028b2a4d9a6a779e2e0c73131
Parents: 83a6dc3
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Wed Feb 10 12:29:02 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../membership/gms/membership/GMSJoinLeave.java   | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/9e241d5d/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index b246344..56b644c 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -195,24 +195,24 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       }
 
       SearchState state = searchState;
-      
-      long locatorWaitTime = ((long)services.getConfig().getLocatorWaitTime()) * 1000L;
+
+      long locatorWaitTime = ((long) services.getConfig().getLocatorWaitTime()) * 1000L;
       long timeout = services.getConfig().getJoinTimeout();
       logger.debug("join timeout is set to {}", timeout);
-      long retrySleep =  JOIN_RETRY_SLEEP;
+      long retrySleep = JOIN_RETRY_SLEEP;
       long startTime = System.currentTimeMillis();
       long locatorGiveUpTime = startTime + locatorWaitTime;
       long giveupTime = startTime + timeout;
 
-      for (int tries=0; !this.isJoined && !this.isStopping; tries++) {
+      for (int tries = 0; !this.isJoined && !this.isStopping; tries++) {
         logger.debug("searching for the membership coordinator");
         boolean found = findCoordinator();
         if (found) {
           logger.debug("found possible coordinator {}", state.possibleCoordinator);
           if (localAddress.getNetMember().preferredForCoordinator()
               && state.possibleCoordinator.equals(this.localAddress)) {
-            if (tries > 2 || System.currentTimeMillis() < giveupTime ) {
-              synchronized(viewInstallationLock) {
+            if (tries > 2 || System.currentTimeMillis() < giveupTime) {
+              synchronized (viewInstallationLock) {
                 becomeCoordinator();
               }
               return true;
@@ -713,6 +713,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     if (preparing) {
       this.preparedView = view;
     } else {
+      //Added a check in the view processor to turn off the ViewCreator
+      //if another server is the coordinator - GEODE-870
       if(!localAddress.equals(view.getCoordinator()) && getViewCreator() != null)
       {
         stopCoordinatorServices();
@@ -1075,12 +1077,16 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
   /**
    * for testing, do not use in any other case as it is not thread safe
+   *
+   * @param req
    */
   JoinResponseMessage[] getJoinResponseMessage() {
     return joinResponse;
   }
+
   /***
    * for testing purpose
+   *
    * @param jrm
    */
   void setJoinResponseMessage(JoinResponseMessage jrm) {


[4/7] incubator-geode git commit: GEODE-870: Rejecting of old view

Posted by ud...@apache.org.
GEODE-870: Rejecting of old view


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/036f2205
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/036f2205
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/036f2205

Branch: refs/heads/feature/GEODE-870
Commit: 036f22059aac18eeb3cb84c016d671d6bb8b7343
Parents: d50623b
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Mon Feb 1 14:08:32 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../gms/messages/ViewRejectMessage.java         |  96 +++
 .../membership/gms/membership/GMSJoinLeave.java |  92 +--
 .../internal/DataSerializableFixedID.java       |   3 +-
 .../gemfire/distributed/LocatorDUnitTest.java   | 816 +++++++++++--------
 .../test/dunit/SerializableRunnable.java        |  12 +-
 5 files changed, 608 insertions(+), 411 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/036f2205/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
new file mode 100755
index 0000000..e5bf9e2
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed.internal.membership.gms.messages;
+
+import com.gemstone.gemfire.DataSerializer;
+import com.gemstone.gemfire.distributed.internal.DistributionManager;
+import com.gemstone.gemfire.distributed.internal.HighPriorityDistributionMessage;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.distributed.internal.membership.NetView;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class ViewRejectMessage extends HighPriorityDistributionMessage {
+
+  private int viewId;
+  private NetView rejectedView;
+  private String reason;
+
+  public ViewRejectMessage(InternalDistributedMember recipient, int viewId, NetView rejectedView, String reason) {
+    super();
+    setRecipient(recipient);
+    this.viewId = viewId;
+    this.rejectedView = rejectedView;
+    this.reason = reason;
+  }
+
+  public ViewRejectMessage() {
+    // no-arg constructor for serialization
+  }
+  
+  public int getViewId() {
+    return viewId;
+  }
+  
+  public NetView getRejectedView() {
+    return this.rejectedView;
+  }
+  
+
+  @Override
+  public int getDSFID() {
+    // TODO Auto-generated method stub
+    return VIEW_REJECT_MESSAGE;
+  }
+
+  public String getReason() {
+    return reason;
+  }
+
+  @Override
+  public int getProcessorType() {
+    return 0;
+  }
+
+  @Override
+  public void process(DistributionManager dm) {
+    throw new IllegalStateException("this message is not intended to execute in a thread pool");
+  }
+
+  @Override
+  public void toData(DataOutput out) throws IOException {
+    super.toData(out);
+    out.writeInt(this.viewId);
+    DataSerializer.writeObject(this.rejectedView, out);
+  }
+
+  @Override
+  public void fromData(DataInput in) throws IOException, ClassNotFoundException {
+    super.fromData(in);
+    this.viewId = in.readInt();
+    this.rejectedView = DataSerializer.readObject(in);
+  }
+  
+  @Override
+  public String toString() {
+    String s = getSender() == null? getRecipientsDescription() : ""+getSender();
+    return "ViewRejectMessage("+s+"; "+this.viewId+";  rejectedView="+this.rejectedView +")";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/036f2205/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index 5d34041..c7eacfa 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -16,41 +16,6 @@
  */
 package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
 
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.FIND_COORDINATOR_REQ;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.FIND_COORDINATOR_RESP;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.INSTALL_VIEW_MESSAGE;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.JOIN_REQUEST;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.JOIN_RESPONSE;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.LEAVE_REQUEST_MESSAGE;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.NETWORK_PARTITION_MESSAGE;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.REMOVE_MEMBER_REQUEST;
-import static com.gemstone.gemfire.internal.DataSerializableFixedID.VIEW_ACK_MESSAGE;
-import static com.gemstone.gemfire.distributed.internal.membership.gms.ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TimerTask;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.logging.log4j.Logger;
-
 import com.gemstone.gemfire.GemFireConfigException;
 import com.gemstone.gemfire.SystemConnectException;
 import com.gemstone.gemfire.distributed.DistributedMember;
@@ -68,24 +33,26 @@ import com.gemstone.gemfire.distributed.internal.membership.gms.interfaces.JoinL
 import com.gemstone.gemfire.distributed.internal.membership.gms.interfaces.MessageHandler;
 import com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorRequest;
 import com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorResponse;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.HasMemberID;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.InstallViewMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.JoinRequestMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.JoinResponseMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.LeaveRequestMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.NetworkPartitionMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.RemoveMemberMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.ViewAckMessage;
+import com.gemstone.gemfire.distributed.internal.membership.gms.messages.*;
 import com.gemstone.gemfire.distributed.internal.tcpserver.TcpClient;
 import com.gemstone.gemfire.internal.Version;
 import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
 import com.gemstone.gemfire.security.AuthenticationFailedException;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.gemstone.gemfire.distributed.internal.membership.gms.ServiceConfig.MEMBER_REQUEST_COLLECTION_INTERVAL;
+import static com.gemstone.gemfire.internal.DataSerializableFixedID.*;
 
 /**
  * GMSJoinLeave handles membership communication with other processes in the
  * distributed system.  It replaces the JGroups channel membership services
  * that Geode formerly used for this purpose.
- * 
  */
 public class GMSJoinLeave implements JoinLeave, MessageHandler {
   
@@ -221,7 +188,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     try {
       if (Boolean.getBoolean(BYPASS_DISCOVERY_PROPERTY)) {
-        synchronized(viewInstallationLock) {
+        synchronized (viewInstallationLock) {
           becomeCoordinator();
         }
         return true;
@@ -391,7 +358,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     }
     return response;
   }
-  
+
   @Override
   public boolean isMemberLeaving(DistributedMember mbr) {
     if (getPendingRequestIDs(LEAVE_REQUEST_MESSAGE).contains(mbr)
@@ -471,8 +438,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       recordViewRequest(incomingRequest);
       return;
     }
-    
-    
+
     InternalDistributedMember mbr = incomingRequest.getMemberID();
 
     if (logger.isDebugEnabled()) {
@@ -543,7 +509,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     if (!fromMe) {
       logger.info("Membership received a request to remove " + mbr
-        + " from " + incomingRequest.getSender() 
+        + " from " + incomingRequest.getSender()
         + " reason="+incomingRequest.getReason());
     }
 
@@ -630,7 +596,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   public void delayViewCreationForTest(int millis) {
     requestCollectionInterval = millis;
   }
-  
 
   /**
    * Transitions this member into the coordinator role.  This must
@@ -872,6 +837,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   void setTcpClientWrapper(TcpClientWrapper tcpClientWrapper) {
     this.tcpClientWrapper = tcpClientWrapper;
   }
+
   /**
    * This contacts the locators to find out who the current coordinator is.
    * All locators are contacted. If they don't agree then we choose the oldest
@@ -940,8 +906,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         }
       }
     } while (!anyResponses && System.currentTimeMillis() < giveUpTime);
-    
-    
+
     if (coordinators.isEmpty()) {
       return false;
     }
@@ -1523,6 +1488,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
   /***
    * test method
+   *
    * @return ViewReplyProcessor
    */
   protected ViewReplyProcessor getPrepareViewReplyProcessor() {
@@ -1558,6 +1524,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     boolean isWaiting(){
       return waiting;
     }
+
     synchronized void processPendingRequests(Set<InternalDistributedMember> pendingLeaves, Set<InternalDistributedMember> pendingRemovals) {
       // there's no point in waiting for members who have already
       // requested to leave or who have been declared crashed.
@@ -1711,7 +1678,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         services.getMessenger().sendUnreliably(msg);
       }
     }
-    
   }
 
   class ViewCreator extends Thread {
@@ -2094,7 +2060,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         }
 
         logger.debug("unresponsive members that could not be reached: {}", unresponsive);
-        
+
         List<InternalDistributedMember> failures = new ArrayList<>(currentView.getCrashedMembers().size() + unresponsive.size());
 
         if (conflictingView != null && !conflictingView.getCreator().equals(localAddress) && conflictingView.getViewId() > newView.getViewId()
@@ -2188,10 +2154,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
       filterMembers(suspects, newRemovals, REMOVE_MEMBER_REQUEST);
       filterMembers(suspects, newLeaves, LEAVE_REQUEST_MESSAGE);
-      newRemovals.removeAll(newLeaves);  // if we received a Leave req the member is "healthy" 
-      
+      newRemovals.removeAll(newLeaves);  // if we received a Leave req the member is "healthy"
+
       suspects.removeAll(newLeaves);
-      
+
       for (InternalDistributedMember mbr : suspects) {
         if (newRemovals.contains(mbr) || newLeaves.contains(mbr)) {
           continue; // no need to check this member - it's already been checked or is leaving
@@ -2215,7 +2181,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           }
         });
       }
-      
+
       if (checkers.isEmpty()) {
         logger.debug("all unresponsive members are already scheduled to be removed");
         return;
@@ -2236,19 +2202,19 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         long giveUpTime = System.currentTimeMillis() + viewAckTimeout;
         // submit the tasks that will remove dead members from the suspects collection
         submitAll(svc, checkers);
-        
+
         // now wait for the tasks to do their work
         long waitTime = giveUpTime - System.currentTimeMillis();
         synchronized (viewRequests) {
           while ( waitTime > 0 ) {
             logger.debug("removeHealthyMembers: mbrs" + suspects.size());
-            
+
             filterMembers(suspects, newRemovals, REMOVE_MEMBER_REQUEST);
             filterMembers(suspects, newLeaves, LEAVE_REQUEST_MESSAGE);
             newRemovals.removeAll(newLeaves);
-            
+
             suspects.removeAll(newLeaves);
-            
+
             if(suspects.isEmpty() || newRemovals.containsAll(suspects)) {
               break;
             }
@@ -2270,7 +2236,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
      */
     protected void filterMembers(Collection<InternalDistributedMember> mbrs, Set<InternalDistributedMember> matchingMembers, short requestType) {
       Set<InternalDistributedMember> requests = getPendingRequestIDs(requestType);
-      
+
       if(!requests.isEmpty()) {
         logger.debug("filterMembers: processing " + requests.size() + " requests for type " + requestType);
         Iterator<InternalDistributedMember> itr = requests.iterator();

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/036f2205/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
index 7b263bf..22ac457 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
@@ -82,7 +82,8 @@ public interface DataSerializableFixedID extends SerializationVersions {
     case FOO:
       return new FOO(in);
   */
-  
+  public static final short VIEW_REJECT_MESSAGE = -158;
+
   public static final short NETWORK_PARTITION_MESSAGE = -157;
   public static final short SUSPECT_MEMBERS_MESSAGE = -156;
   

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/036f2205/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
index 25ce97e..1917692 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
@@ -33,12 +33,7 @@ import com.gemstone.gemfire.cache.Cache;
 import com.gemstone.gemfire.cache.CacheFactory;
 import com.gemstone.gemfire.cache.Region;
 import com.gemstone.gemfire.cache.RegionShortcut;
-import com.gemstone.gemfire.distributed.internal.DistributionConfig;
-import com.gemstone.gemfire.distributed.internal.DistributionException;
-import com.gemstone.gemfire.distributed.internal.DistributionManager;
-import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
-import com.gemstone.gemfire.distributed.internal.InternalLocator;
-import com.gemstone.gemfire.distributed.internal.MembershipListener;
+import com.gemstone.gemfire.distributed.internal.*;
 import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
 import com.gemstone.gemfire.distributed.internal.membership.MembershipManager;
 import com.gemstone.gemfire.distributed.internal.membership.MembershipTestHook;
@@ -180,8 +175,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         service.lock("foo2", 0, 0);
       }
     });
-      
-      
+
     // cause elder failover.  vm1 will become the lock grantor
     system.disconnect();
     
@@ -195,6 +189,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
             public boolean done() {
               return service.isLockGrantor();
             }
+
             @Override
             public String description() {
               return "waiting to become lock grantor after shutting down locator/grantor";
@@ -323,11 +318,11 @@ public class LocatorDUnitTest extends DistributedTestCase {
             async2.join();
             Object result1 = async1.getReturnValue();
             if (result1 instanceof Exception) {
-              throw (Exception)result1;
+              throw (Exception) result1;
             }
             Object result2 = async2.getReturnValue();
             if (result2 instanceof Exception) {
-              throw (Exception)result2;
+              throw (Exception) result2;
             }
             // verify that they found each other
             SerializableCallable verify = new SerializableCallable("verify no split-brain") {
@@ -338,7 +333,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
                 }
                 Assert.assertTrue(sys.getDM().getViewMembers().size() == 2,
                     "expected 2 members but found " + sys.getDM().getViewMembers().size()
-                    );
+                );
                 return true;
               }
             };
@@ -359,8 +354,8 @@ public class LocatorDUnitTest extends DistributedTestCase {
         loc1.invoke(r);
       }
     }
-    
   }
+
   /**
    * test lead member selection
    */
@@ -370,7 +365,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm1 = host.getVM(1);
     VM vm2 = host.getVM(2);
     VM vm3 = host.getVM(3);
-    
+
     port1 = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
     DistributedTestUtils.deleteLocatorStateFile(port1);
     final String locators = NetworkUtils.getServerHostName(host) + "[" + port1 + "]";
@@ -379,69 +374,69 @@ public class LocatorDUnitTest extends DistributedTestCase {
     properties.put("locators", locators);
     properties.put("enable-network-partition-detection", "true");
     properties.put("disable-auto-reconnect", "true");
-    
+
     File logFile = new File("");
     if (logFile.exists()) {
       logFile.delete();
     }
     Locator locator = Locator.startLocatorAndDS(port1, logFile, properties);
     try {
-    DistributedSystem sys = locator.getDistributedSystem();
-    
-    Object[] connectArgs = new Object[]{ properties };
-    
-    SerializableRunnable disconnect =
-      new SerializableRunnable("Disconnect from " + locators) {
-          public void run() {
-            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-            if (sys != null && sys.isConnected()) {
-              sys.disconnect();
+      DistributedSystem sys = locator.getDistributedSystem();
+
+      Object[] connectArgs = new Object[] { properties };
+
+      SerializableRunnable disconnect =
+          new SerializableRunnable("Disconnect from " + locators) {
+            public void run() {
+              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+              if (sys != null && sys.isConnected()) {
+                sys.disconnect();
+              }
             }
-          }
-        };
-        
-    assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
-    
-    // connect three vms and then watch the lead member selection as they
-    // are disconnected/reconnected
-    properties.put("name", "vm1");
-    DistributedMember mem1 = (DistributedMember)vm1.invoke(this.getClass(),
-        "getDistributedMember", connectArgs);
-    
-//    assertTrue(MembershipManagerHelper.getLeadMember(sys) != null);
-    assertLeadMember(mem1, sys, 5000);
-    
-    properties.put("name", "vm2");
-    DistributedMember mem2 = (DistributedMember)vm2.invoke(this.getClass(),
-        "getDistributedMember", connectArgs);
-    assertLeadMember(mem1, sys, 5000);
-    
-    properties.put("name", "vm3");
-    DistributedMember mem3 = (DistributedMember)vm3.invoke(this.getClass(),
-        "getDistributedMember", connectArgs);
-    assertLeadMember(mem1, sys, 5000);
-    
-    // after disconnecting the first vm, the second one should become the leader
-    vm1.invoke(disconnect);
-    MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
-    assertLeadMember(mem2, sys, 5000);
-    
-    properties.put("name", "vm1");
-    mem1 = (DistributedMember)vm1.invoke(this.getClass(),
-        "getDistributedMember", connectArgs);
-    assertLeadMember(mem2, sys, 5000);
-    
-    vm2.invoke(disconnect);
-    MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem2);
-    assertLeadMember(mem3, sys, 5000);
-    
-    vm1.invoke(disconnect);
-    MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
-    assertLeadMember(mem3, sys, 5000);
+          };
 
-    vm3.invoke(disconnect);
-    MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem3);
-    assertLeadMember(null, sys, 5000);
+      assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
+
+      // connect three vms and then watch the lead member selection as they
+      // are disconnected/reconnected
+      properties.put("name", "vm1");
+      DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(),
+          "getDistributedMember", connectArgs);
+
+      //    assertTrue(MembershipManagerHelper.getLeadMember(sys) != null);
+      assertLeadMember(mem1, sys, 5000);
+
+      properties.put("name", "vm2");
+      DistributedMember mem2 = (DistributedMember) vm2.invoke(this.getClass(),
+          "getDistributedMember", connectArgs);
+      assertLeadMember(mem1, sys, 5000);
+
+      properties.put("name", "vm3");
+      DistributedMember mem3 = (DistributedMember) vm3.invoke(this.getClass(),
+          "getDistributedMember", connectArgs);
+      assertLeadMember(mem1, sys, 5000);
+
+      // after disconnecting the first vm, the second one should become the leader
+      vm1.invoke(disconnect);
+      MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
+      assertLeadMember(mem2, sys, 5000);
+
+      properties.put("name", "vm1");
+      mem1 = (DistributedMember) vm1.invoke(this.getClass(),
+          "getDistributedMember", connectArgs);
+      assertLeadMember(mem2, sys, 5000);
+
+      vm2.invoke(disconnect);
+      MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem2);
+      assertLeadMember(mem3, sys, 5000);
+
+      vm1.invoke(disconnect);
+      MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
+      assertLeadMember(mem3, sys, 5000);
+
+      vm3.invoke(disconnect);
+      MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem3);
+      assertLeadMember(null, sys, 5000);
 
     } finally {
       locator.stop();
@@ -458,13 +453,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
         }
         return (lead == null);
       }
+
       public String description() {
         return null;
       }
     };
     Wait.waitForCriterion(ev, timeout, 200, true);
   }
-  
+
   /**
    * test lead member and coordinator failure with network partition detection
    * enabled.  It would be nice for this test to have more than two "server"
@@ -527,44 +523,43 @@ public class LocatorDUnitTest extends DistributedTestCase {
           }
         }
       });
-      
-      Object[] connectArgs = new Object[]{ properties };
-      
-      SerializableRunnable crashLocator =
-        new SerializableRunnable("Crash locator") {
-          public void run() {
-            Locator loc = Locator.getLocators().iterator().next();
-            DistributedSystem msys = loc.getDistributedSystem();
-            MembershipManagerHelper.crashDistributedSystem(msys);
-            loc.stop();
-          }
-      };
 
+      Object[] connectArgs = new Object[] { properties };
+
+      SerializableRunnable crashLocator =
+          new SerializableRunnable("Crash locator") {
+            public void run() {
+              Locator loc = Locator.getLocators().iterator().next();
+              DistributedSystem msys = loc.getDistributedSystem();
+              MembershipManagerHelper.crashDistributedSystem(msys);
+              loc.stop();
+            }
+          };
 
       assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
-      
-//      properties.put("log-level", getDUnitLogLevel());
-      
-      DistributedMember mem1 = (DistributedMember)vm1.invoke(this.getClass(),
+
+      //      properties.put("log-level", getDUnitLogLevel());
+
+      DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
       vm2.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
       assertLeadMember(mem1, sys, 5000);
-      
+
       assertEquals(sys.getDistributedMember(), MembershipManagerHelper.getCoordinator(sys));
-      
+
       // crash the second vm and the locator.  Should be okay
       DistributedTestUtils.crashDistributedSystem(vm2);
       locvm.invoke(crashLocator);
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm1.invoke(() -> LocatorDUnitTest.isSystemConnected()));
       
       // ensure quorumLost is properly invoked
-      DistributionManager dm = (DistributionManager)((InternalDistributedSystem)sys).getDistributionManager();
+      DistributionManager dm = (DistributionManager) ((InternalDistributedSystem) sys).getDistributionManager();
       MyMembershipListener listener = new MyMembershipListener();
       dm.addMembershipListener(listener);
-  
+
       // disconnect the first vm and demonstrate that the third vm and the
       // locator notice the failure and exit
       DistributedTestUtils.crashDistributedSystem(vm1);
@@ -579,6 +574,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         public boolean done() {
           return !sys.isConnected();
         }
+
         public String description() {
           return null;
         }
@@ -590,20 +586,18 @@ public class LocatorDUnitTest extends DistributedTestCase {
       // quorumLost should be invoked if we get a ForcedDisconnect in this situation
       assertTrue("expected quorumLost to be invoked", listener.quorumLostInvoked);
       assertTrue("expected suspect processing initiated by TCPConduit", listener.suspectReasons.contains(Connection.INITIATING_SUSPECT_PROCESSING));
-    }
-    finally {
+    } finally {
       if (locator != null) {
         locator.stop();
       }
       LogWriter bLogger =
-        new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
+          new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
       bLogger.info("<ExpectedException action=remove>service failure</ExpectedException>");
       bLogger.info("<ExpectedException action=remove>java.net.ConnectException</ExpectedException>");
       bLogger.info("<ExpectedException action=remove>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
       disconnectAllFromDS();
     }
   }
-  
 
   /**
    * test lead member failure and normal coordinator shutdown with network partition detection
@@ -662,31 +656,31 @@ public class LocatorDUnitTest extends DistributedTestCase {
           }
         }
       });
-      
-      Object[] connectArgs = new Object[]{ properties };
-      
+
+      Object[] connectArgs = new Object[] { properties };
+
       SerializableRunnable crashSystem =
-        new SerializableRunnable("Crash system") {
-          public void run() {
-            DistributedSystem msys = InternalDistributedSystem.getAnyInstance();
-            msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
-            MembershipManagerHelper.crashDistributedSystem(msys);
-          }
-      };
+          new SerializableRunnable("Crash system") {
+            public void run() {
+              DistributedSystem msys = InternalDistributedSystem.getAnyInstance();
+              msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
+              MembershipManagerHelper.crashDistributedSystem(msys);
+            }
+          };
 
       assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
-      
-      DistributedMember mem1 = (DistributedMember)vm1.invoke(this.getClass(),
+
+      DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
-      DistributedMember mem2 = (DistributedMember)vm2.invoke(this.getClass(),
+      DistributedMember mem2 = (DistributedMember) vm2.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
 
       assertEquals(mem1, MembershipManagerHelper.getLeadMember(sys));
-      
+
       assertEquals(sys.getDistributedMember(), MembershipManagerHelper.getCoordinator(sys));
-      
+
       MembershipManagerHelper.inhibitForcedDisconnectLogging(true);
 
       // crash the lead vm. Should be okay
@@ -696,7 +690,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       
       assertTrue("Distributed system should not have disconnected",
           isSystemConnected());
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
       
@@ -705,13 +699,13 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       // stop the locator normally.  This should also be okay
       locator.stop();
-      
+
       if (!Locator.getLocators().isEmpty()) {
         // log this for debugging purposes before throwing assertion error
         com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().warning("found locator " + Locator.getLocators().iterator().next());
       }
       assertTrue("locator is not stopped", Locator.getLocators().isEmpty());
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
       
@@ -724,7 +718,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       		mem2, vm2.invoke(() -> LocatorDUnitTest.getLeadMember()));
       
       SerializableRunnable disconnect =
-        new SerializableRunnable("Disconnect from " + locators) {
+          new SerializableRunnable("Disconnect from " + locators) {
             public void run() {
               DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
               if (sys != null && sys.isConnected()) {
@@ -737,8 +731,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       // locator notice the failure and exit
       vm2.invoke(disconnect);
       locvm.invoke(stopLocator);
-    }
-    finally {
+    } finally {
       MembershipManagerHelper.inhibitForcedDisconnectLogging(false);
       if (locator != null) {
         locator.stop();
@@ -792,7 +785,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     properties.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
     SerializableRunnable stopLocator = getStopLocatorRunnable();
-    
+
     try {
       final String uname = getUniqueName();
       File logFile = new File("");
@@ -803,51 +796,50 @@ public class LocatorDUnitTest extends DistributedTestCase {
           File lf = new File("");
           try {
             Locator.startLocatorAndDS(port2, lf, properties);
-          }
-          catch (IOException ios) {
+          } catch (IOException ios) {
             com.gemstone.gemfire.test.dunit.Assert.fail("Unable to start locator2", ios);
           }
         }
       });
-      
-      Object[] connectArgs = new Object[]{ properties };
-      
+
+      Object[] connectArgs = new Object[] { properties };
+
       SerializableRunnable crashSystem =
-        new SerializableRunnable("Crash system") {
-          public void run() {
-            DistributedSystem msys = InternalDistributedSystem.getAnyInstance();
-            msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>Possible loss of quorum</ExpectedException>");
-            hook = new TestHook();
-            MembershipManagerHelper.getMembershipManager(msys).registerTestHook(hook);
-            try {
-              MembershipManagerHelper.crashDistributedSystem(msys);
-            } finally {
-              hook.reset();
+          new SerializableRunnable("Crash system") {
+            public void run() {
+              DistributedSystem msys = InternalDistributedSystem.getAnyInstance();
+              msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>Possible loss of quorum</ExpectedException>");
+              hook = new TestHook();
+              MembershipManagerHelper.getMembershipManager(msys).registerTestHook(hook);
+              try {
+                MembershipManagerHelper.crashDistributedSystem(msys);
+              } finally {
+                hook.reset();
+              }
             }
-          }
-      };
+          };
 
       assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
-      
-      final DistributedMember mem1 = (DistributedMember)vm1.invoke(this.getClass(),
+
+      final DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
-      final DistributedMember mem2 = (DistributedMember)vm2.invoke(this.getClass(),
+      final DistributedMember mem2 = (DistributedMember) vm2.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
 
       assertEquals(mem1, MembershipManagerHelper.getLeadMember(sys));
-      
+
       assertEquals(sys.getDistributedMember(), MembershipManagerHelper.getCoordinator(sys));
-      
+
       // crash the lead vm. Should be okay. it should hang in test hook thats
       // why call is asynchronous.
       //vm1.invokeAsync(crashSystem);
 
       assertTrue("Distributed system should not have disconnected",
           isSystemConnected());
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
       
@@ -860,12 +852,12 @@ public class LocatorDUnitTest extends DistributedTestCase {
       
       // request member removal for first peer from second peer.
       vm2.invoke(new SerializableRunnable("Request Member Removal") {
-        
+
         @Override
         public void run() {
           DistributedSystem msys = InternalDistributedSystem.getAnyInstance();
           MembershipManager mmgr = MembershipManagerHelper.getMembershipManager(msys);
-          
+
           // check for shutdown cause in MembershipManager. Following call should
           // throw DistributedSystemDisconnectedException which should have cause as
           // ForceDisconnectException.
@@ -885,8 +877,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         }
       });
 
-    }
-    finally {
+    } finally {
       if (locator != null) {
         locator.stop();
       }
@@ -930,7 +921,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     properties.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
     SerializableRunnable disconnect =
-      new SerializableRunnable("Disconnect from " + locators) {
+        new SerializableRunnable("Disconnect from " + locators) {
           public void run() {
             DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
             if (sys != null && sys.isConnected()) {
@@ -940,11 +931,11 @@ public class LocatorDUnitTest extends DistributedTestCase {
           }
         };
     SerializableRunnable expectedException =
-      new SerializableRunnable("Add expected exceptions") {
-        public void run() {
-          MembershipManagerHelper.inhibitForcedDisconnectLogging(true);
-        }
-    };
+        new SerializableRunnable("Add expected exceptions") {
+          public void run() {
+            MembershipManagerHelper.inhibitForcedDisconnectLogging(true);
+          }
+        };
     try {
       final String uname = getUniqueName();
       locvm.invoke(new SerializableRunnable() {
@@ -952,8 +943,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
           File lf = new File("");
           try {
             Locator.startLocatorAndDS(port2, lf, properties);
-          }
-          catch (IOException ios) {
+          } catch (IOException ios) {
             com.gemstone.gemfire.test.dunit.Assert.fail("Unable to start locator1", ios);
           }
         }
@@ -963,42 +953,42 @@ public class LocatorDUnitTest extends DistributedTestCase {
       locator = Locator.startLocatorAndDS(port1, logFile, properties);
       DistributedSystem sys = locator.getDistributedSystem();
       sys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
-      Object[] connectArgs = new Object[]{ properties };
-      
+      Object[] connectArgs = new Object[] { properties };
+
       SerializableRunnable crashLocator =
-        new SerializableRunnable("Crash locator") {
-          public void run() {
-            Locator loc = Locator.getLocators().iterator().next();
-            DistributedSystem msys = loc.getDistributedSystem();
-            msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
-            msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
-            MembershipManagerHelper.crashDistributedSystem(msys);
-            loc.stop();
-          }
-      };
+          new SerializableRunnable("Crash locator") {
+            public void run() {
+              Locator loc = Locator.getLocators().iterator().next();
+              DistributedSystem msys = loc.getDistributedSystem();
+              msys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
+              msys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
+              MembershipManagerHelper.crashDistributedSystem(msys);
+              loc.stop();
+            }
+          };
 
       assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
-      
-      DistributedMember mem1 = (DistributedMember)vm1.invoke(this.getClass(),
+
+      DistributedMember mem1 = (DistributedMember) vm1.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
       vm1.invoke(expectedException);
-      DistributedMember mem2 = (DistributedMember)vm2.invoke(this.getClass(),
+      DistributedMember mem2 = (DistributedMember) vm2.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
       
       DistributedMember loc1Mbr = (DistributedMember)locvm.invoke(() -> this.getLocatorDistributedMember());
 
       assertLeadMember(mem1, sys, 5000);
-      
+
       assertEquals(loc1Mbr, MembershipManagerHelper.getCoordinator(sys));
-      
+
       // crash the lead locator.  Should be okay
       locvm.invoke(crashLocator);
       Wait.pause(10 * 1000);
 
       assertTrue("Distributed system should not have disconnected",
           sys.isConnected());
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm1.invoke(() -> LocatorDUnitTest.isSystemConnected()));
       
@@ -1016,11 +1006,10 @@ public class LocatorDUnitTest extends DistributedTestCase {
       assertEquals(sys.getDistributedMember(),
           MembershipManagerHelper.getCoordinator(sys));
       assertEquals(mem2, MembershipManagerHelper.getLeadMember(sys));
-      
-    }
-    finally {
+
+    } finally {
       vm2.invoke(disconnect);
-      
+
       if (locator != null) {
         locator.stop();
       }
@@ -1044,17 +1033,17 @@ public class LocatorDUnitTest extends DistributedTestCase {
     props.setProperty("locators", locators);
 
     final String expected = "java.net.ConnectException";
-    final String addExpected = 
-      "<ExpectedException action=add>" + expected + "</ExpectedException>";
-    final String removeExpected = 
-      "<ExpectedException action=remove>" + expected + "</ExpectedException>";
-      
+    final String addExpected =
+        "<ExpectedException action=add>" + expected + "</ExpectedException>";
+    final String removeExpected =
+        "<ExpectedException action=remove>" + expected + "</ExpectedException>";
+
     LogWriter bgexecLogger =
-          new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
+        new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
     bgexecLogger.info(addExpected);
-    
+
     boolean exceptionOccurred = true;
-    String oldValue = (String)System.getProperties().put("p2p.joinTimeout", "15000");
+    String oldValue = (String) System.getProperties().put("p2p.joinTimeout", "15000");
     try {
       DistributedSystem.connect(props);
       exceptionOccurred = false;
@@ -1065,7 +1054,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     } catch (GemFireConfigException ex) {
       String s = ex.getMessage();
       assertTrue(s.indexOf("Locator does not exist") >= 0);
-      
+
     } catch (Exception ex) {
       // if you see this fail, determine if unexpected exception is expected
       // if expected then add in a catch block for it above this catch
@@ -1105,14 +1094,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
     DistributedTestUtils.deleteLocatorStateFile(port1);
     final String locators = NetworkUtils.getServerHostName(host) + "[" + port + "]";
     final String uniqueName = getUniqueName();
-    
+
     vm0.invoke(new SerializableRunnable("Start locator " + locators) {
-        public void run() {
-          File logFile = new File("");
-          try {
-            Properties locProps = new Properties();
-            locProps.setProperty("mcast-port", "0");
-            locProps.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
+      public void run() {
+        File logFile = new File("");
+        try {
+          Properties locProps = new Properties();
+          locProps.setProperty("mcast-port", "0");
+          locProps.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
             Locator.startLocatorAndDS(port, logFile, locProps);
           } catch (IOException ex) {
@@ -1122,22 +1111,22 @@ public class LocatorDUnitTest extends DistributedTestCase {
       });
     try {
 
-    SerializableRunnable connect =
-      new SerializableRunnable("Connect to " + locators) {
-          public void run() {
-            //System.setProperty("p2p.joinTimeout", "5000");
-            Properties props = new Properties();
-            props.setProperty("mcast-port", "0");
-            props.setProperty("locators", locators);
-            DistributedSystem.connect(props);
-          }
-        };
-    vm1.invoke(connect);
-    vm2.invoke(connect);
+      SerializableRunnable connect =
+          new SerializableRunnable("Connect to " + locators) {
+            public void run() {
+              //System.setProperty("p2p.joinTimeout", "5000");
+              Properties props = new Properties();
+              props.setProperty("mcast-port", "0");
+              props.setProperty("locators", locators);
+              DistributedSystem.connect(props);
+            }
+          };
+      vm1.invoke(connect);
+      vm2.invoke(connect);
 
-    Properties props = new Properties();
-    props.setProperty("mcast-port", "0");
-    props.setProperty("locators", locators);
+      Properties props = new Properties();
+      props.setProperty("mcast-port", "0");
+      props.setProperty("locators", locators);
 
     system = (InternalDistributedSystem)DistributedSystem.connect(props);
     
@@ -1184,14 +1173,6 @@ public class LocatorDUnitTest extends DistributedTestCase {
     }
   }
 
-//  public void testRepeat() throws Exception {
-//    for (int i=0; i<10; i++) {
-//      System.out.println(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> run #"+i);
-//      testLocatorBecomesCoordinator();
-//      tearDown();
-//      setUp();
-//    }
-//  }
   /**
    * Tests starting one locator in a remote VM and having multiple
    * members of the distributed system join it.  This ensures that
@@ -1202,10 +1183,10 @@ public class LocatorDUnitTest extends DistributedTestCase {
   public void testLocatorBecomesCoordinator() throws Exception {
     disconnectAllFromDS();
     final String expected = "java.net.ConnectException";
-    final String addExpected = 
-      "<ExpectedException action=add>" + expected + "</ExpectedException>";
-    final String removeExpected = 
-      "<ExpectedException action=remove>" + expected + "</ExpectedException>";
+    final String addExpected =
+        "<ExpectedException action=add>" + expected + "</ExpectedException>";
+    final String removeExpected =
+        "<ExpectedException action=remove>" + expected + "</ExpectedException>";
 
     Host host = Host.getHost(0);
     VM vm0 = host.getVM(0);
@@ -1220,23 +1201,23 @@ public class LocatorDUnitTest extends DistributedTestCase {
     vm0.invoke(getStartSBLocatorRunnable(port, getUniqueName()+"1"));
     try {
 
-    final Properties props = new Properties();
-    props.setProperty("mcast-port", "0");
-    props.setProperty("locators", locators);
-    props.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
+      final Properties props = new Properties();
+      props.setProperty("mcast-port", "0");
+      props.setProperty("locators", locators);
+      props.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
-    SerializableRunnable connect =
-      new SerializableRunnable("Connect to " + locators) {
-          public void run() {
-            //System.setProperty("p2p.joinTimeout", "5000");
-            DistributedSystem sys = getSystem(props);
-            sys.getLogWriter().info(addExpected);
-          }
-        };
-    vm1.invoke(connect);
-    vm2.invoke(connect);
+      SerializableRunnable connect =
+          new SerializableRunnable("Connect to " + locators) {
+            public void run() {
+              //System.setProperty("p2p.joinTimeout", "5000");
+              DistributedSystem sys = getSystem(props);
+              sys.getLogWriter().info(addExpected);
+            }
+          };
+      vm1.invoke(connect);
+      vm2.invoke(connect);
 
-    system = (InternalDistributedSystem)getSystem(props);
+      system = (InternalDistributedSystem) getSystem(props);
 
     final DistributedMember coord = MembershipManagerHelper.getCoordinator(system);
     com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("coordinator before termination of locator is " + coord);
@@ -1254,7 +1235,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       }
     };
     Wait.waitForCriterion(ev, 15000, 200, true);
-    DistributedMember newCoord = MembershipManagerHelper.getCoordinator(system); 
+    DistributedMember newCoord = MembershipManagerHelper.getCoordinator(system);
     com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("coordinator after shutdown of locator was " +
         newCoord);
     if (newCoord == null || coord.equals(newCoord)) {
@@ -1284,23 +1265,23 @@ public class LocatorDUnitTest extends DistributedTestCase {
       new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
     bgexecLogger.info(removeExpected);
 
-    SerializableRunnable disconnect =
-      new SerializableRunnable("Disconnect from " + locators) {
-          public void run() {
-            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-            if (sys != null && sys.isConnected()) {
-              sys.disconnect();
+      SerializableRunnable disconnect =
+          new SerializableRunnable("Disconnect from " + locators) {
+            public void run() {
+              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+              if (sys != null && sys.isConnected()) {
+                sys.disconnect();
+              }
+              // connectExceptions occur during disconnect, so we need the
+              // expectedexception hint to be in effect until this point
+              LogWriter bLogger =
+                  new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
+              bLogger.info(removeExpected);
             }
-            // connectExceptions occur during disconnect, so we need the
-            // expectedexception hint to be in effect until this point
-            LogWriter bLogger =
-              new LocalLogWriter(InternalLogWriter.ALL_LEVEL, System.out);
-            bLogger.info(removeExpected);
-          }
-        };
-    vm1.invoke(disconnect);
-    vm2.invoke(disconnect);
-    vm0.invoke(getStopLocatorRunnable());
+          };
+      vm1.invoke(disconnect);
+      vm2.invoke(disconnect);
+      vm0.invoke(getStopLocatorRunnable());
     } finally {
       vm0.invoke(getStopLocatorRunnable());
     }
@@ -1317,7 +1298,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
   public static void resetRefreshWait() {
     System.getProperties().remove("p2p.gossipRefreshRate");
   }
-  
+
   public static boolean isSystemConnected() {
     DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
     if (sys != null && sys.isConnected()) {
@@ -1325,12 +1306,10 @@ public class LocatorDUnitTest extends DistributedTestCase {
     }
     return false;
   }
-  
 
   static boolean beforeFailureNotificationReceived;
   static boolean afterFailureNotificationReceived;
 
-
   /**
    * Tests starting multiple locators in multiple VMs.
    */
@@ -1348,15 +1327,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
     final int port2 = freeTCPPorts[1];
     this.port2 = port2;
     DistributedTestUtils.deleteLocatorStateFile(port1, port2);
-    final String host0 = NetworkUtils.getServerHostName(host); 
+    final String host0 = NetworkUtils.getServerHostName(host);
     final String locators = host0 + "[" + port1 + "]," +
-                            host0 + "[" + port2 + "]";
+        host0 + "[" + port2 + "]";
 
     final Properties dsProps = new Properties();
     dsProps.setProperty("locators", locators);
     dsProps.setProperty("mcast-port", "0");
     dsProps.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
-//    dsProps.setProperty("log-level", "finest");
     final String uniqueName = getUniqueName();
 
     vm0.invoke(new SerializableRunnable("Start locator on " + port1) {
@@ -1387,14 +1365,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
       try {
 
         SerializableRunnable connect =
-          new SerializableRunnable("Connect to " + locators) {
-          public void run() {
-            Properties props = new Properties();
-            props.setProperty("mcast-port", "0");
-            props.setProperty("locators", locators);
-            DistributedSystem.connect(props);
-          }
-        };
+            new SerializableRunnable("Connect to " + locators) {
+              public void run() {
+                Properties props = new Properties();
+                props.setProperty("mcast-port", "0");
+                props.setProperty("locators", locators);
+                DistributedSystem.connect(props);
+              }
+            };
         vm1.invoke(connect);
         vm2.invoke(connect);
 
@@ -1402,19 +1380,19 @@ public class LocatorDUnitTest extends DistributedTestCase {
         props.setProperty("mcast-port", "0");
         props.setProperty("locators", locators);
 
-        system = (InternalDistributedSystem)DistributedSystem.connect(props);
+        system = (InternalDistributedSystem) DistributedSystem.connect(props);
 
         WaitCriterion ev = new WaitCriterion() {
           public boolean done() {
             try {
               return system.getDM().getViewMembers().size() >= 3;
-            }
-            catch (Exception e) {
+            } catch (Exception e) {
               e.printStackTrace();
               fail("unexpected exception");
             }
             return false; // NOTREACHED
           }
+
           public String description() {
             return null;
           }
@@ -1427,14 +1405,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
         system.disconnect();
 
         SerializableRunnable disconnect =
-          new SerializableRunnable("Disconnect from " + locators) {
-          public void run() {
-            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-            if (sys != null && sys.isConnected()) {
-              sys.disconnect();
-            }
-          }
-        };
+            new SerializableRunnable("Disconnect from " + locators) {
+              public void run() {
+                DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+                if (sys != null && sys.isConnected()) {
+                  sys.disconnect();
+                }
+              }
+            };
         vm1.invoke(disconnect);
         vm2.invoke(disconnect);
 
@@ -1447,6 +1425,158 @@ public class LocatorDUnitTest extends DistributedTestCase {
   }
   
   /**
+   * Tests starting multiple locators at the same time and ensuring that the locators
+   * end up only have 1 master.
+   * GEODE-870
+   */
+  public void testMultipleLocatorsRestartingAtSameTime() throws Exception {
+    disconnectAllFromDS();
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+    VM vm2 = host.getVM(2);
+    VM vm3 = host.getVM(3);
+    VM vm4 = host.getVM(4);
+
+    int[] freeTCPPorts = AvailablePortHelper.getRandomAvailableTCPPorts(3);
+    this.port1 = freeTCPPorts[0];
+    this.port2 = freeTCPPorts[1];
+    int port3 = freeTCPPorts[2];
+    deleteLocatorStateFile(port1, port2, port3);
+    final String host0 = getServerHostName(host);
+    final String locators = host0 + "[" + port1 + "]," +
+        host0 + "[" + port2 + "]," +
+        host0 + "[" + port3 + "]";
+
+    final Properties dsProps = new Properties();
+    dsProps.setProperty("locators", locators);
+    dsProps.setProperty("mcast-port", "0");
+    dsProps.setProperty("log-level", "FINE");
+    dsProps.setProperty("enable-network-partition-detection", "true");
+    dsProps.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
+    final String uniqueName = getUniqueName();
+
+    startLocatorSync(vm0, new Object[] { port1, dsProps });
+    startLocatorSync(vm1, new Object[] { port2, dsProps });
+    startLocatorSync(vm2, new Object[] { port3, dsProps });
+    try {
+      try {
+
+        SerializableRunnable connect =
+            new SerializableRunnable("Connect to " + locators) {
+              public void run() {
+                Properties props = new Properties();
+                props.setProperty("mcast-port", "0");
+                props.setProperty("locators", locators);
+                props.setProperty("log-level", "FINE");
+                props.setProperty("enable-network-partition-detection", "true");
+                DistributedSystem.connect(props);
+              }
+            };
+        vm3.invoke(connect);
+        vm4.invoke(connect);
+
+        Properties props = new Properties();
+        props.setProperty("mcast-port", "0");
+        props.setProperty("locators", locators);
+
+        system = (InternalDistributedSystem) DistributedSystem.connect(props);
+
+        WaitCriterion waitCriterion = new WaitCriterion() {
+          public boolean done() {
+            try {
+              return system.getDM().getViewMembers().size() >= 3;
+            } catch (Exception e) {
+              e.printStackTrace();
+              fail("unexpected exception");
+            }
+            return false; // NOTREACHED
+          }
+
+          public String description() {
+            return null;
+          }
+        };
+        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+
+        // three applications plus
+        assertEquals(6, system.getDM().getViewMembers().size());
+
+        vm0.invoke(getStopLocatorRunnable());
+        vm1.invoke(getStopLocatorRunnable());
+        vm2.invoke(getStopLocatorRunnable());
+
+        final String newLocators = host0 + "[" + port2 + "]," +
+            host0 + "[" + port3 + "]";
+        dsProps.setProperty("locators", newLocators);
+
+        startLocatorAsync(vm1, new Object[] { port2, dsProps });
+        startLocatorAsync(vm2, new Object[] { port3, dsProps });
+
+        waitCriterion = new WaitCriterion() {
+          public boolean done() {
+            try {
+              return system.getDM().getAllHostedLocators().size() == 2;
+            } catch (Exception e) {
+              e.printStackTrace();
+              fail("unexpected exception");
+            }
+            return false; // NOTREACHED
+          }
+
+          public String description() {
+            return null;
+          }
+        };
+        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+
+      } finally {
+        system.disconnect();
+        SerializableRunnable disconnect =
+            new SerializableRunnable("Disconnect from " + locators) {
+              public void run() {
+                DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+                if (sys != null && sys.isConnected()) {
+                  sys.disconnect();
+                }
+              }
+            };
+        vm3.invoke(disconnect);
+        vm4.invoke(disconnect);
+        vm2.invoke(getStopLocatorRunnable());
+        vm1.invoke(getStopLocatorRunnable());
+      }
+    } finally {
+    }
+  }
+
+  private void startLocatorSync(VM vm, Object[] args) {
+    vm.invoke(new SerializableRunnable("Starting process on " + args[0], args) {
+      public void run() {
+        File logFile = new File("");
+        try {
+          Locator.startLocatorAndDS((int) args[0], logFile, (Properties) args[1]);
+        } catch (IOException ex) {
+          fail("While starting process on port " + args[0], ex);
+        }
+      }
+    });
+  }
+
+  private void startLocatorAsync(VM vm, Object[] args) {
+    vm.invokeAsync(new SerializableRunnable("Starting process on " + args[0], args) {
+      public void run() {
+        File logFile = new File("");
+        try {
+          Locator.startLocatorAndDS((int) args[0], logFile, (Properties) args[1]);
+        } catch (IOException ex) {
+          fail("While starting process on port " + args[0], ex);
+        }
+      }
+    });
+  }
+
+  /**
    * Tests starting multiple locators in multiple VMs.
    */
   public void testMultipleMcastLocators() throws Exception {
@@ -1466,11 +1596,11 @@ public class LocatorDUnitTest extends DistributedTestCase {
     DistributedTestUtils.deleteLocatorStateFile(port1, port2);
     final int mcastport = AvailablePort.getRandomAvailablePort(AvailablePort.MULTICAST);
     
-    final String host0 = NetworkUtils.getServerHostName(host); 
+    final String host0 = NetworkUtils.getServerHostName(host);
     final String locators = host0 + "[" + port1 + "]," +
-                            host0 + "[" + port2 + "]";
+        host0 + "[" + port2 + "]";
     final String uniqueName = getUniqueName();
-    
+
     vm0.invoke(new SerializableRunnable("Start locator on " + port1) {
         public void run() {
           File logFile = new File("");
@@ -1510,7 +1640,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       });
 
     SerializableRunnable connect =
-      new SerializableRunnable("Connect to " + locators) {
+        new SerializableRunnable("Connect to " + locators) {
           public void run() {
             Properties props = new Properties();
             props.setProperty("mcast-port", String.valueOf(mcastport));
@@ -1532,17 +1662,17 @@ public class LocatorDUnitTest extends DistributedTestCase {
       props.setProperty("mcast-ttl", "0");
       props.setProperty("enable-network-partition-detection", "true");
 
-      system = (InternalDistributedSystem)DistributedSystem.connect(props);
+      system = (InternalDistributedSystem) DistributedSystem.connect(props);
       WaitCriterion ev = new WaitCriterion() {
         public boolean done() {
           try {
             return system.getDM().getViewMembers().size() == 5;
-          }
-          catch (Exception e) {
+          } catch (Exception e) {
             com.gemstone.gemfire.test.dunit.Assert.fail("unexpected exception", e);
           }
           return false; // NOTREACHED
         }
+
         public String description() {
           return "waiting for 5 members - have " + system.getDM().getViewMembers().size();
         }
@@ -1551,7 +1681,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       system.disconnect();
 
       SerializableRunnable disconnect =
-        new SerializableRunnable("Disconnect from " + locators) {
+          new SerializableRunnable("Disconnect from " + locators) {
             public void run() {
               DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
               if (sys != null && sys.isConnected()) {
@@ -1561,8 +1691,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
           };
       vm1.invoke(disconnect);
       vm2.invoke(disconnect);
-    }
-    finally {
+    } finally {
       SerializableRunnable stop = getStopLocatorRunnable();
       vm0.invoke(stop);
       vm3.invoke(stop);
@@ -1572,7 +1701,6 @@ public class LocatorDUnitTest extends DistributedTestCase {
     }
   }
 
-
   /**
    * Tests that a VM can connect to a locator that is hosted in its
    * own VM.
@@ -1590,14 +1718,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
     final String locators = NetworkUtils.getServerHostName(host) + "[" + port1 + "]";
 
-    Properties props = new Properties();
-    props.setProperty("mcast-port", "0");
-    props.setProperty("locators", locators);
-    props.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
-    system = (InternalDistributedSystem)DistributedSystem.connect(props);
-    system.disconnect();
+      Properties props = new Properties();
+      props.setProperty("mcast-port", "0");
+      props.setProperty("locators", locators);
+      props.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
+      system = (InternalDistributedSystem) DistributedSystem.connect(props);
+      system.disconnect();
     } finally {
-    locator.stop();
+      locator.stop();
     }
   }
 
@@ -1610,7 +1738,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     Host host = Host.getHost(0);
     VM vm1 = host.getVM(1);
     Locator locator = null;
-    
+
     try {
       port1 = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
       DistributedTestUtils.deleteLocatorStateFile(port1);
@@ -1623,37 +1751,36 @@ public class LocatorDUnitTest extends DistributedTestCase {
       properties.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
       File logFile = new File("");
       locator = Locator.startLocatorAndDS(port1, logFile, properties);
-  
+
       final Properties properties2 = new Properties();
       properties2.put("mcast-port", "0");
       properties2.put("locators", locators);
       properties2.put(DistributionConfig.ENABLE_NETWORK_PARTITION_DETECTION_NAME, "false");
       properties2.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
       properties2.put("disable-auto-reconnect", "true");
-      
+
       vm1.invoke(new SerializableRunnable("try to connect") {
         public void run() {
           DistributedSystem s = null;
           try {
             s = DistributedSystem.connect(properties2);
-            boolean enabled = ((InternalDistributedSystem)s).getConfig().getEnableNetworkPartitionDetection();
+            boolean enabled = ((InternalDistributedSystem) s).getConfig().getEnableNetworkPartitionDetection();
             s.disconnect();
             if (!enabled) {
               fail("should not have been able to connect with different enable-network-partition-detection settings");
             }
-          }
-          catch (GemFireConfigException e) {
+          } catch (GemFireConfigException e) {
             fail("should have been able to connect and have enable-network-partion-detection enabled");
           }
         }
       });
-      
+
       locator.stop();
-      
+
       // now start the locator with enable-network-partition-detection=false
       logFile = new File("");
-      locator = Locator.startLocatorAndDS(port1, logFile , properties2);
-  
+      locator = Locator.startLocatorAndDS(port1, logFile, properties2);
+
       vm1.invoke(new SerializableRunnable("try to connect") {
         public void run() {
           DistributedSystem s = null;
@@ -1661,17 +1788,15 @@ public class LocatorDUnitTest extends DistributedTestCase {
             s = DistributedSystem.connect(properties);
             s.disconnect();
             fail("should not have been able to connect with different enable-network-partition-detection settings");
-          }
-          catch (GemFireConfigException e) {
+          } catch (GemFireConfigException e) {
             // passed
           }
         }
       });
-      
+
       locator.stop();
       locator = null;
-    }
-    finally {
+    } finally {
       if (locator != null) {
         locator.stop();
       }
@@ -1688,16 +1813,16 @@ public class LocatorDUnitTest extends DistributedTestCase {
     //VM vm1 = host.getVM(1);
 
     port1 =
-      AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
+        AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
     File logFile1 = new File("");
     DistributedTestUtils.deleteLocatorStateFile(port1);
     Locator locator1 = Locator.startLocator(port1, logFile1);
-    
+
     try {
 
-    int port2 =
-      AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
-    File logFile2 = new File("");
+      int port2 =
+          AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
+      File logFile2 = new File("");
 
     DistributedTestUtils.deleteLocatorStateFile(port2);
     
@@ -1724,21 +1849,21 @@ public class LocatorDUnitTest extends DistributedTestCase {
     connect.run();
     //vm1.invoke(connect);
 
-    SerializableRunnable disconnect =
-      new SerializableRunnable("Disconnect from " + locators) {
-          public void run() {
-            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-            if (sys != null && sys.isConnected()) {
-              sys.disconnect();
+      SerializableRunnable disconnect =
+          new SerializableRunnable("Disconnect from " + locators) {
+            public void run() {
+              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+              if (sys != null && sys.isConnected()) {
+                sys.disconnect();
+              }
             }
-          }
-        };
+          };
 
-    disconnect.run();
-    //vm1.invoke(disconnect);
+      disconnect.run();
+      //vm1.invoke(disconnect);
 
     } finally {
-    locator1.stop();
+      locator1.stop();
     }
   }
 
@@ -1754,10 +1879,10 @@ public class LocatorDUnitTest extends DistributedTestCase {
       AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
     DistributedTestUtils.deleteLocatorStateFile(port1);
     File logFile = new File("");
-    File stateFile = new File("locator"+port1+"state.dat");
+    File stateFile = new File("locator" + port1 + "state.dat");
     VM vm0 = Host.getHost(0).getVM(0);
     final Properties p = new Properties();
-    p.setProperty(DistributionConfig.LOCATORS_NAME, Host.getHost(0).getHostName() + "["+port1+"]");
+    p.setProperty(DistributionConfig.LOCATORS_NAME, Host.getHost(0).getHostName() + "[" + port1 + "]");
     p.setProperty(DistributionConfig.MCAST_PORT_NAME, "0");
     p.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
     if (stateFile.exists()) {
@@ -1778,7 +1903,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     
     com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("Stopping locator");
     locator.stop();
-    
+
     com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("Starting locator");
     locator = Locator.startLocatorAndDS(port1, logFile, p);
     
@@ -1789,14 +1914,14 @@ public class LocatorDUnitTest extends DistributedTestCase {
     });
     
     } finally {
-    locator.stop();
+      locator.stop();
     }
 
   }
   
   /** return the distributed member id for the ds on this vm */
   public static DistributedMember getDistributedMember(Properties props) {
-    props.put("name", "vm_"+VM.getCurrentVMNum());
+    props.put("name", "vm_" + VM.getCurrentVMNum());
     DistributedSystem sys = DistributedSystem.connect(props);
     sys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
     sys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
@@ -1807,7 +1932,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
   /** find a running locator and return its distributed member id */
   public static DistributedMember getLocatorDistributedMember() {
     return (Locator.getLocators().iterator().next())
-      .getDistributedSystem().getDistributedMember();
+        .getDistributedSystem().getDistributedMember();
   }
   
   /** find the lead member and return its id */
@@ -1828,7 +1953,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       }
     };
   }
-  
+
   private SerializableRunnable getStartSBLocatorRunnable(final int port, final String name) {
     return new SerializableRunnable("Start locator on port " + port) {
       public void run() {
@@ -1850,26 +1975,25 @@ public class LocatorDUnitTest extends DistributedTestCase {
       }
     };
   }
-  
+
   protected void nukeJChannel(DistributedSystem sys) {
     sys.getLogWriter().info("<ExpectedException action=add>service failure</ExpectedException>");
     sys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ConnectException</ExpectedException>");
     sys.getLogWriter().info("<ExpectedException action=add>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
     try {
       MembershipManagerHelper.crashDistributedSystem(sys);
-    }
-    catch (DistributedSystemDisconnectedException se) {
+    } catch (DistributedSystemDisconnectedException se) {
       // it's okay for the system to already be shut down
     }
     sys.getLogWriter().info("<ExpectedException action=remove>service failure</ExpectedException>");
     sys.getLogWriter().info("<ExpectedException action=remove>com.gemstone.gemfire.ForcedDisconnectException</ExpectedException>");
   }
 
-  
   //New test hook which blocks before closing channel.
   class TestHook implements MembershipTestHook {
 
     volatile boolean unboundedWait = true;
+
     @Override
     public void beforeMembershipFailure(String reason, Throwable cause) {
       System.out.println("Inside TestHook.beforeMembershipFailure with " + cause);
@@ -1892,22 +2016,26 @@ public class LocatorDUnitTest extends DistributedTestCase {
     }
 
   }
+
   class MyMembershipListener implements MembershipListener {
     boolean quorumLostInvoked;
     List<String> suspectReasons = new ArrayList<>(50);
-    
-    public void memberJoined(InternalDistributedMember id) {  }
-    public void memberDeparted(InternalDistributedMember id, boolean crashed) { }
+
+    public void memberJoined(InternalDistributedMember id) {
+    }
+
+    public void memberDeparted(InternalDistributedMember id, boolean crashed) {
+    }
+
     public void memberSuspect(InternalDistributedMember id,
         InternalDistributedMember whoSuspected, String reason) {
       suspectReasons.add(reason);
     }
+
     public void quorumLost(Set<InternalDistributedMember> failures,
         List<InternalDistributedMember> remaining) {
       quorumLostInvoked = true;
       com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("quorumLost invoked in test code");
     }
   }
-  
-  
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/036f2205/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
index 353cdc7..4caf815 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/SerializableRunnable.java
@@ -49,8 +49,9 @@ import java.io.Serializable;
 public abstract class SerializableRunnable implements SerializableRunnableIF {
 
   private static final long serialVersionUID = 7584289978241650456L;
-  
+
   private String name;
+  private Object[] args;
 
   public SerializableRunnable() {
     this.name = null;
@@ -70,11 +71,16 @@ public abstract class SerializableRunnable implements SerializableRunnableIF {
   public SerializableRunnable(String name) {
     this.name = name;
   }
-  
+
+  public SerializableRunnable(String name, Object[] args) {
+    this.name = name;
+    this.args = args;
+  }
+
   public void setName(String newName) {
     this.name = newName;
   }
-  
+
   public String getName() {
     return this.name;
   }


[3/7] incubator-geode git commit: GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination

Posted by ud...@apache.org.
GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/dfe248de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/dfe248de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/dfe248de

Branch: refs/heads/feature/GEODE-870
Commit: dfe248de316125d1c59e30f787e730741241dd4a
Parents: 036f220
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Wed Feb 10 09:20:09 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../gms/messages/ViewRejectMessage.java         |  96 ------
 .../gms/membership/GMSJoinLeaveHelper.java      |  60 ++++
 .../internal/membership/NetView.java            |   5 +
 .../membership/gms/membership/GMSJoinLeave.java | 319 ++++++++++---------
 .../gms/messages/InstallViewMessage.java        |  18 +-
 .../gemstone/gemfire/internal/DSFIDFactory.java |  11 +-
 .../internal/DataSerializableFixedID.java       |   2 -
 .../gemfire/distributed/LocatorDUnitTest.java   | 209 ++++++------
 gradle/rat.gradle                               |   2 +-
 9 files changed, 352 insertions(+), 370 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
deleted file mode 100755
index e5bf9e2..0000000
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/ViewRejectMessage.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.gemstone.gemfire.distributed.internal.membership.gms.messages;
-
-import com.gemstone.gemfire.DataSerializer;
-import com.gemstone.gemfire.distributed.internal.DistributionManager;
-import com.gemstone.gemfire.distributed.internal.HighPriorityDistributionMessage;
-import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
-import com.gemstone.gemfire.distributed.internal.membership.NetView;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-public class ViewRejectMessage extends HighPriorityDistributionMessage {
-
-  private int viewId;
-  private NetView rejectedView;
-  private String reason;
-
-  public ViewRejectMessage(InternalDistributedMember recipient, int viewId, NetView rejectedView, String reason) {
-    super();
-    setRecipient(recipient);
-    this.viewId = viewId;
-    this.rejectedView = rejectedView;
-    this.reason = reason;
-  }
-
-  public ViewRejectMessage() {
-    // no-arg constructor for serialization
-  }
-  
-  public int getViewId() {
-    return viewId;
-  }
-  
-  public NetView getRejectedView() {
-    return this.rejectedView;
-  }
-  
-
-  @Override
-  public int getDSFID() {
-    // TODO Auto-generated method stub
-    return VIEW_REJECT_MESSAGE;
-  }
-
-  public String getReason() {
-    return reason;
-  }
-
-  @Override
-  public int getProcessorType() {
-    return 0;
-  }
-
-  @Override
-  public void process(DistributionManager dm) {
-    throw new IllegalStateException("this message is not intended to execute in a thread pool");
-  }
-
-  @Override
-  public void toData(DataOutput out) throws IOException {
-    super.toData(out);
-    out.writeInt(this.viewId);
-    DataSerializer.writeObject(this.rejectedView, out);
-  }
-
-  @Override
-  public void fromData(DataInput in) throws IOException, ClassNotFoundException {
-    super.fromData(in);
-    this.viewId = in.readInt();
-    this.rejectedView = DataSerializer.readObject(in);
-  }
-  
-  @Override
-  public String toString() {
-    String s = getSender() == null? getRecipientsDescription() : ""+getSender();
-    return "ViewRejectMessage("+s+"; "+this.viewId+";  rejectedView="+this.rejectedView +")";
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
new file mode 100644
index 0000000..b8311bc
--- /dev/null
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
+
+import com.gemstone.gemfire.distributed.Locator;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.distributed.internal.membership.gms.Services;
+import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager;
+
+public class GMSJoinLeaveHelper {
+  public static boolean isViewCreator() {
+    GMSJoinLeave gmsJoinLeave = getGmsJoinLeave();
+    if (gmsJoinLeave != null) {
+      GMSJoinLeave.ViewCreator viewCreator = gmsJoinLeave.getViewCreator();
+      if (viewCreator != null && !viewCreator.isShutdown()) {
+        return true;
+      } else {
+        return false;
+      }
+    }
+    throw new RuntimeException("This should not have happened. There should be a JoinLeave for every DS");
+  }
+
+  private static GMSJoinLeave getGmsJoinLeave() {
+    InternalDistributedSystem distributedSystem = getInternalDistributedSystem();
+    DM dm = distributedSystem.getDM();
+    GMSMembershipManager membershipManager = (GMSMembershipManager) dm.getMembershipManager();
+    Services services = membershipManager.getServices();
+    return (GMSJoinLeave) services.getJoinLeave();
+  }
+
+  public static Integer getViewId() {
+    return getGmsJoinLeave().getView().getViewId();
+  }
+
+  private static InternalDistributedSystem getInternalDistributedSystem() {
+    InternalDistributedSystem distributedSystem = InternalDistributedSystem.getAnyInstance();
+    if (distributedSystem == null) {
+      Locator locator = Locator.getLocator();
+      return (InternalDistributedSystem) locator.getDistributedSystem();
+    } else {
+      return distributedSystem;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
index 40f5f71..af05f82 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.Random;
 import java.util.Set;
 
+import com.gemstone.gemfire.internal.logging.LogService;
 import org.apache.logging.log4j.Logger;
 
 import com.gemstone.gemfire.DataSerializer;
@@ -47,6 +48,9 @@ import com.gemstone.gemfire.internal.Version;
  */
 public class NetView implements DataSerializableFixedID {
 
+  private static final Logger logger = LogService.getLogger();
+
+
   private int viewId;
   private List<InternalDistributedMember> members;
   private int[] failureDetectionPorts = new int[10];
@@ -86,6 +90,7 @@ public class NetView implements DataSerializableFixedID {
     crashedMembers = Collections.emptySet();
     this.creator = creator;
     Arrays.fill(failureDetectionPorts, -1);
+
   }
 
   // legacy method for JGMM

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index c7eacfa..b246344 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -259,14 +259,14 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       // unable to contact any of the locators
       if (!this.isJoined && state.hasContactedAJoinedLocator) {
         throw new SystemConnectException("Unable to join the distributed system in "
-           + (System.currentTimeMillis()-startTime) + "ms");
+            + (System.currentTimeMillis() - startTime) + "ms");
       }
 
       return this.isJoined;
     } finally {
       // notify anyone waiting on the address to be completed
       if (this.isJoined) {
-        synchronized(this.localAddress) {
+        synchronized (this.localAddress) {
           this.localAddress.notifyAll();
         }
       }
@@ -277,10 +277,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   /**
    * send a join request and wait for a reply.  Process the reply.
    * This may throw a SystemConnectException or an AuthenticationFailedException
-   * 
+   *
    * @return true if the attempt succeeded, false if it timed out
    */
-  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="WA_NOT_IN_LOOP")
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "WA_NOT_IN_LOOP")
   boolean attemptToJoin() {
     SearchState state = searchState;
 
@@ -302,7 +302,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       Thread.currentThread().interrupt();
       return false;
     }
-    
+
     if (response == null) {
       if (!isJoined) {
         logger.debug("received no join response");
@@ -314,13 +314,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     joinResponse[0] = null;
     String failReason = response.getRejectionMessage();
     if (failReason != null) {
-      if (failReason.contains("Rejecting the attempt of a member using an older version") 
+      if (failReason.contains("Rejecting the attempt of a member using an older version")
           || failReason.contains("15806")) {
         throw new SystemConnectException(failReason);
       }
       throw new AuthenticationFailedException(failReason);
     }
-    
+
     if (response.getCurrentView() == null) {
       logger.info("received join response with no membership view: {}", response);
       return isJoined;
@@ -328,19 +328,18 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     if (response.getBecomeCoordinator()) {
       logger.info("I am being told to become the membership coordinator by {}", coord);
-      synchronized(viewInstallationLock) {
+      synchronized (viewInstallationLock) {
         this.currentView = response.getCurrentView();
         becomeCoordinator(null);
       }
       return true;
     }
-    
+
     this.birthViewId = response.getMemberID().getVmViewId();
     this.localAddress.setVmViewId(this.birthViewId);
     GMSMember me = (GMSMember) this.localAddress.getNetMember();
     me.setBirthViewId(birthViewId);
     installView(response.getCurrentView());
-
     return true;
   }
 
@@ -384,7 +383,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
    * this method will enqueue the request for processing in another thread.
    * If this is not the coordinator but the coordinator is known, the message
    * is forwarded to the coordinator.
-   * 
+   *
    * @param incomingRequest
    */
   private void processJoinRequest(JoinRequestMessage incomingRequest) {
@@ -413,12 +412,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       return;
     }
 
-    if (!this.localAddress.getNetMember().preferredForCoordinator() && 
-        incomingRequest.getMemberID().getNetMember().preferredForCoordinator()) {
-      JoinResponseMessage m = new JoinResponseMessage(incomingRequest.getMemberID(), currentView, true);
-      services.getMessenger().send(m);
-      return;
-    }
+//      Remove JoinResponseMessage to fix GEODE-870
+//        if (!this.localAddress.getNetMember().preferredForCoordinator() &&
+//            incomingRequest.getMemberID().getNetMember().preferredForCoordinator()){
+//          JoinResponseMessage joinResponseMessage = new JoinResponseMessage(incomingRequest.getMemberID(), currentView, true);
+//          services.getMessenger().send(joinResponseMessage);
+//          return;
+//        }
     recordViewRequest(incomingRequest);
   }
 
@@ -426,7 +426,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
    * Process a Leave request from another member. This may cause this member
    * to become the new membership coordinator. If this is the coordinator
    * a new view will be triggered.
-   * 
+   *
    * @param incomingRequest
    */
   private void processLeaveRequest(LeaveRequestMessage incomingRequest) {
@@ -442,7 +442,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     InternalDistributedMember mbr = incomingRequest.getMemberID();
 
     if (logger.isDebugEnabled()) {
-      logger.debug("JoinLeave.processLeaveRequest invoked.  isCoordinator="+isCoordinator+ "; isStopping="+isStopping 
+      logger.debug("JoinLeave.processLeaveRequest invoked.  isCoordinator="+isCoordinator+ "; isStopping="+isStopping
           +"; cancelInProgress="+ services.getCancelCriterion().isCancelInProgress());
     }
 
@@ -487,7 +487,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
    * Process a Remove request from another member. This may cause this member
    * to become the new membership coordinator. If this is the coordinator
    * a new view will be triggered.
-   * 
+   *
    * @param incomingRequest
    */
   private void processRemoveRequest(RemoveMemberMessage incomingRequest) {
@@ -501,7 +501,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       logger.info("Membership ignoring removal request for " + mbr + " from non-member " + incomingRequest.getSender());
       return;
     }
-    
+
     if (v == null) {
       // not yet a member
       return;
@@ -532,11 +532,11 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         check.addCrashedMembers(removedMembers);
         check.removeAll(removedMembers);
       }
-      synchronized(leftMembers) {
+      synchronized (leftMembers) {
         check.removeAll(leftMembers);
       }
       if (check.getCoordinator().equals(localAddress)) {
-        synchronized(viewInstallationLock) {
+        synchronized (viewInstallationLock) {
           becomeCoordinator(mbr);
         }
       }
@@ -580,17 +580,18 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   void becomeCoordinator() { // package access for unit testing
     becomeCoordinator(null);
   }
- 
+
   public void becomeCoordinatorForTest() {
-    synchronized(viewInstallationLock) {
+    synchronized (viewInstallationLock) {
       becomeCoordinator();
     }
   }
-  
+
   /**
    * Test hook for delaying the creation of new views.
    * This should be invoked before this member becomes coordinator
    * and creates its ViewCreator thread.
+   *
    * @param millis
    */
   public void delayViewCreationForTest(int millis) {
@@ -602,17 +603,17 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
    * be invoked under a synch on viewInstallationLock that was held
    * at the time the decision was made to become coordinator so that
    * the decision is atomic with actually becoming coordinator.
+   *
    * @param oldCoordinator may be null
    */
   private void becomeCoordinator(InternalDistributedMember oldCoordinator) {
-    boolean testing = unitTesting.contains("noRandomViewChange");
 
     assert Thread.holdsLock(viewInstallationLock);
-    
+
     if (isCoordinator) {
       return;
     }
-    
+
     logger.info("This member is becoming the membership coordinator with address {}", localAddress);
     isCoordinator = true;
     if (currentView == null) {
@@ -622,55 +623,59 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       this.localAddress.setVmViewId(0);
       installView(newView);
       isJoined = true;
-      if (viewCreator == null || viewCreator.isShutdown()) {
-        createViewCreator();
-        viewCreator.setDaemon(true);
-        viewCreator.start();
-        startViewBroadcaster();
-      }
+      createAndStartViewCreator(newView);
+      startViewBroadcaster();
     } else {
       // create and send out a new view
-      NetView newView;
-      Set<InternalDistributedMember> leaving = new HashSet<>();
-      Set<InternalDistributedMember> removals;
-      synchronized(viewInstallationLock) {
-        int rand = testing? 0 : NetView.RANDOM.nextInt(10);
-        int viewNumber = currentView.getViewId() + 5 + rand;
-        if (this.localAddress.getVmViewId() < 0) {
-          this.localAddress.setVmViewId(viewNumber);
-        }
-        List<InternalDistributedMember> mbrs = new ArrayList<>(currentView.getMembers());
-        if (!mbrs.contains(localAddress)) {
-          mbrs.add(localAddress);
-        }
-        synchronized(this.removedMembers) {
-          removals = new HashSet<>(this.removedMembers);
-        }
-        synchronized(this.leftMembers) {
-          leaving.addAll(leftMembers);
-        }
-        if (oldCoordinator != null && !removals.contains(oldCoordinator)) {
-          leaving.add(oldCoordinator);
-        }
-        mbrs.removeAll(removals);
-        mbrs.removeAll(leaving);
-        newView = new NetView(this.localAddress, viewNumber, mbrs, leaving,
-            removals);
-        newView.setFailureDetectionPorts(currentView);
-        newView.setFailureDetectionPort(this.localAddress, services.getHealthMonitor().getFailureDetectionPort());
-      }
-      if (viewCreator == null || viewCreator.isShutdown()) {
-        createViewCreator();
-        viewCreator.setInitialView(newView, newView.getNewMembers(), leaving, removals);
-        viewCreator.setDaemon(true);
-        viewCreator.start();
-        startViewBroadcaster();
+      NetView newView = addMemberToNetView(this.currentView, oldCoordinator);
+      createAndStartViewCreator(newView);
+      startViewBroadcaster();
+    }
+  }
+
+  private void createAndStartViewCreator(NetView newView) {
+    if (viewCreator == null || viewCreator.isShutdown()) {
+      viewCreator = new ViewCreator("Geode Membership View Creator", Services.getThreadGroup());
+      if (newView != null) {
+        viewCreator.setInitialView(newView, newView.getNewMembers(), newView.getShutdownMembers(), newView.getCrashedMembers());
       }
+      viewCreator.setDaemon(true);
+      viewCreator.start();
     }
   }
 
-  protected void createViewCreator() {
-    viewCreator = new ViewCreator("Geode Membership View Creator", Services.getThreadGroup());
+  private NetView addMemberToNetView(NetView netView, InternalDistributedMember oldCoordinator) {
+    boolean testing = unitTesting.contains("noRandomViewChange");
+    NetView newView = null;
+    Set<InternalDistributedMember> leaving = new HashSet<>();
+    Set<InternalDistributedMember> removals;
+    synchronized (viewInstallationLock) {
+      int rand = testing ? 0 : NetView.RANDOM.nextInt(10);
+      int viewNumber = currentView.getViewId() + 5 + rand;
+      if (this.localAddress.getVmViewId() < 0) {
+        this.localAddress.setVmViewId(viewNumber);
+      }
+      List<InternalDistributedMember> mbrs = new ArrayList<>(currentView.getMembers());
+      if (!mbrs.contains(localAddress)) {
+        mbrs.add(localAddress);
+      }
+      synchronized (this.removedMembers) {
+        removals = new HashSet<>(this.removedMembers);
+      }
+      synchronized (this.leftMembers) {
+        leaving.addAll(leftMembers);
+      }
+      if (oldCoordinator != null && !removals.contains(oldCoordinator)) {
+        leaving.add(oldCoordinator);
+      }
+      mbrs.removeAll(removals);
+      mbrs.removeAll(leaving);
+      newView = new NetView(this.localAddress, viewNumber, mbrs, leaving,
+          removals);
+      newView.setFailureDetectionPorts(currentView);
+      newView.setFailureDetectionPort(this.localAddress, services.getHealthMonitor().getFailureDetectionPort());
+    }
+    return newView;
   }
 
   private void sendRemoveMessages(List<InternalDistributedMember> removals, List<String> reasons, NetView newView) {
@@ -689,7 +694,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     sendView(view, newMembers, false, this.viewProcessor);
   }
 
-  boolean sendView(NetView view, List<InternalDistributedMember> newMembers, boolean preparing, ViewReplyProcessor rp) {
+  private boolean sendView(NetView view, List<InternalDistributedMember> newMembers, boolean preparing, ViewReplyProcessor viewReplyProcessor) {
 
     int id = view.getViewId();
     InstallViewMessage msg = new InstallViewMessage(view, services.getAuthenticator().getCredentials(this.localAddress), preparing);
@@ -708,6 +713,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     if (preparing) {
       this.preparedView = view;
     } else {
+      if(!localAddress.equals(view.getCoordinator()) && getViewCreator() != null)
+      {
+        stopCoordinatorServices();
+      }
       installView(view);
     }
 
@@ -716,25 +725,25 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       return true;
     }
 
-    StringBuilder s = new StringBuilder();
+    StringBuilder stringBuilder = new StringBuilder();
     int[] ports = view.getFailureDetectionPorts();
     int numMembers = view.size();
-    for (int i=0; i<numMembers; i++) {
+    for (int i = 0; i < numMembers; i++) {
       if (i > 0) {
-        s.append(' ');
+        stringBuilder.append(' ');
       }
-      s.append(ports[i]);
+      stringBuilder.append(ports[i]);
     }
     logger.info((preparing ? "preparing" : "sending") + " new view " + view
-        + "\nfailure detection ports: " + s.toString());
+        + "\nfailure detection ports: " + stringBuilder.toString());
 
     msg.setRecipients(recips);
 
     Set<InternalDistributedMember> pendingLeaves = getPendingRequestIDs(LEAVE_REQUEST_MESSAGE);
     Set<InternalDistributedMember> pendingRemovals = getPendingRequestIDs(REMOVE_MEMBER_REQUEST);
     pendingRemovals.removeAll(view.getCrashedMembers());
-    rp.initialize(id, responders);
-    rp.processPendingRequests(pendingLeaves, pendingRemovals);
+    viewReplyProcessor.initialize(id, responders);
+    viewReplyProcessor.processPendingRequests(pendingLeaves, pendingRemovals);
     services.getMessenger().send(msg);
 
     // only wait for responses during preparation
@@ -745,10 +754,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
       logger.info("finished waiting for responses to view preparation");
 
-      InternalDistributedMember conflictingViewSender = rp.getConflictingViewSender();
-      NetView conflictingView = rp.getConflictingView();
+      InternalDistributedMember conflictingViewSender = viewReplyProcessor.getConflictingViewSender();
+      NetView conflictingView = viewReplyProcessor.getConflictingView();
       if (conflictingView != null) {
-        logger.warn("received a conflicting membership view from " + conflictingViewSender 
+        logger.warn("received a conflicting membership view from " + conflictingViewSender
             + " during preparation: " + conflictingView);
         return false;
       }
@@ -762,7 +771,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     return true;
   }
 
-  private void processViewMessage(InstallViewMessage m) {
+  private void processViewMessage(final InstallViewMessage installViewMessage) {
 
     NetView view = m.getView();
 
@@ -829,9 +838,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   }
 
   private TcpClientWrapper tcpClientWrapper = new TcpClientWrapper();
-  
+
   /***
    * testing purpose
+   *
    * @param tcpClientWrapper
    */
   void setTcpClientWrapper(TcpClientWrapper tcpClientWrapper) {
@@ -847,21 +857,21 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     SearchState state = searchState;
 
     assert this.localAddress != null;
-    
+
     // If we've already tried to bootstrap from locators that
     // haven't joined the system (e.g., a collocated locator)
     // then jump to using the membership view to try to find
     // the coordinator
-    if ( !state.hasContactedAJoinedLocator && state.view != null) {
+    if (!state.hasContactedAJoinedLocator && state.view != null) {
       return findCoordinatorFromView();
     }
 
     FindCoordinatorRequest request = new FindCoordinatorRequest(this.localAddress, state.alreadyTried, state.viewId);
     Set<InternalDistributedMember> coordinators = new HashSet<InternalDistributedMember>();
-    
-    long giveUpTime = System.currentTimeMillis() + ((long)services.getConfig().getLocatorWaitTime() * 1000L);
-    
-    int connectTimeout = (int)services.getConfig().getMemberTimeout() * 2;
+
+    long giveUpTime = System.currentTimeMillis() + ((long) services.getConfig().getLocatorWaitTime() * 1000L);
+
+    int connectTimeout = (int) services.getConfig().getMemberTimeout() * 2;
     boolean anyResponses = false;
     boolean flagsSet = false;
 
@@ -869,12 +879,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
     state.hasContactedAJoinedLocator = false;
     state.locatorsContacted = 0;
-    
+
     do {
       for (InetSocketAddress addr : locators) {
         try {
           Object o = tcpClientWrapper.sendCoordinatorFindRequest(addr, request, connectTimeout);
-          FindCoordinatorResponse response = (o instanceof FindCoordinatorResponse) ? (FindCoordinatorResponse)o : null;
+          FindCoordinatorResponse response = (o instanceof FindCoordinatorResponse) ? (FindCoordinatorResponse) o : null;
           if (response != null) {
             state.locatorsContacted++;
             if (!state.hasContactedAJoinedLocator &&
@@ -886,7 +896,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
             if (response.getCoordinator() != null) {
               anyResponses = true;
               NetView v = response.getView();
-              int viewId = v == null? -1 : v.getViewId();
+              int viewId = v == null ? -1 : v.getViewId();
               if (viewId > state.viewId) {
                 state.viewId = viewId;
                 state.view = v;
@@ -946,17 +956,17 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     }
     return true;
   }
-  
+
   protected class TcpClientWrapper {
-    protected Object sendCoordinatorFindRequest(InetSocketAddress addr, FindCoordinatorRequest request, int connectTimeout) 
-        throws ClassNotFoundException, IOException{
+    protected Object sendCoordinatorFindRequest(InetSocketAddress addr, FindCoordinatorRequest request, int connectTimeout)
+        throws ClassNotFoundException, IOException {
       return TcpClient.requestToServer(
-          addr.getAddress(), addr.getPort(), request, connectTimeout, 
+          addr.getAddress(), addr.getPort(), request, connectTimeout,
           true);
     }
-  }    
+  }
 
-  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="WA_NOT_IN_LOOP")
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "WA_NOT_IN_LOOP")
   boolean findCoordinatorFromView() {
     ArrayList<FindCoordinatorResponse> result;
     SearchState state = searchState;
@@ -1028,8 +1038,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   private void inheritSettingsFromLocator(InetSocketAddress addr, FindCoordinatorResponse response) {
     boolean enabled = response.isNetworkPartitionDetectionEnabled();
     if (!enabled && services.getConfig().isNetworkPartitionDetectionEnabled()) {
-      throw new GemFireConfigException("locator at "+addr 
-          +" does not have network-partition-detection enabled but my configuration has it enabled");
+      throw new GemFireConfigException("locator at " + addr
+          + " does not have network-partition-detection enabled but my configuration has it enabled");
     }
 
     GMSMember mbr = (GMSMember) this.localAddress.getNetMember();
@@ -1040,8 +1050,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     if (response.isUsePreferredCoordinators()) {
       this.quorumRequired = true;
       logger.debug("The locator indicates that all locators should be preferred as coordinators");
-      if (services.getLocator() != null 
-          || Locator.hasLocator() 
+      if (services.getLocator() != null
+          || Locator.hasLocator()
           || !services.getConfig().getDistributionConfig().getStartLocator().isEmpty()
           || localAddress.getVmKind() == DistributionManager.LOCATOR_DM_TYPE) {
         ((GMSMember) localAddress.getNetMember()).setPreferredForCoordinator(true);
@@ -1053,7 +1063,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
   /**
    * receives a JoinResponse holding a membership view or rejection message
-   * 
+   *
    * @param rsp
    */
   private void processJoinResponse(JoinResponseMessage rsp) {
@@ -1062,7 +1072,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       joinResponse.notifyAll();
     }
   }
-  
+
   /**
    * for testing, do not use in any other case as it is not thread safe
    */
@@ -1142,7 +1152,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           return;
         }
       }
-      
+
       if (isJoined && isNetworkPartition(newView, true)) {
         if (quorumRequired) {
           Set<InternalDistributedMember> crashes = newView.getActualCrashedMembers(currentView);
@@ -1150,7 +1160,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           return;
         }
       }
-      
+
       previousView = currentView;
       currentView = newView;
       preparedView = null;
@@ -1160,7 +1170,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       if (!isJoined) {
         logger.debug("notifying join thread");
         isJoined = true;
-        synchronized(joinResponse) {
+        synchronized (joinResponse) {
           joinResponse.notifyAll();
         }
       }
@@ -1179,7 +1189,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         // newer than the view just processed - the senders will have to
         // resend these
         synchronized (viewRequests) {
-          for (Iterator<DistributionMessage> it = viewRequests.iterator(); it.hasNext();) {
+          for (Iterator<DistributionMessage> it = viewRequests.iterator(); it.hasNext(); ) {
             DistributionMessage m = it.next();
             if (m instanceof JoinRequestMessage) {
               it.remove();
@@ -1199,7 +1209,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     synchronized (removedMembers) {
       removeMembersFromCollectionIfNotInView(removedMembers, currentView);
     }
-    synchronized(leftMembers) {
+    synchronized (leftMembers) {
       removeMembersFromCollectionIfNotInView(leftMembers, currentView);
     }
   }
@@ -1208,7 +1218,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     Iterator<InternalDistributedMember> iterator = members.iterator();
     while (iterator.hasNext()) {
       if (!currentView.contains(iterator.next())) {
-          iterator.remove();
+        iterator.remove();
       }
     }
   }
@@ -1216,7 +1226,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   /**
    * Sends a message declaring a network partition to the
    * members of the given view via Messenger
-   * 
+   *
    * @param view
    */
   void sendNetworkPartitionMessage(NetView view) {
@@ -1279,10 +1289,11 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
   private void stopCoordinatorServices() {
     if (viewCreator != null && !viewCreator.isShutdown()) {
+      logger.debug("Shutting down ViewCreator");
       viewCreator.shutdown();
     }
   }
-  
+
   private void startViewBroadcaster() {
     services.getTimer().schedule(new ViewBroadcaster(), VIEW_BROADCAST_INTERVAL, VIEW_BROADCAST_INTERVAL);
   }
@@ -1342,7 +1353,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       stopCoordinatorServices();
       if (view != null) {
         if (view.size() > 1) {
-          List<InternalDistributedMember> coords = view.getPreferredCoordinators(Collections.<InternalDistributedMember> emptySet(), localAddress, 5);
+          List<InternalDistributedMember> coords = view.getPreferredCoordinators(Collections.<InternalDistributedMember>emptySet(), localAddress, 5);
           logger.debug("JoinLeave sending a leave request to {}", coords);
           LeaveRequestMessage m = new LeaveRequestMessage(coords, this.localAddress, "this member is shutting down");
           services.getMessenger().send(m);
@@ -1365,7 +1376,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       processRemoveRequest(msg);
       if (!this.isCoordinator) {
         msg.resetRecipients();
-        msg.setRecipients(v.getPreferredCoordinators(Collections.<InternalDistributedMember> emptySet(), localAddress, 10));
+        msg.setRecipients(v.getPreferredCoordinators(Collections.<InternalDistributedMember>emptySet(), localAddress, 10));
         services.getMessenger().send(msg);
       }
     }
@@ -1373,8 +1384,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
   @Override
   public void memberShutdown(DistributedMember mbr, String reason) {
-    LeaveRequestMessage msg = new LeaveRequestMessage(Collections.singleton(this.localAddress), (InternalDistributedMember)mbr, reason);
-    msg.setSender((InternalDistributedMember)mbr);
+    LeaveRequestMessage msg = new LeaveRequestMessage(Collections.singleton(this.localAddress), (InternalDistributedMember) mbr, reason);
+    msg.setSender((InternalDistributedMember) mbr);
     processLeaveRequest(msg);
   }
 
@@ -1494,11 +1505,11 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   protected ViewReplyProcessor getPrepareViewReplyProcessor() {
     return prepareProcessor;
   }
-  
-  protected boolean testPrepareProcessorWaiting(){
+
+  protected boolean testPrepareProcessorWaiting() {
     return prepareProcessor.isWaiting();
   }
-  
+
   class ViewReplyProcessor {
     volatile int viewId = -1;
     final Set<InternalDistributedMember> notRepliedYet = new HashSet<>();
@@ -1521,7 +1532,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       pendingRemovals.clear();
     }
 
-    boolean isWaiting(){
+    boolean isWaiting() {
       return waiting;
     }
 
@@ -1620,7 +1631,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           }
         }
       } finally {
-        synchronized(this) {
+        synchronized (this) {
           if (!this.waiting) {
             // if we've set waiting to false due to incoming messages then
             // we've discounted receiving any other responses from the
@@ -1663,7 +1674,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         sendCurrentView();
       }
     }
-    
+
     void sendCurrentView() {
       NetView v = currentView;
       if (v != null) {
@@ -1726,9 +1737,9 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
      * All views should be sent by the ViewCreator thread, so
      * if this member becomes coordinator it may have an initial
      * view to transmit that announces the removal of the former coordinator to
-     * 
+     *
      * @param newView
-     * @param leaving - members leaving in this view
+     * @param leaving  - members leaving in this view
      * @param removals - members crashed in this view
      */
     synchronized void setInitialView(NetView newView, List<InternalDistributedMember> newMembers,
@@ -1752,7 +1763,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         List<InternalDistributedMember> iJoins;
         Set<InternalDistributedMember> iLeaves;
         Set<InternalDistributedMember> iRemoves;
-        synchronized(this) {
+        synchronized (this) {
           iView = initialView;
           iJoins = initialJoins;
           iLeaves = initialLeaving;
@@ -1770,7 +1781,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
      * During initial view processing a prepared view was discovered.
      * This method will extract its new members and create a new
      * initial view containing them.
-     * 
+     *
      * @param v The prepared view
      */
     private void processPreparedView(NetView v) {
@@ -1778,7 +1789,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       if (currentView == null || currentView.getViewId() < v.getViewId()) {
         // we have a prepared view that is newer than the current view
         // form a new View ID
-        int viewId = Math.max(initialView.getViewId(),v.getViewId());
+        int viewId = Math.max(initialView.getViewId(), v.getViewId());
         viewId += 1;
         NetView newView = new NetView(initialView, viewId);
 
@@ -1790,13 +1801,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         } else {
           newMembers = v.getMembers();
         }
-        for (InternalDistributedMember newMember: newMembers) {
+        for (InternalDistributedMember newMember : newMembers) {
           newView.add(newMember);
           newView.setFailureDetectionPort(newMember, v.getFailureDetectionPort(newMember));
         }
 
         // use the new view as the initial view
-        synchronized(this) {
+        synchronized (this) {
           setInitialView(newView, newMembers, initialLeaving, initialRemovals);
         }
       }
@@ -1899,7 +1910,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         InternalDistributedMember mbr = null;
         switch (msg.getDSFID()) {
         case JOIN_REQUEST:
-          JoinRequestMessage jmsg = (JoinRequestMessage)msg; 
+          JoinRequestMessage jmsg = (JoinRequestMessage) msg;
           mbr = jmsg.getMemberID();
           int port = jmsg.getFailureDetectionPort();
           // see if an old member ID is being reused. If
@@ -1940,8 +1951,8 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
               removalReqs.add(mbr);
               removalReasons.add(((RemoveMemberMessage) msg).getReason());
             } else {
-              sendRemoveMessages(Collections.<InternalDistributedMember> singletonList(mbr),
-                  Collections.<String> singletonList(((RemoveMemberMessage) msg).getReason()), currentView);
+              sendRemoveMessages(Collections.<InternalDistributedMember>singletonList(mbr),
+                  Collections.<String>singletonList(((RemoveMemberMessage) msg).getReason()), currentView);
             }
           }
           break;
@@ -1977,7 +1988,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         // be reused in an auto-reconnect and get a new vmViewID
         mbrs.addAll(joinReqs);
         newView = new NetView(localAddress, viewNumber, mbrs, leaveReqs, new HashSet<InternalDistributedMember>(removalReqs));
-        for (InternalDistributedMember mbr: joinReqs) {
+        for (InternalDistributedMember mbr : joinReqs) {
           if (mbrs.contains(mbr)) {
             newView.setFailureDetectionPort(mbr, joinPorts.get(mbr));
           }
@@ -1998,14 +2009,14 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         mbr.setVmViewId(newView.getViewId());
         mbr.getNetMember().setSplitBrainEnabled(services.getConfig().isNetworkPartitionDetectionEnabled());
       }
-      
+
       if (isShutdown()) {
         return;
       }
       // send removal messages before installing the view so we stop
       // getting messages from members that have been kicked out
       sendRemoveMessages(removalReqs, removalReasons, newView);
-      
+
       prepareAndSendView(newView, joinReqs, leaveReqs, newView.getCrashedMembers());
 
       return;
@@ -2078,7 +2089,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           List<InternalDistributedMember> newMembers = conflictingView.getNewMembers();
           if (!newMembers.isEmpty()) {
             logger.info("adding these new members from a conflicting view to the new view: {}", newMembers);
-            for (InternalDistributedMember mbr: newMembers) {
+            for (InternalDistributedMember mbr : newMembers) {
               int port = conflictingView.getFailureDetectionPort(mbr);
               newView.add(mbr);
               newView.setFailureDetectionPort(mbr, port);
@@ -2087,7 +2098,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           }
           // trump the view ID of the conflicting view so mine will be accepted
           if (conflictingView.getViewId() >= newView.getViewId()) {
-            newView = new NetView(newView, conflictingView.getViewId()+1);
+            newView = new NetView(newView, conflictingView.getViewId() + 1);
           }
         }
 
@@ -2105,7 +2116,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           List<InternalDistributedMember> newMembers = new ArrayList<>(newView.getMembers());
           newMembers.removeAll(removalReqs);
           NetView tempView = new NetView(localAddress, newView.getViewId() + 1, newMembers, leaveReqs, removalReqs);
-          for (InternalDistributedMember mbr: newView.getMembers()) {
+          for (InternalDistributedMember mbr : newView.getMembers()) {
             if (tempView.contains(mbr)) {
               tempView.setFailureDetectionPort(mbr, newView.getFailureDetectionPort(mbr));
             }
@@ -2113,12 +2124,12 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           newView = tempView;
           int size = failures.size();
           List<String> reasons = new ArrayList<>(size);
-          for (int i=0; i<size; i++) {
+          for (int i = 0; i < size; i++) {
             reasons.add("Failed to acknowledge a new membership view and then failed tcp/ip connection attempt");
           }
           sendRemoveMessages(failures, reasons, newView);
         }
-        
+
         // if there is no conflicting view then we can count
         // the current state as being prepared.  All members
         // who are going to ack have already done so or passed
@@ -2126,13 +2137,13 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         if (conflictingView == null) {
           prepared = true;
         }
-        
+
       } while (!prepared);
 
       lastConflictingView = null;
 
       sendView(newView, joinReqs);
-      
+
       // after sending a final view we need to stop this thread if
       // the GMS is shutting down
       if (isStopping()) {
@@ -2143,11 +2154,11 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     /**
      * performs health checks on the collection of members, removing any that
      * are found to be healthy
-     * 
-     * @param suspects
+     *
+     * @param mbrs
      */
-    private void removeHealthyMembers(final Set<InternalDistributedMember> suspects) throws InterruptedException {
-      List<Callable<InternalDistributedMember>> checkers = new ArrayList<Callable<InternalDistributedMember>>(suspects.size());
+    private void removeHealthyMembers(final Collection<InternalDistributedMember> mbrs) throws InterruptedException {
+      List<Callable<InternalDistributedMember>> checkers = new ArrayList<Callable<InternalDistributedMember>>(mbrs.size());
 
       Set<InternalDistributedMember> newRemovals = new HashSet<>();
       Set<InternalDistributedMember> newLeaves = new HashSet<>();
@@ -2166,7 +2177,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
           @Override
           public InternalDistributedMember call() throws Exception {
             boolean available = GMSJoinLeave.this.checkIfAvailable(mbr);
-            
+
             synchronized (viewRequests) {
               if (available) {
                 suspects.remove(mbr);
@@ -2218,7 +2229,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
             if(suspects.isEmpty() || newRemovals.containsAll(suspects)) {
               break;
             }
-            
+
             viewRequests.wait(waitTime);
             waitTime = giveUpTime - System.currentTimeMillis();
           }
@@ -2249,22 +2260,22 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         }
       }
     }
-    
+
     private <T> List<Future<T>> submitAll ( ExecutorService executor, Collection<? extends Callable<T> > tasks ) {
       List<Future<T>> result = new ArrayList<Future<T>>( tasks.size() );
 
-      for ( Callable<T> task : tasks ) {
+      for (Callable<T> task : tasks) {
         result.add(executor.submit(task));
       }
 
       return result;
     }
-    
+
     boolean getTestFlageForRemovalRequest() {
       return testFlagForRemovalRequest;
     }
   }
-  
+
   boolean checkIfAvailable(InternalDistributedMember fmbr) {
  // return the member id if it fails health checks
     logger.info("checking state of member " + fmbr);

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage.java
index 91f6918..9e42f1f 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage.java
@@ -30,18 +30,19 @@ import com.gemstone.gemfire.distributed.internal.membership.NetView;
 import com.gemstone.gemfire.internal.InternalDataSerializer;
 
 public class InstallViewMessage extends HighPriorityDistributionMessage {
-
   enum messageType {
     INSTALL, PREPARE, SYNC
   }
   private NetView view;
   private Object credentials;
   private messageType kind;
+  private int previousViewId;
 
   public InstallViewMessage(NetView view, Object credentials) {
     this.view = view;
     this.kind = messageType.INSTALL;
     this.credentials = credentials;
+    this.previousViewId = view.getViewId();
   }
 
   public InstallViewMessage(NetView view, Object credentials, boolean preparing) {
@@ -49,6 +50,13 @@ public class InstallViewMessage extends HighPriorityDistributionMessage {
     this.kind = preparing? messageType.PREPARE : messageType.INSTALL;
     this.credentials = credentials;
   }
+
+  public InstallViewMessage(NetView view, Object credentials, int previousViewId, boolean preparing) {
+    this.view = view;
+    this.kind = preparing? messageType.PREPARE : messageType.INSTALL;
+    this.credentials = credentials;
+    this.previousViewId = previousViewId;
+  }
   
   public InstallViewMessage() {
     // no-arg constructor for serialization
@@ -62,6 +70,10 @@ public class InstallViewMessage extends HighPriorityDistributionMessage {
     return view;
   }
 
+  public int getPreviousViewId() {
+    return previousViewId;
+  }
+
   public Object getCredentials() {
     return credentials;
   }
@@ -83,6 +95,7 @@ public class InstallViewMessage extends HighPriorityDistributionMessage {
   @Override
   public void toData(DataOutput out) throws IOException {
     super.toData(out);
+    out.writeInt(previousViewId);
     out.writeInt(kind.ordinal());
     DataSerializer.writeObject(this.view, out);
     DataSerializer.writeObject(this.credentials, out);
@@ -91,6 +104,7 @@ public class InstallViewMessage extends HighPriorityDistributionMessage {
   @Override
   public void fromData(DataInput in) throws IOException, ClassNotFoundException {
     super.fromData(in);
+    this.previousViewId = in.readInt();
     this.kind = messageType.values()[in.readInt()];
     this.view = DataSerializer.readObject(in);
     this.credentials = DataSerializer.readObject(in);
@@ -98,7 +112,7 @@ public class InstallViewMessage extends HighPriorityDistributionMessage {
 
   @Override
   public String toString() {
-    return "InstallViewMessage(type="+this.kind+"; "+this.view
+    return "InstallViewMessage(type="+this.kind+"; Current ViewID="+view.getViewId()+"; Previous View ID="+previousViewId+"; "+this.view
             +"; cred="+(credentials==null?"null": "not null")
              +")";
   }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java
index b77dfdb..67e7c8d 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/DSFIDFactory.java
@@ -19,6 +19,7 @@ package com.gemstone.gemfire.internal;
 
 
 
+import com.gemstone.gemfire.distributed.internal.membership.gms.messages.*;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 
 import java.io.DataInput;
@@ -104,16 +105,6 @@ import com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoor
 import com.gemstone.gemfire.distributed.internal.membership.gms.locator.FindCoordinatorResponse;
 import com.gemstone.gemfire.distributed.internal.membership.gms.locator.GetViewRequest;
 import com.gemstone.gemfire.distributed.internal.membership.gms.locator.GetViewResponse;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.InstallViewMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.JoinRequestMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.JoinResponseMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.LeaveRequestMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.HeartbeatRequestMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.HeartbeatMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.NetworkPartitionMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.RemoveMemberMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.SuspectMembersMessage;
-import com.gemstone.gemfire.distributed.internal.membership.gms.messages.ViewAckMessage;
 import com.gemstone.gemfire.distributed.internal.streaming.StreamingOperation.StreamingReplyMessage;
 import com.gemstone.gemfire.internal.admin.ClientMembershipMessage;
 import com.gemstone.gemfire.internal.admin.remote.AddHealthListenerRequest;

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
index 22ac457..035ba56 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/DataSerializableFixedID.java
@@ -82,8 +82,6 @@ public interface DataSerializableFixedID extends SerializationVersions {
     case FOO:
       return new FOO(in);
   */
-  public static final short VIEW_REJECT_MESSAGE = -158;
-
   public static final short NETWORK_PARTITION_MESSAGE = -157;
   public static final short SUSPECT_MEMBERS_MESSAGE = -156;
   

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
index 1917692..c800acf 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
@@ -39,12 +39,21 @@ import com.gemstone.gemfire.distributed.internal.membership.MembershipManager;
 import com.gemstone.gemfire.distributed.internal.membership.MembershipTestHook;
 import com.gemstone.gemfire.distributed.internal.membership.NetView;
 import com.gemstone.gemfire.distributed.internal.membership.gms.MembershipManagerHelper;
+import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveHelper;
 import com.gemstone.gemfire.internal.Assert;
 import com.gemstone.gemfire.internal.AvailablePort;
 import com.gemstone.gemfire.internal.AvailablePortHelper;
 import com.gemstone.gemfire.internal.logging.InternalLogWriter;
 import com.gemstone.gemfire.internal.logging.LocalLogWriter;
 import com.gemstone.gemfire.internal.tcp.Connection;
+import com.gemstone.gemfire.test.dunit.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
 import com.gemstone.gemfire.test.dunit.AsyncInvocation;
 import com.gemstone.gemfire.test.dunit.DistributedTestCase;
 import com.gemstone.gemfire.test.dunit.DistributedTestUtils;
@@ -68,7 +77,7 @@ import com.gemstone.gemfire.test.dunit.WaitCriterion;
 public class LocatorDUnitTest extends DistributedTestCase {
 
   static TestHook hook;
-  
+
   /**
    * Creates a new <code>LocatorDUnitTest</code>
    */
@@ -78,12 +87,12 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
   private static final String WAIT2_MS_NAME = "LocatorDUnitTest.WAIT2_MS";
   private static final int WAIT2_MS_DEFAULT = 5000; // 2000 -- see bug 36470
-  private static final int WAIT2_MS 
+  private static final int WAIT2_MS
       = Integer.getInteger(WAIT2_MS_NAME, WAIT2_MS_DEFAULT).intValue();
-  
+
   private int port1;
   private int port2;
-  
+
   @Override
   public void setUp() throws Exception {
     super.setUp();
@@ -91,7 +100,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     port2 = -1;
     IgnoredException.addIgnoredException("Removing shunned member");
   }
-  
+
   @Override
   protected final void preTearDown() throws Exception {
     if (Locator.hasLocator()) {
@@ -106,7 +115,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       DistributedTestUtils.deleteLocatorStateFile(port2);
     }
   }
-  
+
   ////////  Test Methods
 
   /**
@@ -123,7 +132,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm1 = host.getVM(1);
     VM vm2 = host.getVM(2);
     VM vm3 = host.getVM(3);
-    
+
     port1 = AvailablePort.getRandomAvailablePort(AvailablePort.SOCKET);
     DistributedTestUtils.deleteLocatorStateFile(port1);
 
@@ -140,7 +149,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     InternalDistributedMember mbr = system.getDistributedMember();
     assertEquals("expected the VM to have NORMAL vmKind",
         DistributionManager.NORMAL_DM_TYPE, system.getDistributedMember().getVmKind());
-    
+
     properties.remove("start-locator");
     properties.put("log-level", LogWriterUtils.getDUnitLogLevel());
     properties.put("locators", locators);
@@ -156,12 +165,12 @@ public class LocatorDUnitTest extends DistributedTestCase {
     Cache cache = CacheFactory.create(system);
     Region r = cache.createRegionFactory(RegionShortcut.REPLICATE).create("test region");
     assertNotNull("expected to create a region", r);
-    
+
     // create a lock service and have every vm get a lock
     DistributedLockService service = DistributedLockService.create("test service", system);
     service.becomeLockGrantor();
     service.lock("foo0", 0, 0);
-    
+
     vm1.invoke(new SerializableRunnable("get the lock service and lock something") {
       public void run() {
         final DistributedLockService service = DistributedLockService.create("test service", system);
@@ -178,7 +187,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
     // cause elder failover.  vm1 will become the lock grantor
     system.disconnect();
-    
+
     try {
       vm1.invoke(new SerializableRunnable("ensure grantor failover") {
         public void run() {
@@ -194,12 +203,12 @@ public class LocatorDUnitTest extends DistributedTestCase {
             public String description() {
               return "waiting to become lock grantor after shutting down locator/grantor";
             }
-            
+
           }, DistributionConfig.DEFAULT_MEMBER_TIMEOUT * 2, 1000, true);
           assertTrue(service.isLockGrantor());
         }
       });
-  
+
       properties.put("start-locator", locators);
       properties.put("log-level", LogWriterUtils.getDUnitLogLevel());
       system = (InternalDistributedSystem)DistributedSystem.connect(properties);
@@ -222,7 +231,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       });
 
       assertFalse("should not have become lock grantor", service.isLockGrantor());
-      
+
       // Now demonstrate that a new member can join and use the lock service
       properties.remove("start-locator");
       vm3.invoke(startSystem);
@@ -232,7 +241,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
           service.lock("foo5", 0, 0);
         }
       });
-      
+
     } finally {
       disconnectAllFromDS();
     }
@@ -243,7 +252,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
    * split-brain configuration.  To work around this we have always told customers that they
    * need to stagger the starting of locators.  This test configures two locators to start up
    * simultaneously and shows that they find each other and form a single system.
-   * 
+   *
    * @throws Exception
    */
   public void testStartTwoLocators() throws Exception {
@@ -251,7 +260,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     Host host = Host.getHost(0);
     VM loc1 = host.getVM(1);
     VM loc2 = host.getVM(2);
-    
+
     int ports[] = AvailablePortHelper.getRandomAvailableTCPPorts(2);
     final int port1 = ports[0];
     this.port1 = port1;
@@ -259,7 +268,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     this.port2 = port2; // for cleanup in tearDown2
     DistributedTestUtils.deleteLocatorStateFile(port1);
     DistributedTestUtils.deleteLocatorStateFile(port2);
-    final String host0 = NetworkUtils.getServerHostName(host); 
+    final String host0 = NetworkUtils.getServerHostName(host);
     final String locators = host0 + "[" + port1 + "]," +
                             host0 + "[" + port2 + "]";
     final Properties properties = new Properties();
@@ -385,16 +394,6 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       Object[] connectArgs = new Object[] { properties };
 
-      SerializableRunnable disconnect =
-          new SerializableRunnable("Disconnect from " + locators) {
-            public void run() {
-              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-              if (sys != null && sys.isConnected()) {
-                sys.disconnect();
-              }
-            }
-          };
-
       assertTrue(MembershipManagerHelper.getLeadMember(sys) == null);
 
       // connect three vms and then watch the lead member selection as they
@@ -417,7 +416,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       assertLeadMember(mem1, sys, 5000);
 
       // after disconnecting the first vm, the second one should become the leader
-      vm1.invoke(disconnect);
+      vm1.invoke(getDisconnectRunnable(locators));
       MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
       assertLeadMember(mem2, sys, 5000);
 
@@ -426,15 +425,15 @@ public class LocatorDUnitTest extends DistributedTestCase {
           "getDistributedMember", connectArgs);
       assertLeadMember(mem2, sys, 5000);
 
-      vm2.invoke(disconnect);
+      vm2.invoke(getDisconnectRunnable(locators));
       MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem2);
       assertLeadMember(mem3, sys, 5000);
 
-      vm1.invoke(disconnect);
+      vm1.invoke(getDisconnectRunnable(locators));
       MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem1);
       assertLeadMember(mem3, sys, 5000);
 
-      vm3.invoke(disconnect);
+      vm3.invoke(getDisconnectRunnable(locators));
       MembershipManagerHelper.getMembershipManager(sys).waitForDeparture(mem3);
       assertLeadMember(null, sys, 5000);
 
@@ -486,13 +485,13 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm2 = host.getVM(2);
     VM locvm = host.getVM(3);
     Locator locator = null;
-    
+
     int ports[] = AvailablePortHelper.getRandomAvailableTCPPorts(2);
     final int port1 = ports[0];
     this.port1 = port1;
     final int port2 = ports[1];
     DistributedTestUtils.deleteLocatorStateFile(port1, port2);
-    final String host0 = NetworkUtils.getServerHostName(host); 
+    final String host0 = NetworkUtils.getServerHostName(host);
     final String locators = host0 + "[" + port1 + "]," +
                             host0 + "[" + port2 + "]";
     final Properties properties = new Properties();
@@ -504,7 +503,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     properties.put("log-level", LogWriterUtils.getDUnitLogLevel());
 //    properties.put("log-level", "fine");
     properties.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
-    
+
     try {
       final String uname = getUniqueName();
       File logFile = new File("");
@@ -554,7 +553,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       assertTrue("Distributed system should not have disconnected",
           vm1.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       // ensure quorumLost is properly invoked
       DistributionManager dm = (DistributionManager) ((InternalDistributedSystem) sys).getDistributionManager();
       MyMembershipListener listener = new MyMembershipListener();
@@ -619,7 +618,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm2 = host.getVM(2);
     VM locvm = host.getVM(3);
     Locator locator = null;
-    
+
     final int[] ports = AvailablePortHelper.getRandomAvailableTCPPorts(2);
     final int port1 = ports[0];
     this.port1 = port1;
@@ -638,7 +637,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     properties.put(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
     SerializableRunnable stopLocator = getStopLocatorRunnable();
-    
+
     try {
       final String uname = getUniqueName();
       File logFile = new File("");
@@ -693,7 +692,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       assertTrue("Distributed system should not have disconnected",
           locvm.invoke(() -> LocatorDUnitTest.isSystemConnected()));
 
@@ -708,7 +707,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       assertTrue("Distributed system should not have disconnected",
           locvm.invoke(() -> LocatorDUnitTest.isSystemConnected()));
 
@@ -716,7 +715,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       assertEquals("This test sometimes fails.  If the log contains " +
       		"'failed to collect all ACKs' it is a false failure.",
       		mem2, vm2.invoke(() -> LocatorDUnitTest.getLeadMember()));
-      
+
       SerializableRunnable disconnect =
           new SerializableRunnable("Disconnect from " + locators) {
             public void run() {
@@ -729,7 +728,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       // disconnect the first vm and demonstrate that the third vm and the
       // locator notice the failure and exit
-      vm2.invoke(disconnect);
+      vm2.invoke(getDisconnectRunnable(locators));
       locvm.invoke(stopLocator);
     } finally {
       MembershipManagerHelper.inhibitForcedDisconnectLogging(false);
@@ -766,7 +765,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm2 = host.getVM(2);
     VM locvm = host.getVM(3);
     Locator locator = null;
-    
+
     int ports[] = AvailablePortHelper.getRandomAvailableTCPPorts(2);
     final int port1 = ports[0];
     this.port1 = port1;
@@ -842,7 +841,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       assertTrue("Distributed system should not have disconnected",
           locvm.invoke(() -> LocatorDUnitTest.isSystemConnected()));
 
@@ -903,7 +902,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     VM vm2 = host.getVM(2);
     VM locvm = host.getVM(3);
     Locator locator = null;
-    
+
     int ports[] = AvailablePortHelper.getRandomAvailableTCPPorts(2);
     final int port1 = ports[0];
     this.port1 = port1;
@@ -975,7 +974,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       vm1.invoke(expectedException);
       DistributedMember mem2 = (DistributedMember) vm2.invoke(this.getClass(),
           "getDistributedMember", connectArgs);
-      
+
       DistributedMember loc1Mbr = (DistributedMember)locvm.invoke(() -> this.getLocatorDistributedMember());
 
       assertLeadMember(mem1, sys, 5000);
@@ -991,7 +990,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
       assertTrue("Distributed system should not have disconnected",
           vm1.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
 
@@ -1002,7 +1001,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       
       assertTrue("Distributed system should not have disconnected",
           vm2.invoke(() -> LocatorDUnitTest.isSystemConnected()));
-      
+
       assertEquals(sys.getDistributedMember(),
           MembershipManagerHelper.getCoordinator(sys));
       assertEquals(mem2, MembershipManagerHelper.getLeadMember(sys));
@@ -1146,7 +1145,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
       }
     };
     Wait.waitForCriterion(ev, 15 * 1000, 200, false);
-    DistributedMember newCoord = MembershipManagerHelper.getCoordinator(system); 
+    DistributedMember newCoord = MembershipManagerHelper.getCoordinator(system);
     com.gemstone.gemfire.test.dunit.LogWriterUtils.getLogWriter().info("coordinator after shutdown of locator was " +
         newCoord);
     if (coord.equals(newCoord)) {
@@ -1156,17 +1155,8 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
     system.disconnect();
 
-    SerializableRunnable disconnect =
-      new SerializableRunnable("Disconnect from " + locators) {
-          public void run() {
-            DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-            if (sys != null && sys.isConnected()) {
-              sys.disconnect();
-            }
-          }
-        };
-    vm1.invoke(disconnect);
-    vm2.invoke(disconnect);
+      vm1.invoke(getDisconnectRunnable(locators));
+      vm2.invoke(getDisconnectRunnable(locators));
 
     } finally {
       vm0.invoke(getStopLocatorRunnable());
@@ -1404,17 +1394,8 @@ public class LocatorDUnitTest extends DistributedTestCase {
 
         system.disconnect();
 
-        SerializableRunnable disconnect =
-            new SerializableRunnable("Disconnect from " + locators) {
-              public void run() {
-                DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-                if (sys != null && sys.isConnected()) {
-                  sys.disconnect();
-                }
-              }
-            };
-        vm1.invoke(disconnect);
-        vm2.invoke(disconnect);
+        vm1.invoke(getDisconnectRunnable(locators));
+        vm2.invoke(getDisconnectRunnable(locators));
 
       } finally {
         vm3.invoke(getStopLocatorRunnable());
@@ -1423,7 +1404,18 @@ public class LocatorDUnitTest extends DistributedTestCase {
       vm0.invoke(getStopLocatorRunnable());
     }
   }
-  
+
+  private SerializableRunnable getDisconnectRunnable(final String locators) {
+    return new SerializableRunnable("Disconnect from " + locators) {
+      public void run() {
+        DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
+        if (sys != null && sys.isConnected()) {
+          sys.disconnect();
+        }
+      }
+    };
+  }
+
   /**
    * Tests starting multiple locators at the same time and ensuring that the locators
    * end up only have 1 master.
@@ -1454,7 +1446,6 @@ public class LocatorDUnitTest extends DistributedTestCase {
     dsProps.setProperty("log-level", "FINE");
     dsProps.setProperty("enable-network-partition-detection", "true");
     dsProps.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
-    final String uniqueName = getUniqueName();
 
     startLocatorSync(vm0, new Object[] { port1, dsProps });
     startLocatorSync(vm1, new Object[] { port2, dsProps });
@@ -1485,7 +1476,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         WaitCriterion waitCriterion = new WaitCriterion() {
           public boolean done() {
             try {
-              return system.getDM().getViewMembers().size() >= 3;
+              return system.getDM().getViewMembers().size() == 6;
             } catch (Exception e) {
               e.printStackTrace();
               fail("unexpected exception");
@@ -1506,10 +1497,31 @@ public class LocatorDUnitTest extends DistributedTestCase {
         vm1.invoke(getStopLocatorRunnable());
         vm2.invoke(getStopLocatorRunnable());
 
+        waitCriterion = new WaitCriterion() {
+          public boolean done() {
+            try {
+              return system.getDM().getAllHostedLocators().size() == 0;
+            } catch (Exception e) {
+              e.printStackTrace();
+              fail("unexpected exception");
+            }
+            return false; // NOTREACHED
+          }
+
+          public String description() {
+            return null;
+          }
+        };
+        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+
         final String newLocators = host0 + "[" + port2 + "]," +
             host0 + "[" + port3 + "]";
         dsProps.setProperty("locators", newLocators);
 
+        assertTrue(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        //Given the start up order of servers, this server is the elder server
+        assertTrue(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+
         startLocatorAsync(vm1, new Object[] { port2, dsProps });
         startLocatorAsync(vm2, new Object[] { port3, dsProps });
 
@@ -1530,19 +1542,23 @@ public class LocatorDUnitTest extends DistributedTestCase {
         };
         DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
 
+        int netviewId = vm1.invoke(() -> GMSJoinLeaveHelper.getViewId());
+        assertEquals(netviewId, (int) vm2.invoke(() -> GMSJoinLeaveHelper.getViewId()));
+        assertEquals(netviewId, (int) vm3.invoke(() -> GMSJoinLeaveHelper.getViewId()));
+        assertEquals(netviewId, (int) vm4.invoke(() -> GMSJoinLeaveHelper.getViewId()));
+        assertFalse(vm4.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        //Given the start up order of servers, this server is the elder server
+        assertFalse(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        if (vm1.invoke(() -> GMSJoinLeaveHelper.isViewCreator())) {
+          assertFalse(vm2.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        } else {
+          assertTrue(vm2.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        }
+
       } finally {
         system.disconnect();
-        SerializableRunnable disconnect =
-            new SerializableRunnable("Disconnect from " + locators) {
-              public void run() {
-                DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-                if (sys != null && sys.isConnected()) {
-                  sys.disconnect();
-                }
-              }
-            };
-        vm3.invoke(disconnect);
-        vm4.invoke(disconnect);
+        vm3.invoke(getDisconnectRunnable(locators));
+        vm4.invoke(getDisconnectRunnable(locators));
         vm2.invoke(getStopLocatorRunnable());
         vm1.invoke(getStopLocatorRunnable());
       }
@@ -1680,17 +1696,8 @@ public class LocatorDUnitTest extends DistributedTestCase {
       Wait.waitForCriterion(ev, WAIT2_MS, 200, true);
       system.disconnect();
 
-      SerializableRunnable disconnect =
-          new SerializableRunnable("Disconnect from " + locators) {
-            public void run() {
-              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-              if (sys != null && sys.isConnected()) {
-                sys.disconnect();
-              }
-            }
-          };
-      vm1.invoke(disconnect);
-      vm2.invoke(disconnect);
+      vm1.invoke(getDisconnectRunnable(locators));
+      vm2.invoke(getDisconnectRunnable(locators));
     } finally {
       SerializableRunnable stop = getStopLocatorRunnable();
       vm0.invoke(stop);
@@ -1849,15 +1856,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     connect.run();
     //vm1.invoke(connect);
 
-      SerializableRunnable disconnect =
-          new SerializableRunnable("Disconnect from " + locators) {
-            public void run() {
-              DistributedSystem sys = InternalDistributedSystem.getAnyInstance();
-              if (sys != null && sys.isConnected()) {
-                sys.disconnect();
-              }
-            }
-          };
+      SerializableRunnable disconnect = getDisconnectRunnable(locators);
 
       disconnect.run();
       //vm1.invoke(disconnect);

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/dfe248de/gradle/rat.gradle
----------------------------------------------------------------------
diff --git a/gradle/rat.gradle b/gradle/rat.gradle
index 496b20a..68e04c4 100644
--- a/gradle/rat.gradle
+++ b/gradle/rat.gradle
@@ -61,7 +61,7 @@ rat {
     '**/*.log',
     '**/*.patch',
     '**/*.diff',
-          '**/*.MF',
+    '**/*.MF',
 
     // binary files
     '**/*.cer',


[2/7] incubator-geode git commit: GEODE-870: Amending code as per code review Adding wait on CqQueryUsingPoolDUnitTest to make test pass

Posted by ud...@apache.org.
GEODE-870: Amending code as per code review
Adding wait on CqQueryUsingPoolDUnitTest to make test pass


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/83e4d249
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/83e4d249
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/83e4d249

Branch: refs/heads/feature/GEODE-870
Commit: 83e4d2495eae2cb994d0ff209d98ceb24c173467
Parents: 9e241d5
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Mon Feb 15 10:05:41 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../gms/membership/GMSJoinLeaveHelper.java      | 60 -----------------
 .../gms/membership/GMSJoinLeaveTestHelper.java  | 67 ++++++++++++++++++
 .../internal/membership/NetView.java            |  3 -
 .../membership/gms/membership/GMSJoinLeave.java | 25 ++++---
 .../gemfire/distributed/LocatorDUnitTest.java   | 30 ++++-----
 .../gms/membership/GMSJoinLeaveJUnitTest.java   | 71 ++++++++++----------
 .../sanctionedDataSerializables.txt             |  4 +-
 .../cq/dunit/CqQueryUsingPoolDUnitTest.java     |  4 +-
 8 files changed, 135 insertions(+), 129 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
deleted file mode 100644
index b8311bc..0000000
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveHelper.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
-
-import com.gemstone.gemfire.distributed.Locator;
-import com.gemstone.gemfire.distributed.internal.DM;
-import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
-import com.gemstone.gemfire.distributed.internal.membership.gms.Services;
-import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager;
-
-public class GMSJoinLeaveHelper {
-  public static boolean isViewCreator() {
-    GMSJoinLeave gmsJoinLeave = getGmsJoinLeave();
-    if (gmsJoinLeave != null) {
-      GMSJoinLeave.ViewCreator viewCreator = gmsJoinLeave.getViewCreator();
-      if (viewCreator != null && !viewCreator.isShutdown()) {
-        return true;
-      } else {
-        return false;
-      }
-    }
-    throw new RuntimeException("This should not have happened. There should be a JoinLeave for every DS");
-  }
-
-  private static GMSJoinLeave getGmsJoinLeave() {
-    InternalDistributedSystem distributedSystem = getInternalDistributedSystem();
-    DM dm = distributedSystem.getDM();
-    GMSMembershipManager membershipManager = (GMSMembershipManager) dm.getMembershipManager();
-    Services services = membershipManager.getServices();
-    return (GMSJoinLeave) services.getJoinLeave();
-  }
-
-  public static Integer getViewId() {
-    return getGmsJoinLeave().getView().getViewId();
-  }
-
-  private static InternalDistributedSystem getInternalDistributedSystem() {
-    InternalDistributedSystem distributedSystem = InternalDistributedSystem.getAnyInstance();
-    if (distributedSystem == null) {
-      Locator locator = Locator.getLocator();
-      return (InternalDistributedSystem) locator.getDistributedSystem();
-    } else {
-      return distributedSystem;
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
new file mode 100644
index 0000000..493c625
--- /dev/null
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveTestHelper.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed.internal.membership.gms.membership;
+
+import com.gemstone.gemfire.distributed.Locator;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.distributed.internal.membership.gms.Services;
+import com.gemstone.gemfire.distributed.internal.membership.gms.mgr.GMSMembershipManager;
+
+public class GMSJoinLeaveTestHelper {
+
+  public static void becomeCoordinatorForTest(GMSJoinLeave gmsJoinLeave) {
+    synchronized (gmsJoinLeave.getViewInstallationLock()) {
+      gmsJoinLeave.becomeCoordinator();
+    }
+  }
+
+  public static boolean isViewCreator() {
+    GMSJoinLeave gmsJoinLeave = getGmsJoinLeave();
+    if (gmsJoinLeave != null) {
+      GMSJoinLeave.ViewCreator viewCreator = gmsJoinLeave.getViewCreator();
+      if (viewCreator != null && !viewCreator.isShutdown()) {
+        return true;
+      } else {
+        return false;
+      }
+    }
+    throw new RuntimeException("This should not have happened. There should be a JoinLeave for every DS");
+  }
+
+  private static GMSJoinLeave getGmsJoinLeave() {
+    InternalDistributedSystem distributedSystem = getInternalDistributedSystem();
+    DM dm = distributedSystem.getDM();
+    GMSMembershipManager membershipManager = (GMSMembershipManager) dm.getMembershipManager();
+    Services services = membershipManager.getServices();
+    return (GMSJoinLeave) services.getJoinLeave();
+  }
+
+  public static Integer getViewId() {
+    return getGmsJoinLeave().getView().getViewId();
+  }
+
+  private static InternalDistributedSystem getInternalDistributedSystem() {
+    InternalDistributedSystem distributedSystem = InternalDistributedSystem.getAnyInstance();
+    if (distributedSystem == null) {
+      Locator locator = Locator.getLocator();
+      return (InternalDistributedSystem) locator.getDistributedSystem();
+    } else {
+      return distributedSystem;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
index af05f82..af4aec3 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/NetView.java
@@ -48,9 +48,6 @@ import com.gemstone.gemfire.internal.Version;
  */
 public class NetView implements DataSerializableFixedID {
 
-  private static final Logger logger = LogService.getLogger();
-
-
   private int viewId;
   private List<InternalDistributedMember> members;
   private int[] failureDetectionPorts = new int[10];

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index 56b644c..8ba4952 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -152,6 +152,9 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
   /** the view where quorum was most recently lost */
   NetView quorumLostView;
 
+  /** a flag to mark a coordinator's viewCreator for shutdown */
+  private boolean markViewCreatorForShutdown = false;
+
   static class SearchState {
     Set<InternalDistributedMember> alreadyTried = new HashSet<>();
     Set<InternalDistributedMember> registrants = new HashSet<>();
@@ -172,6 +175,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     }
   }
 
+  Object getViewInstallationLock() {
+    return viewInstallationLock;
+  }
+
   /**
    * attempt to join the distributed system
    * loop
@@ -581,12 +588,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     becomeCoordinator(null);
   }
 
-  public void becomeCoordinatorForTest() {
-    synchronized (viewInstallationLock) {
-      becomeCoordinator();
-    }
-  }
-
   /**
    * Test hook for delaying the creation of new views.
    * This should be invoked before this member becomes coordinator
@@ -640,6 +641,7 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         viewCreator.setInitialView(newView, newView.getNewMembers(), newView.getShutdownMembers(), newView.getCrashedMembers());
       }
       viewCreator.setDaemon(true);
+      logger.info("ViewCreator starting on:" + localAddress);
       viewCreator.start();
     }
   }
@@ -715,9 +717,9 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     } else {
       //Added a check in the view processor to turn off the ViewCreator
       //if another server is the coordinator - GEODE-870
-      if(!localAddress.equals(view.getCoordinator()) && getViewCreator() != null)
-      {
-        stopCoordinatorServices();
+      if (isCoordinator && !localAddress.equals(view.getCoordinator()) && getViewCreator() != null) {
+        markViewCreatorForShutdown = true;
+        this.isCoordinator = false;
       }
       installView(view);
     }
@@ -2077,7 +2079,6 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
         }
 
         logger.debug("unresponsive members that could not be reached: {}", unresponsive);
-
         List<InternalDistributedMember> failures = new ArrayList<>(currentView.getCrashedMembers().size() + unresponsive.size());
 
         if (conflictingView != null && !conflictingView.getCreator().equals(localAddress) && conflictingView.getViewId() > newView.getViewId()
@@ -2150,6 +2151,10 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
 
       sendView(newView, joinReqs);
 
+      if (markViewCreatorForShutdown && getViewCreator() != null) {
+        shutdown = true;
+      }
+
       // after sending a final view we need to stop this thread if
       // the GMS is shutting down
       if (isStopping()) {

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
index eb3cc62..5969c51 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
@@ -39,14 +39,13 @@ import com.gemstone.gemfire.distributed.internal.membership.MembershipManager;
 import com.gemstone.gemfire.distributed.internal.membership.MembershipTestHook;
 import com.gemstone.gemfire.distributed.internal.membership.NetView;
 import com.gemstone.gemfire.distributed.internal.membership.gms.MembershipManagerHelper;
-import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveHelper;
+import com.gemstone.gemfire.distributed.internal.membership.gms.membership.GMSJoinLeaveTestHelper;
 import com.gemstone.gemfire.internal.Assert;
 import com.gemstone.gemfire.internal.AvailablePort;
 import com.gemstone.gemfire.internal.AvailablePortHelper;
 import com.gemstone.gemfire.internal.logging.InternalLogWriter;
 import com.gemstone.gemfire.internal.logging.LocalLogWriter;
 import com.gemstone.gemfire.internal.tcp.Connection;
-import com.gemstone.gemfire.test.dunit.*;
 
 import java.io.File;
 import java.io.IOException;
@@ -1325,7 +1324,6 @@ public class LocatorDUnitTest extends DistributedTestCase {
     dsProps.setProperty("locators", locators);
     dsProps.setProperty("mcast-port", "0");
     dsProps.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
-    final String uniqueName = getUniqueName();
 
     vm0.invoke(new SerializableRunnable("Start locator on " + port1) {
       public void run() {
@@ -1443,7 +1441,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
     final Properties dsProps = new Properties();
     dsProps.setProperty("locators", locators);
     dsProps.setProperty("mcast-port", "0");
-    dsProps.setProperty("log-level", "FINE");
+    dsProps.setProperty("log-level", LogWriterUtils.getDUnitLogLevel());
     dsProps.setProperty("enable-network-partition-detection", "true");
     dsProps.setProperty(DistributionConfig.ENABLE_CLUSTER_CONFIGURATION_NAME, "false");
 
@@ -1459,7 +1457,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
                 Properties props = new Properties();
                 props.setProperty("mcast-port", "0");
                 props.setProperty("locators", locators);
-                props.setProperty("log-level", "FINE");
+                props.setProperty("log-level", LogWriterUtils.getDUnitLogLevel());
                 props.setProperty("enable-network-partition-detection", "true");
                 DistributedSystem.connect(props);
               }
@@ -1518,9 +1516,9 @@ public class LocatorDUnitTest extends DistributedTestCase {
             host0 + "[" + port3 + "]";
         dsProps.setProperty("locators", newLocators);
 
-        assertTrue(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        assertTrue(vm3.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
         //Given the start up order of servers, this server is the elder server
-        assertTrue(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        assertTrue(vm3.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
 
         startLocatorAsync(vm1, new Object[] { port2, dsProps });
         startLocatorAsync(vm2, new Object[] { port3, dsProps });
@@ -1542,17 +1540,17 @@ public class LocatorDUnitTest extends DistributedTestCase {
         };
         Wait.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
 
-        int netviewId = vm1.invoke(() -> GMSJoinLeaveHelper.getViewId());
-        assertEquals(netviewId, (int) vm2.invoke(() -> GMSJoinLeaveHelper.getViewId()));
-        assertEquals(netviewId, (int) vm3.invoke(() -> GMSJoinLeaveHelper.getViewId()));
-        assertEquals(netviewId, (int) vm4.invoke(() -> GMSJoinLeaveHelper.getViewId()));
-        assertFalse(vm4.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        int netviewId = vm1.invoke(() -> GMSJoinLeaveTestHelper.getViewId());
+        assertEquals(netviewId, (int) vm2.invoke(() -> GMSJoinLeaveTestHelper.getViewId()));
+        assertEquals(netviewId, (int) vm3.invoke(() -> GMSJoinLeaveTestHelper.getViewId()));
+        assertEquals(netviewId, (int) vm4.invoke(() -> GMSJoinLeaveTestHelper.getViewId()));
+        assertFalse(vm4.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
         //Given the start up order of servers, this server is the elder server
-        assertFalse(vm3.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
-        if (vm1.invoke(() -> GMSJoinLeaveHelper.isViewCreator())) {
-          assertFalse(vm2.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+        assertFalse(vm3.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
+        if (vm1.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator())) {
+          assertFalse(vm2.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
         } else {
-          assertTrue(vm2.invoke(() -> GMSJoinLeaveHelper.isViewCreator()));
+          assertTrue(vm2.invoke(() -> GMSJoinLeaveTestHelper.isViewCreator()));
         }
 
       } finally {

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
index 79aa02a..202888c 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
@@ -214,7 +214,7 @@ public class GMSJoinLeaveJUnitTest {
   @Test
   public void testViewWithoutMemberInitiatesForcedDisconnect() throws Exception {
     initMocks();
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     List<InternalDistributedMember> members = Arrays.asList(mockMembers);
     NetView v = new NetView(mockMembers[0], 2, members);
     InstallViewMessage message = new InstallViewMessage(v, null);
@@ -329,7 +329,7 @@ public class GMSJoinLeaveJUnitTest {
         any(String.class), any(Boolean.class))).thenReturn(true);
     
     gmsJoinLeave.delayViewCreationForTest(5000); // ensures multiple requests are queued for a view change
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
 
     NetView oldView = null;
     long giveup = System.currentTimeMillis() + viewInstallationTime;
@@ -422,9 +422,8 @@ public class GMSJoinLeaveJUnitTest {
     String reason = "testing";
     initMocks();
     gmsJoinLeave.unitTesting.add("noRandomViewChange");
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
-    gmsJoinLeave.getView().add(gmsJoinLeaveMemberId);
-    gmsJoinLeave.becomeCoordinatorForTest();
+    prepareAndInstallView(gmsJoinLeaveMemberId, createMemberList(gmsJoinLeaveMemberId,mockMembers[0]));
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
 
     LeaveRequestMessage msg = new LeaveRequestMessage(gmsJoinLeave.getMemberID(), mockMembers[0], reason);
     msg.setSender(mockMembers[0]);
@@ -446,11 +445,10 @@ public class GMSJoinLeaveJUnitTest {
   public void testDuplicateRemoveRequestDoesNotCauseNewView() throws Exception {
     String reason = "testing";
     initMocks();
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
-    gmsJoinLeave.getView().add(gmsJoinLeaveMemberId);
+    prepareAndInstallView(gmsJoinLeaveMemberId, createMemberList(gmsJoinLeaveMemberId,mockMembers[0]));
     gmsJoinLeave.getView().add(mockMembers[1]);
     gmsJoinLeave.unitTesting.add("noRandomViewChange");
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     RemoveMemberMessage msg = new RemoveMemberMessage(gmsJoinLeave.getMemberID(), mockMembers[0], reason);
     msg.setSender(mockMembers[0]);
     gmsJoinLeave.processMessage(msg);
@@ -474,10 +472,9 @@ public class GMSJoinLeaveJUnitTest {
     when(healthMonitor.checkIfAvailable(any(InternalDistributedMember.class),
         any(String.class), any(Boolean.class))).thenReturn(true);
     gmsJoinLeave.unitTesting.add("noRandomViewChange");
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
-    gmsJoinLeave.getView().add(gmsJoinLeaveMemberId);
+    prepareAndInstallView(gmsJoinLeaveMemberId, createMemberList(gmsJoinLeaveMemberId,mockMembers[0]));
     gmsJoinLeave.getView().add(mockMembers[1]);
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     JoinRequestMessage msg = new JoinRequestMessage(gmsJoinLeaveMemberId, mockMembers[2], null, -1);
     msg.setSender(mockMembers[2]);
     gmsJoinLeave.processMessage(msg);
@@ -522,7 +519,7 @@ public class GMSJoinLeaveJUnitTest {
   public void testRemoveCausesForcedDisconnect() throws Exception {
     String reason = "testing";
     initMocks();
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId)); 
+    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
     gmsJoinLeave.getView().add(mockMembers[1]);
     RemoveMemberMessage msg = new RemoveMemberMessage(mockMembers[0], gmsJoinLeave.getMemberID(), reason);
     msg.setSender(mockMembers[1]);
@@ -534,8 +531,7 @@ public class GMSJoinLeaveJUnitTest {
   public void testLeaveCausesForcedDisconnect() throws Exception {
     String reason = "testing";
     initMocks();
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
-    gmsJoinLeave.getView().add(gmsJoinLeaveMemberId);
+    prepareAndInstallView(gmsJoinLeaveMemberId, createMemberList(gmsJoinLeaveMemberId,mockMembers[0]));
     gmsJoinLeave.getView().add(mockMembers[1]);
     LeaveRequestMessage msg = new LeaveRequestMessage(gmsJoinLeave.getMemberID(), gmsJoinLeave.getMemberID(), reason);
     msg.setSender(mockMembers[1]);
@@ -558,7 +554,7 @@ public class GMSJoinLeaveJUnitTest {
   @Test
   public void testBecomeCoordinatorOnStartup() throws Exception {
     initMocks();
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     long giveup = System.currentTimeMillis() + 20000;
     while (System.currentTimeMillis() < giveup && !gmsJoinLeave.isCoordinator()) {
       Thread.sleep(1000);
@@ -616,7 +612,7 @@ public class GMSJoinLeaveJUnitTest {
     NetView oldView = gmsJoinLeave.getView();
     oldView.add(gmsJoinLeaveMemberId);
     InternalDistributedMember creator = oldView.getCreator();
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     NetView view = new NetView(2, gmsJoinLeave.getView().getViewId()+1);
     view.setCreator(creator);
     view.add(creator);
@@ -665,7 +661,7 @@ public class GMSJoinLeaveJUnitTest {
   @Test
   public void testNetworkPartitionMessageReceived() throws Exception {
     initMocks();
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     NetworkPartitionMessage message = new NetworkPartitionMessage();
     gmsJoinLeave.processMessage(message);
     verify(manager).forceDisconnect(any(String.class));
@@ -838,7 +834,7 @@ public class GMSJoinLeaveJUnitTest {
     gmsJoinLeaveMemberId.setVmViewId(1);
     members.add(gmsJoinLeaveMemberId);
     prepareAndInstallView(gmsJoinLeaveMemberId, members);
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     GMSJoinLeave.ViewBroadcaster b = gmsJoinLeave.new ViewBroadcaster();
     b.run();
     verify(messenger).sendUnreliably(isA(InstallViewMessage.class));
@@ -945,20 +941,22 @@ public class GMSJoinLeaveJUnitTest {
         System.getProperties().remove(GMSJoinLeave.BYPASS_DISCOVERY_PROPERTY);
       }
   }
-  
-  @Test
-  public void testJoinResponseMsgWithBecomeCoordinator() throws Exception {
-    initMocks(false);
-    gmsJoinLeaveMemberId.getNetMember().setPreferredForCoordinator(false);
-    JoinRequestMessage reqMsg = new JoinRequestMessage(gmsJoinLeaveMemberId, mockMembers[0], null, 56734);
-    InternalDistributedMember ids = new InternalDistributedMember("localhost", 97898);
-    ids.getNetMember().setPreferredForCoordinator(true);
-    gmsJoinLeave.processMessage(reqMsg);
-    ArgumentCaptor<JoinResponseMessage> ac = ArgumentCaptor.forClass(JoinResponseMessage.class);
-    verify(messenger).send(ac.capture());
-    
-    assertTrue("Should have asked for becoming a coordinator", ac.getValue().getBecomeCoordinator());
-  }
+
+//  With the removal of the JoinResponse message from GMSJoinLeave.processJoinRequest (GEODE-870)
+//  This test now seems to be invalid
+//  @Test
+//  public void testJoinResponseMsgWithBecomeCoordinator() throws Exception {
+//    initMocks(false);
+//    gmsJoinLeaveMemberId.getNetMember().setPreferredForCoordinator(false);
+//    JoinRequestMessage reqMsg = new JoinRequestMessage(gmsJoinLeaveMemberId, mockMembers[0], null, 56734);
+//    InternalDistributedMember ids = new InternalDistributedMember("localhost", 97898);
+//    ids.getNetMember().setPreferredForCoordinator(true);
+//    gmsJoinLeave.processMessage(reqMsg);
+//    ArgumentCaptor<JoinResponseMessage> ac = ArgumentCaptor.forClass(JoinResponseMessage.class);
+//    verify(messenger).send(ac.capture());
+//
+//    assertTrue("Should have asked for becoming a coordinator", ac.getValue().getBecomeCoordinator());
+//  }
   
   @Test
   public void testNetworkPartionMessage() throws Exception {
@@ -1004,8 +1002,7 @@ public class GMSJoinLeaveJUnitTest {
   @Test
   public void testPreparedViewFoundDuringBecomeCoordinator() throws Exception {
     initMocks(false);
-    prepareAndInstallView(mockMembers[0], createMemberList(mockMembers[0], gmsJoinLeaveMemberId));
-    
+    prepareAndInstallView(gmsJoinLeaveMemberId, createMemberList(gmsJoinLeaveMemberId,mockMembers[0]));
     // a new member is joining
     NetView preparedView = new NetView(gmsJoinLeave.getView(), gmsJoinLeave.getView().getViewId()+5);
     mockMembers[1].setVmViewId(preparedView.getViewId());
@@ -1014,7 +1011,7 @@ public class GMSJoinLeaveJUnitTest {
     InstallViewMessage msg = new InstallViewMessage(preparedView, null, true);
     gmsJoinLeave.processMessage(msg);
     
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
 
     Thread.sleep(2000);
     ViewCreator vc = gmsJoinLeave.getViewCreator();
@@ -1152,7 +1149,7 @@ public class GMSJoinLeaveJUnitTest {
   public void testRemoveRequestWhileWaitingForFinalResponse() throws Exception {
     initMocks(true, true);
     
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     
     installView();
     
@@ -1174,7 +1171,7 @@ public class GMSJoinLeaveJUnitTest {
   public void testLeaveRequestWhileWaitingForFinalResponse() throws Exception {
     initMocks(true, true);
     
-    gmsJoinLeave.becomeCoordinatorForTest();
+    GMSJoinLeaveTestHelper.becomeCoordinatorForTest(gmsJoinLeave);
     
     installView();
     

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-core/src/test/resources/com/gemstone/gemfire/codeAnalysis/sanctionedDataSerializables.txt
----------------------------------------------------------------------
diff --git a/geode-core/src/test/resources/com/gemstone/gemfire/codeAnalysis/sanctionedDataSerializables.txt b/geode-core/src/test/resources/com/gemstone/gemfire/codeAnalysis/sanctionedDataSerializables.txt
index 472d821..dd48df4 100644
--- a/geode-core/src/test/resources/com/gemstone/gemfire/codeAnalysis/sanctionedDataSerializables.txt
+++ b/geode-core/src/test/resources/com/gemstone/gemfire/codeAnalysis/sanctionedDataSerializables.txt
@@ -365,8 +365,8 @@ fromData,22,2a2bb900130100b500022a2bb80014c00015b50003b1
 toData,19,2b2ab40002b9001102002ab400032bb80012b1
 
 com/gemstone/gemfire/distributed/internal/membership/gms/messages/InstallViewMessage,2
-fromData,39,2a2bb700102ab800112bb90012010032b500042a2bb80013c00014b500022a2bb80013b50005b1
-toData,35,2a2bb7000c2b2ab40004b6000db9000e02002ab400022bb8000f2ab400052bb8000fb1
+fromData,49,2a2bb700122a2bb900130100b500072ab800142bb90013010032b500042a2bb80015c00016b500022a2bb80015b50005b1
+toData,45,2a2bb7000e2b2ab40007b9000f02002b2ab40004b60010b9000f02002ab400022bb800112ab400052bb80011b1
 
 com/gemstone/gemfire/distributed/internal/membership/gms/messages/JoinRequestMessage,2
 fromData,38,2a2bb80019c0001ab500042a2bb80019b500052a2bb8001bb500022a2bb9001c0100b6001db1

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83e4d249/geode-cq/src/test/java/com/gemstone/gemfire/cache/query/cq/dunit/CqQueryUsingPoolDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-cq/src/test/java/com/gemstone/gemfire/cache/query/cq/dunit/CqQueryUsingPoolDUnitTest.java b/geode-cq/src/test/java/com/gemstone/gemfire/cache/query/cq/dunit/CqQueryUsingPoolDUnitTest.java
index 680112b..f333b77 100644
--- a/geode-cq/src/test/java/com/gemstone/gemfire/cache/query/cq/dunit/CqQueryUsingPoolDUnitTest.java
+++ b/geode-cq/src/test/java/com/gemstone/gemfire/cache/query/cq/dunit/CqQueryUsingPoolDUnitTest.java
@@ -2807,7 +2807,9 @@ public class CqQueryUsingPoolDUnitTest extends CacheTestCase {
     VM server2 = host.getVM(1);
     VM client1 = host.getVM(2);
     VM client2 = host.getVM(3);
-    
+
+    Wait.pause(3 * 1000);
+
     createServer(server1);
     
     final int port1 = server1.invoke(() -> CqQueryUsingPoolDUnitTest.getCacheServerPort());


[5/7] incubator-geode git commit: GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination

Posted by ud...@apache.org.
GEODE-870: Handling multiple concurrent locator restarts. Elder locator nomination


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/83a6dc31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/83a6dc31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/83a6dc31

Branch: refs/heads/feature/GEODE-870
Commit: 83a6dc31eb16d26175608532f7c6ea8105e64e95
Parents: dfe248d
Author: Udo Kohlmeyer <uk...@pivotal.io>
Authored: Wed Feb 10 12:19:33 2016 +1100
Committer: Udo Kohlmeyer <uk...@pivotal.io>
Committed: Thu Feb 25 10:39:49 2016 +1100

----------------------------------------------------------------------
 .../gemfire/distributed/LocatorDUnitTest.java         | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/83a6dc31/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
index c800acf..eb3cc62 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/LocatorDUnitTest.java
@@ -1434,8 +1434,8 @@ public class LocatorDUnitTest extends DistributedTestCase {
     this.port1 = freeTCPPorts[0];
     this.port2 = freeTCPPorts[1];
     int port3 = freeTCPPorts[2];
-    deleteLocatorStateFile(port1, port2, port3);
-    final String host0 = getServerHostName(host);
+    DistributedTestUtils.deleteLocatorStateFile(port1, port2, port3);
+    final String host0 = NetworkUtils.getServerHostName(host);
     final String locators = host0 + "[" + port1 + "]," +
         host0 + "[" + port2 + "]," +
         host0 + "[" + port3 + "]";
@@ -1488,7 +1488,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
             return null;
           }
         };
-        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+        Wait.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
 
         // three applications plus
         assertEquals(6, system.getDM().getViewMembers().size());
@@ -1512,7 +1512,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
             return null;
           }
         };
-        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+        Wait.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
 
         final String newLocators = host0 + "[" + port2 + "]," +
             host0 + "[" + port3 + "]";
@@ -1540,7 +1540,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
             return null;
           }
         };
-        DistributedTestCase.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
+        Wait.waitForCriterion(waitCriterion, 10 * 1000, 200, true);
 
         int netviewId = vm1.invoke(() -> GMSJoinLeaveHelper.getViewId());
         assertEquals(netviewId, (int) vm2.invoke(() -> GMSJoinLeaveHelper.getViewId()));
@@ -1573,7 +1573,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         try {
           Locator.startLocatorAndDS((int) args[0], logFile, (Properties) args[1]);
         } catch (IOException ex) {
-          fail("While starting process on port " + args[0], ex);
+          com.gemstone.gemfire.test.dunit.Assert.fail("While starting process on port " + args[0], ex);
         }
       }
     });
@@ -1586,7 +1586,7 @@ public class LocatorDUnitTest extends DistributedTestCase {
         try {
           Locator.startLocatorAndDS((int) args[0], logFile, (Properties) args[1]);
         } catch (IOException ex) {
-          fail("While starting process on port " + args[0], ex);
+          com.gemstone.gemfire.test.dunit.Assert.fail("While starting process on port " + args[0], ex);
         }
       }
     });