You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kl...@apache.org on 2016/02/09 02:29:56 UTC

[03/50] [abbrv] incubator-geode git commit: Changing HealthMonitor.checkIfAvailable() to perform "final" check

Changing HealthMonitor.checkIfAvailable() to perform "final" check

Somehow we missed modifying checkIfAvailable to use the "final" check
method after Jianxia implemented the tcp/ip checks.  This modifies
checkIfAvailable to do so.

I also noticed an exception being thrown while closing a PR during
forced-disconnect and changed the method to use getDM instead of
getDistributionManager, which can throw a cancellation exception.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/ab25e41c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/ab25e41c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/ab25e41c

Branch: refs/heads/feature/GEODE-268
Commit: ab25e41c6281ef90f04c7534e4d65ba4f24d7fc9
Parents: 77f9811
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Thu Feb 4 09:07:42 2016 -0800
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Thu Feb 4 09:11:13 2016 -0800

----------------------------------------------------------------------
 .../membership/gms/fd/GMSHealthMonitor.java     | 132 +++++++++++--------
 .../membership/gms/membership/GMSJoinLeave.java |  12 +-
 .../internal/cache/PartitionedRegion.java       |   8 +-
 .../gemfire/cache30/ReconnectDUnitTest.java     |   8 +-
 .../membership/gms/MembershipManagerHelper.java |   2 +-
 .../gms/fd/GMSHealthMonitorJUnitTest.java       |  86 ++++--------
 .../gms/membership/GMSJoinLeaveJUnitTest.java   |  20 +--
 7 files changed, 128 insertions(+), 140 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitor.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitor.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitor.java
index 172926b..44186ba 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitor.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitor.java
@@ -32,6 +32,7 @@ import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -44,6 +45,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.ThreadFactory;
@@ -565,11 +567,7 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
 
   @Override
   public boolean checkIfAvailable(DistributedMember mbr, String reason, boolean initiateRemoval) {
-    boolean pinged = doCheckMember((InternalDistributedMember) mbr);
-    if (!pinged && initiateRemoval) {
-      suspect((InternalDistributedMember) mbr, reason);
-    }
-    return pinged;
+    return inlineCheckIfAvailable(localAddress, currentView, initiateRemoval, (InternalDistributedMember)mbr, reason);
   }
 
   public void start() {
@@ -656,9 +654,11 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
               if (GMSHealthMonitor.this.playingDead) {
                 continue;
               }
-              // [bruce] do we really want a timeout on the server-side?
-//              socket.setSoTimeout((int) services.getConfig().getMemberTimeout());
               serverSocketExecutor.execute(new ClientSocketHandler(socket)); //start();  [bruce] I'm seeing a lot of failures due to this thread not being created fast enough, sometimes as long as 30 seconds
+            
+            } catch (RejectedExecutionException e) {
+              // this can happen during shutdown
+
             } catch (IOException e) {
               if (!isStopping) {
                 logger.trace("Unexpected exception", e);
@@ -1095,7 +1095,7 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
         logger.info("received suspect message from {} for {}: {}",
            sender, req.getSuspectMember(), req.getReason());
       }
-      doFinalCheck(sender, sMembers, cv, localAddress);
+      checkIfAvailable(sender, sMembers, cv);
     }// coordinator ends
     else {
 
@@ -1119,7 +1119,7 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
           logger.info("received suspect message from {} for {}: {}",
              sender, req.getSuspectMember(), req.getReason());
         }
-        doFinalCheck(sender, smbr, cv, localAddress);
+        checkIfAvailable(sender, smbr, cv);
       } else {
         recordSuspectRequests(sMembers, cv);
       }
@@ -1148,8 +1148,19 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
     }
   }
 
-  private void doFinalCheck(final InternalDistributedMember initiator,
-      List<SuspectRequest> sMembers, final NetView cv, InternalDistributedMember localAddress) {
+  /**
+   * performs a "final" health check on the member.  If failure-detection
+   * socket information is available for the member (in the view) then
+   * we attempt to connect to its socket and ask if it's the expected member.
+   * Otherwise we send a heartbeat request and wait for a reply.
+   *  
+   * @param initiator
+   * @param sMembers
+   * @param cv
+   * @param initiateRemoval
+   */
+  private void checkIfAvailable(final InternalDistributedMember initiator,
+      List<SuspectRequest> sMembers, final NetView cv) {
 
 //    List<InternalDistributedMember> membersChecked = new ArrayList<>(10);
     try {
@@ -1184,48 +1195,8 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
           @Override
           public void run() {
             try {
-              services.memberSuspected(initiator, mbr, reason);
-              long startTime = System.currentTimeMillis();
-              // for some reason we used to update the timestamp for the member
-              // with the startTime, but we don't want to do that because it looks
-              // like a heartbeat has been received
-
-              logger.info("Performing final check for suspect member {} reason={}", mbr, reason);
-              boolean pinged;
-              int port = cv.getFailureDetectionPort(mbr);
-              if (port <= 0) {
-                logger.info("Unable to locate failure detection port - requesting a heartbeat");
-                if (logger.isDebugEnabled()) {
-                  logger.debug("\ncurrent view: {}\nports: {}", cv, Arrays.toString(cv.getFailureDetectionPorts()));
-                }
-                pinged = GMSHealthMonitor.this.doCheckMember(mbr);
-                GMSHealthMonitor.this.stats.incFinalCheckRequestsSent();
-                GMSHealthMonitor.this.stats.incUdpFinalCheckRequestsSent();
-                if (pinged) {
-                  GMSHealthMonitor.this.stats.incFinalCheckResponsesReceived();
-                  GMSHealthMonitor.this.stats.incUdpFinalCheckResponsesReceived();
-                }
-              } else {
-                pinged = GMSHealthMonitor.this.doTCPCheckMember(mbr, port);
-              }
-
-              boolean failed = false;
-              if (!pinged && !isStopping) {
-                TimeStamp ts = memberTimeStamps.get(mbr);
-                if (ts == null || ts.getTime() <= startTime) {
-                  logger.info("Final check failed - requesting removal of suspect member " + mbr);
-                  services.getJoinLeave().remove(mbr, reason);
-                  failed = true;
-                } else {
-                  logger.info("Final check failed but detected recent message traffic for suspect member " + mbr);
-                }
-              }
-              if (!failed) {
-                logger.info("Final check passed for suspect member " + mbr);
-              }
-              // whether it's alive or not, at this point we allow it to
-              // be watched again
-              suspectedMemberInView.remove(mbr);
+              inlineCheckIfAvailable(initiator, cv, true, mbr,
+                  reason);
             } catch (DistributedSystemDisconnectedException e) {
               return;
             } catch (Exception e) {
@@ -1234,6 +1205,8 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
               GMSHealthMonitor.this.suspectedMemberInView.remove(mbr);
             }
           }
+
+          
         });
         //      }// scheduling for final check and removing it..
       }
@@ -1241,6 +1214,59 @@ public class GMSHealthMonitor implements HealthMonitor, MessageHandler {
 //      membersInFinalCheck.removeAll(membersChecked);
     }
   }
+
+  private boolean inlineCheckIfAvailable(
+      final InternalDistributedMember initiator, final NetView cv,
+      boolean initiateRemoval,
+      final InternalDistributedMember mbr, final String reason) {
+
+    services.memberSuspected(initiator, mbr, reason);
+    long startTime = System.currentTimeMillis();
+    // for some reason we used to update the timestamp for the member
+    // with the startTime, but we don't want to do that because it looks
+    // like a heartbeat has been received
+
+    logger.info("Performing final check for suspect member {} reason={}", mbr, reason);
+    boolean pinged;
+    int port = cv.getFailureDetectionPort(mbr);
+    if (port <= 0) {
+      logger.info("Unable to locate failure detection port - requesting a heartbeat");
+      if (logger.isDebugEnabled()) {
+        logger.debug("\ncurrent view: {}\nports: {}", cv, Arrays.toString(cv.getFailureDetectionPorts()));
+      }
+      pinged = GMSHealthMonitor.this.doCheckMember(mbr);
+      GMSHealthMonitor.this.stats.incFinalCheckRequestsSent();
+      GMSHealthMonitor.this.stats.incUdpFinalCheckRequestsSent();
+      if (pinged) {
+        GMSHealthMonitor.this.stats.incFinalCheckResponsesReceived();
+        GMSHealthMonitor.this.stats.incUdpFinalCheckResponsesReceived();
+      }
+    } else {
+      pinged = GMSHealthMonitor.this.doTCPCheckMember(mbr, port);
+    }
+
+    boolean failed = false;
+    if (!pinged && !isStopping) {
+      TimeStamp ts = memberTimeStamps.get(mbr);
+      if (ts == null || ts.getTime() <= startTime) {
+        logger.info("Final check failed - requesting removal of suspect member " + mbr);
+        if (initiateRemoval) {
+          services.getJoinLeave().remove(mbr, reason);
+        }
+        failed = true;
+      } else {
+        logger.info("Final check failed but detected recent message traffic for suspect member " + mbr);
+      }
+    }
+    if (!failed) {
+      logger.info("Final check passed for suspect member " + mbr);
+    }
+    // whether it's alive or not, at this point we allow it to
+    // be watched again
+    suspectedMemberInView.remove(mbr);
+    return !failed;
+  }
+    
   @Override
   public void memberShutdown(DistributedMember mbr, String reason) {
   }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
index 0b0cfa0..0f16ba9 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeave.java
@@ -2161,16 +2161,16 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
       filterMembers(mbrs, newLeaves, LEAVE_REQUEST_MESSAGE);   
       
       for (InternalDistributedMember mbr : mbrs) {
-        final InternalDistributedMember fmbr = mbr;
         checkers.add(new Callable<InternalDistributedMember>() {
           @Override
           public InternalDistributedMember call() throws Exception {
             // return the member id if it fails health checks
-            InternalDistributedMember mbr = GMSJoinLeave.this.checkIfAvailable(fmbr);
+            boolean available = GMSJoinLeave.this.checkIfAvailable(mbr);
             
             synchronized (viewRequests) {
-              if(mbr != null)
+              if (available) {
                 mbrs.remove(mbr);
+              }
               viewRequests.notifyAll();
             }
             return mbr;
@@ -2259,14 +2259,14 @@ public class GMSJoinLeave implements JoinLeave, MessageHandler {
     }
   }
   
-  InternalDistributedMember checkIfAvailable(InternalDistributedMember fmbr) {
+  boolean checkIfAvailable(InternalDistributedMember fmbr) {
  // return the member id if it fails health checks
     logger.info("checking state of member " + fmbr);
     if (services.getHealthMonitor().checkIfAvailable(fmbr, "Member failed to acknowledge a membership view", false)) {
       logger.info("member " + fmbr + " passed availability check");
-      return fmbr;
+      return true;
     }
     logger.info("member " + fmbr + " failed availability check");
-    return null;
+    return false;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
index 09e945b..b145a91 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
@@ -9760,17 +9760,13 @@ public class PartitionedRegion extends LocalRegion implements
     }
 
     // Include current VM in the status...
-    // remoteInfos.add(new Object[] {
-    // getDistributionManager().getId(),
-    // new Boolean(
-    // getRegionAdvisor().getBucket(bucketId).isHosting())});
     if (getRegionAdvisor().getBucket(bucketId).isHosting()) {
       if (getRegionAdvisor().isPrimaryForBucket(bucketId)) {
-        remoteInfos.add(new Object[] { getDistributionManager().getId(),
+        remoteInfos.add(new Object[] { getSystem().getDM().getId(),
             Boolean.TRUE, "" });
       }
       else {
-        remoteInfos.add(new Object[] { getDistributionManager().getId(),
+        remoteInfos.add(new Object[] { getSystem().getDM().getId(),
             Boolean.FALSE, "" });
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/test/java/com/gemstone/gemfire/cache30/ReconnectDUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/cache30/ReconnectDUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/cache30/ReconnectDUnitTest.java
index cdc3283..0700074 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/cache30/ReconnectDUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/cache30/ReconnectDUnitTest.java
@@ -434,9 +434,11 @@ public class ReconnectDUnitTest extends CacheTestCase
           fail("interrupted while waiting for reconnect");
         }
         assertTrue("expected system to be reconnected", ds.getReconnectedSystem() != null);
-        int viewId = MembershipManagerHelper.getMembershipManager(ds.getReconnectedSystem()).getView().getViewId();
-        int memberViewId = ((InternalDistributedMember)ds.getReconnectedSystem().getDistributedMember()).getVmViewId();
-        assertEquals("expected a new ID to be assigned", viewId, memberViewId);
+        int oldViewId = MembershipManagerHelper.getMembershipManager(ds).getLocalMember().getVmViewId();
+        int newViewId = ((InternalDistributedMember)ds.getReconnectedSystem().getDistributedMember()).getVmViewId();
+        if ( !(newViewId > oldViewId) ) {
+          fail("expected a new ID to be assigned.  oldViewId="+oldViewId + "; newViewId=" + newViewId);
+        }
         return ds.getReconnectedSystem().getDistributedMember();
       }
     });

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/MembershipManagerHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/MembershipManagerHelper.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/MembershipManagerHelper.java
index 2c2d4a9..cda4e29 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/MembershipManagerHelper.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/MembershipManagerHelper.java
@@ -41,7 +41,7 @@ public class MembershipManagerHelper
   /** returns the JGroupMembershipManager for the given distributed system */
   public static MembershipManager getMembershipManager(DistributedSystem sys) {
     InternalDistributedSystem isys = (InternalDistributedSystem)sys;
-    DistributionManager dm = (DistributionManager)isys.getDistributionManager();
+    DistributionManager dm = (DistributionManager)isys.getDM();
     MembershipManager mgr = dm.getMembershipManager();
     return mgr;
   }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java
index 70285c7..9562e41 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/fd/GMSHealthMonitorJUnitTest.java
@@ -163,30 +163,14 @@ public class GMSHealthMonitorJUnitTest {
    */
   @Test
   public void testHMNextNeighborVerify() throws IOException {
-
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-    gmsHealthMonitor.started();
-
-    gmsHealthMonitor.installView(v);
-
+    installAView();
     Assert.assertEquals(mockMembers.get(4), gmsHealthMonitor.getNextNeighbor());
-
   }
 
   @Test
   public void testHMNextNeighborAfterTimeout() throws Exception {
     System.out.println("testHMNextNeighborAfterTimeout starting");
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-//    System.out.printf("memberID is %s view is %s\n", mockMembers.get(3), v);
-    
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-    gmsHealthMonitor.started();
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     // allow the monitor to give up on the initial "next neighbor" and
     // move on to the one after it
@@ -201,7 +185,7 @@ public class GMSHealthMonitorJUnitTest {
     // neighbor should change to 5th
     System.out.println("testHMNextNeighborAfterTimeout ending");
     Assert.assertEquals("expected " + expected + " but found " + neighbor
-        + ".  view="+v, expected, neighbor);  
+        + ".  view="+joinLeave.getView(), expected, neighbor);  
   }
 
   /**
@@ -210,13 +194,7 @@ public class GMSHealthMonitorJUnitTest {
 
   @Test
   public void testHMNextNeighborBeforeTimeout() throws IOException {
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-    gmsHealthMonitor.started();
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     //Should we remove these sleeps and force the checkmember directly instead of waiting?
     try {
@@ -228,7 +206,7 @@ public class GMSHealthMonitorJUnitTest {
     // neighbor should be same
     System.out.println("next neighbor is " + gmsHealthMonitor.getNextNeighbor() +
         "\nmy address is " + mockMembers.get(3) +
-        "\nview is " + v);
+        "\nview is " + joinLeave.getView());
 
     Assert.assertEquals(mockMembers.get(4), gmsHealthMonitor.getNextNeighbor());
   }
@@ -238,14 +216,7 @@ public class GMSHealthMonitorJUnitTest {
    */
   @Test
   public void testSuspectMembersCalledThroughMemberCheckThread() throws Exception {
-    System.out.println("testSuspectMembersCalledThroughMemberCheckThread starting");
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-    gmsHealthMonitor.started();
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     // when the view is installed we start a heartbeat timeout.  After
     // that expires we request a heartbeat
@@ -257,18 +228,23 @@ public class GMSHealthMonitorJUnitTest {
     Assert.assertTrue(gmsHealthMonitor.getStats().getSuspectsSent() > 0);
   }
 
-  /***
-   * checks ping thread didn't sends suspectMembers message before timeout
-   */
-  @Test
-  public void testSuspectMembersNotCalledThroughPingThreadBeforeTimeout() {
+  private void installAView() {
+    System.out.println("testSuspectMembersCalledThroughMemberCheckThread starting");
     NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
 
     // 3rd is current member
     when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
     gmsHealthMonitor.started();
-    
+
     gmsHealthMonitor.installView(v);
+  }
+
+  /***
+   * checks ping thread didn't sends suspectMembers message before timeout
+   */
+  @Test
+  public void testSuspectMembersNotCalledThroughPingThreadBeforeTimeout() {
+    installAView();
     InternalDistributedMember neighbor = gmsHealthMonitor.getNextNeighbor();
 
     try {
@@ -286,12 +262,7 @@ public class GMSHealthMonitorJUnitTest {
    */
   @Test
   public void testSuspectMembersCalledThroughSuspectThread() throws Exception {
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-    
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     gmsHealthMonitor.suspect(mockMembers.get(1), "Not responding");
 
@@ -308,12 +279,7 @@ public class GMSHealthMonitorJUnitTest {
   @Test
   public void testSuspectMembersNotCalledThroughSuspectThreadBeforeTimeout() {
 
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     gmsHealthMonitor.suspect(mockMembers.get(1), "Not responding");
 
@@ -437,16 +403,19 @@ public class GMSHealthMonitorJUnitTest {
    */
   @Test
   public void testCheckIfAvailableNoHeartBeatDontRemoveMember() {
+    installAView();
     long startTime = System.currentTimeMillis();
-    boolean retVal = gmsHealthMonitor.checkIfAvailable(mockMembers.get(1), "Not responding", false);
+    boolean retVal = gmsHealthMonitor.checkIfAvailable(mockMembers.get(1), "Not responding", true);
     long timeTaken = System.currentTimeMillis() - startTime;
 
-    assertTrue("This should have taken member ping timeout 100ms ", timeTaken >= gmsHealthMonitor.memberTimeout);
     assertFalse("CheckIfAvailable should have return false", retVal);
+    assertTrue("This should have taken member ping timeout 100ms but it took " + timeTaken, timeTaken >= gmsHealthMonitor.memberTimeout);
   }
 
   @Test
   public void testCheckIfAvailableWithSimulatedHeartBeat() {
+    installAView();
+    
     InternalDistributedMember memberToCheck = mockMembers.get(1);
     HeartbeatMessage fakeHeartbeat = new HeartbeatMessage();
     fakeHeartbeat.setSender(memberToCheck);
@@ -466,12 +435,7 @@ public class GMSHealthMonitorJUnitTest {
   @Test
   public void testShutdown() {
 
-    NetView v = new NetView(mockMembers.get(0), 2, mockMembers);
-
-    // 3rd is current member
-    when(messenger.getMemberID()).thenReturn(mockMembers.get(3));
-
-    gmsHealthMonitor.installView(v);
+    installAView();
 
     gmsHealthMonitor.stop();
 

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ab25e41c/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
index 5becc6a..5b58c27 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/distributed/internal/membership/gms/membership/GMSJoinLeaveJUnitTest.java
@@ -139,10 +139,12 @@ public class GMSJoinLeaveJUnitTest {
     }
     mockOldMember = new InternalDistributedMember("localhost", 8700, Version.GFE_56);
 
-    if(useTestGMSJoinLeave)
+    if(useTestGMSJoinLeave) {
       gmsJoinLeave = new GMSJoinLeaveTest();
-    else
+    }
+    else {
       gmsJoinLeave = new GMSJoinLeave();
+    }
     gmsJoinLeave.init(services);
     gmsJoinLeave.start();
     gmsJoinLeave.started();
@@ -1105,7 +1107,7 @@ public class GMSJoinLeaveJUnitTest {
       super();
     }
     @Override
-    InternalDistributedMember checkIfAvailable(InternalDistributedMember fmbr) {
+    boolean checkIfAvailable(InternalDistributedMember fmbr) {
       if(removeMember != null) {
         try {
           if(removeMember.equals(fmbr)) {
@@ -1114,17 +1116,17 @@ public class GMSJoinLeaveJUnitTest {
           }
         } catch (InterruptedException e) {
         }
-        return fmbr;
-      }else if(leaveMember != null) {
+        return true;
+      } else if(leaveMember != null) {
         try {
-          if(leaveMember.equals(fmbr)) {
+          if (leaveMember.equals(fmbr)) {
             GMSJoinLeaveJUnitTest.this.processLeaveMessage(fmbr);
             Thread.sleep(1000000);
           }
         } catch (InterruptedException e) {
         }
-        return fmbr;
-      }else {
+        return true;
+      } else {
         return super.checkIfAvailable(fmbr);
       }
     }
@@ -1132,7 +1134,6 @@ public class GMSJoinLeaveJUnitTest {
   
   @Test
   public void testRemoveRequestWhileWaitingForFinalResponse() throws Exception {
-    String reason = "testing";
     initMocks(true, true);
     
     gmsJoinLeave.becomeCoordinatorForTest();
@@ -1155,7 +1156,6 @@ public class GMSJoinLeaveJUnitTest {
   
   @Test
   public void testLeaveRequestWhileWaitingForFinalResponse() throws Exception {
-    String reason = "testing";
     initMocks(true, true);
     
     gmsJoinLeave.becomeCoordinatorForTest();