You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by kl...@apache.org on 2017/02/22 17:33:37 UTC

[1/4] geode git commit: GEODE-2497 surprise members are never timed out during startup [Forced Update!]

Repository: geode
Updated Branches:
  refs/heads/feature/GEODE-2514 d6ca901f7 -> 7e85e08e0 (forced update)


GEODE-2497 surprise members are never timed out during startup

Merge of 8d45ca22737282abe279d3c863478f904f2e1926 and
a6dfa4ca630a82fcf92942a834f8255e86d2bfcb from feature/GEODE-2497.

Moved the creation of the timer to GMSMembershipManager.started()

Removed write-lock in timer-creation method since it's only called from
one place now

Altered the way that the timer-creation method finds the
InternalDistributedSystem.  The old way of using getAnyInstance() was
the primary source of the problem since it returns null until startup
is completed.

Altered the surprise-member unit test to ensure that it's using the
timer and not relying on installation of a new membership view to clean
things up.

Altered the surprise-member unit test to run faster.  It now completes in
under 10 seconds.

This closes #402


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/135bd77a
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/135bd77a
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/135bd77a

Branch: refs/heads/feature/GEODE-2514
Commit: 135bd77a11c10609310838bb77cf9bb57d38fee3
Parents: c6b941f
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Tue Feb 21 16:07:51 2017 -0800
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Tue Feb 21 16:30:19 2017 -0800

----------------------------------------------------------------------
 .../gms/mgr/GMSMembershipManager.java           | 190 +++++++++----------
 .../internal/DistributionManagerDUnitTest.java  |  72 +++----
 2 files changed, 122 insertions(+), 140 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/135bd77a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
index cf17025..050e201 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
@@ -128,7 +128,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Trick class to make the startup synch more visible in stack traces
-   * 
+   *
    * @see GMSMembershipManager#startupLock
    */
   static class EventProcessingLock {
@@ -143,7 +143,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
     /**
      * indicates whether the event is a departure, a surprise connect (i.e., before the view message
      * arrived), a view, or a regular message
-     * 
+     *
      * @see #SURPRISE_CONNECT
      * @see #VIEW
      * @see #MESSAGE
@@ -181,7 +181,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Create a surprise connect event
-     * 
+     *
      * @param member the member connecting
      */
     StartupEvent(final InternalDistributedMember member) {
@@ -191,7 +191,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Indicate if this is a surprise connect event
-     * 
+     *
      * @return true if this is a connect event
      */
     boolean isSurpriseConnect() {
@@ -200,7 +200,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Create a view event
-     * 
+     *
      * @param v the new view
      */
     StartupEvent(NetView v) {
@@ -210,7 +210,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Indicate if this is a view event
-     * 
+     *
      * @return true if this is a view event
      */
     boolean isGmsView() {
@@ -219,7 +219,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Create a message event
-     * 
+     *
      * @param d the message
      */
     StartupEvent(DistributionMessage d) {
@@ -229,7 +229,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Indicate if this is a message event
-     * 
+     *
      * @return true if this is a message event
      */
     boolean isDistributionMessage() {
@@ -248,14 +248,14 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * This is the latest view (ordered list of DistributedMembers) that has been installed
-   * 
+   *
    * All accesses to this object are protected via {@link #latestViewLock}
    */
   private NetView latestView = new NetView();
 
   /**
    * This is the lock for protecting access to latestView
-   * 
+   *
    * @see #latestView
    */
   private final ReadWriteLock latestViewLock = new ReentrantReadWriteLock();
@@ -290,11 +290,11 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * Members of the distributed system that we believe have shut down. Keys are instances of
    * {@link InternalDistributedMember}, values are Longs indicating the time this member was
    * shunned.
-   * 
+   *
    * Members are removed after {@link #SHUNNED_SUNSET} seconds have passed.
-   * 
+   *
    * Accesses to this list needs to be under the read or write lock of {@link #latestViewLock}
-   * 
+   *
    * @see System#currentTimeMillis()
    */
   // protected final Set shunnedMembers = Collections.synchronizedSet(new HashSet());
@@ -310,7 +310,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * per bug 39552, keep a list of members that have been shunned and for which a message is
    * printed. Contents of this list are cleared at the same time they are removed from
    * {@link #shunnedMembers}.
-   * 
+   *
    * Accesses to this list needs to be under the read or write lock of {@link #latestViewLock}
    */
   private final HashSet<DistributedMember> shunnedAndWarnedMembers = new HashSet<>();
@@ -326,7 +326,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * arrived, the member is removed from membership and member-left notification is performed.
    * <p>
    * > Accesses to this list needs to be under the read or write lock of {@link #latestViewLock}
-   * 
+   *
    * @see System#currentTimeMillis()
    */
   private final Map<InternalDistributedMember, Long> surpriseMembers = new ConcurrentHashMap<>();
@@ -344,7 +344,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Length of time, in seconds, that a member is retained in the zombie set
-   * 
+   *
    * @see #shunnedMembers
    */
   static private final int SHUNNED_SUNSET = Integer
@@ -368,7 +368,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   /**
    * A list of messages received during channel startup that couldn't be processed yet. Additions or
    * removals of this list must be synchronized via {@link #startupLock}.
-   * 
+   *
    * @since GemFire 5.0
    */
   private final LinkedList<StartupEvent> startupMessages = new LinkedList<>();
@@ -381,8 +381,8 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   /**
    * Insert our own MessageReceiver between us and the direct channel, in order to correctly filter
    * membership events.
-   * 
-   * 
+   *
+   *
    */
   class MyDCReceiver implements DirectChannelListener {
 
@@ -390,9 +390,9 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
     /**
      * Don't provide events until the caller has told us we are ready.
-     * 
+     *
      * Synchronization provided via GroupMembershipService.class.
-     * 
+     *
      * Note that in practice we only need to delay accepting the first client; we don't need to put
      * this check before every call...
      *
@@ -509,7 +509,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
           continue; // no additions processed after shutdown begins
         } else {
           boolean wasShunned = endShun(m); // bug #45158 - no longer shun a process that is now in
-                                           // view
+          // view
           if (wasShunned && logger.isDebugEnabled()) {
             logger.debug("No longer shunning {} as it is in the current membership view", m);
           }
@@ -576,7 +576,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
       // expire surprise members, add others to view
       long oldestAllowed = System.currentTimeMillis() - this.surpriseMemberTimeout;
       for (Iterator<Map.Entry<InternalDistributedMember, Long>> it =
-          surpriseMembers.entrySet().iterator(); it.hasNext();) {
+           surpriseMembers.entrySet().iterator(); it.hasNext();) {
         Map.Entry<InternalDistributedMember, Long> entry = it.next();
         Long birthtime = entry.getValue();
         if (birthtime.longValue() < oldestAllowed) {
@@ -608,7 +608,6 @@ public class GMSMembershipManager implements MembershipManager, Manager {
       }
       try {
         listener.viewInstalled(latestView);
-        startCleanupTimer();
       } catch (DistributedSystemDisconnectedException se) {
       }
     } finally {
@@ -616,9 +615,13 @@ public class GMSMembershipManager implements MembershipManager, Manager {
     }
   }
 
+  public boolean isCleanupTimerStarted() {
+    return this.cleanupTimer != null;
+  }
+
   /**
    * the timer used to perform periodic tasks
-   * 
+   *
    * Concurrency: protected by {@link #latestViewLock} ReentrantReadWriteLock
    */
   private SystemTimer cleanupTimer;
@@ -637,7 +640,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Joins the distributed system
-   * 
+   *
    * @throws GemFireConfigException - configuration error
    * @throws SystemConnectException - problem joining
    */
@@ -767,7 +770,9 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   }
 
   @Override
-  public void started() {}
+  public void started() {
+    startCleanupTimer();
+  }
 
 
   /** this is invoked by JoinLeave when there is a loss of quorum in the membership system */
@@ -843,7 +848,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Process a surprise connect event, or place it on the startup queue.
-   * 
+   *
    * @param member the member
    */
   protected void handleOrDeferSurpriseConnect(InternalDistributedMember member) {
@@ -875,7 +880,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * <p>
    * Must be called with {@link #latestViewLock} held. Waits until there is a stable view. If the
    * member has already been added, simply returns; else adds the member.
-   * 
+   *
    * @param dm the member joining
    */
   public boolean addSurpriseMember(DistributedMember dm) {
@@ -942,12 +947,6 @@ public class GMSMembershipManager implements MembershipManager, Manager {
         surpriseMembers.remove(member);
       } else {
 
-        // Now that we're sure the member is new, add them.
-        // make sure the surprise-member cleanup task is running
-        if (this.cleanupTimer == null) {
-          startCleanupTimer();
-        } // cleanupTimer == null
-
         // Ensure that the member is accounted for in the view
         // Conjure up a new view including the new member. This is necessary
         // because we are about to tell the listener about a new member, so
@@ -978,40 +977,39 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /** starts periodic task to perform cleanup chores such as expire surprise members */
   private void startCleanupTimer() {
+    if (this.dcReceiver == null) {
+      // junit tests don't provide a direct-channel receiver
+      return;
+    }
+    DistributedSystem ds = this.dcReceiver.getDM().getSystem();
+    this.cleanupTimer = new SystemTimer(ds, true);
+    SystemTimer.SystemTimerTask st = new SystemTimer.SystemTimerTask() {
+      @Override
+      public void run2() {
+        cleanUpSurpriseMembers();
+      }
+    };
+    this.cleanupTimer.scheduleAtFixedRate(st, surpriseMemberTimeout, surpriseMemberTimeout / 3);
+  }
+
+  // invoked from the cleanupTimer task
+  private void cleanUpSurpriseMembers() {
     latestViewWriteLock.lock();
     try {
-      if (this.cleanupTimer != null) {
-        return;
+      long oldestAllowed = System.currentTimeMillis() - surpriseMemberTimeout;
+      for (Iterator it = surpriseMembers.entrySet().iterator(); it.hasNext();) {
+        Map.Entry entry = (Map.Entry) it.next();
+        Long birthtime = (Long) entry.getValue();
+        if (birthtime.longValue() < oldestAllowed) {
+          it.remove();
+          InternalDistributedMember m = (InternalDistributedMember) entry.getKey();
+          logger.info(LocalizedMessage.create(
+              LocalizedStrings.GroupMembershipService_MEMBERSHIP_EXPIRING_MEMBERSHIP_OF_SURPRISE_MEMBER_0,
+              m));
+          removeWithViewLock(m, true,
+              "not seen in membership view in " + surpriseMemberTimeout + "ms");
+        }
       }
-      DistributedSystem ds = InternalDistributedSystem.getAnyInstance();
-      if (ds != null && ds.isConnected()) {
-        this.cleanupTimer = new SystemTimer(ds, true);
-        SystemTimer.SystemTimerTask st = new SystemTimer.SystemTimerTask() {
-          @Override
-          public void run2() {
-            latestViewWriteLock.lock();
-            try {
-              long oldestAllowed = System.currentTimeMillis() - surpriseMemberTimeout;
-              for (Iterator it = surpriseMembers.entrySet().iterator(); it.hasNext();) {
-                Map.Entry entry = (Map.Entry) it.next();
-                Long birthtime = (Long) entry.getValue();
-                if (birthtime.longValue() < oldestAllowed) {
-                  it.remove();
-                  InternalDistributedMember m = (InternalDistributedMember) entry.getKey();
-                  logger.info(LocalizedMessage.create(
-                      LocalizedStrings.GroupMembershipService_MEMBERSHIP_EXPIRING_MEMBERSHIP_OF_SURPRISE_MEMBER_0,
-                      m));
-                  removeWithViewLock(m, true,
-                      "not seen in membership view in " + surpriseMemberTimeout + "ms");
-                }
-              }
-            } finally {
-              latestViewWriteLock.unlock();
-            }
-          }
-        };
-        this.cleanupTimer.scheduleAtFixedRate(st, surpriseMemberTimeout, surpriseMemberTimeout / 3);
-      } // ds != null && ds.isConnected()
     } finally {
       latestViewWriteLock.unlock();
     }
@@ -1019,7 +1017,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Dispatch the distribution message, or place it on the startup queue.
-   * 
+   *
    * @param msg the message to process
    */
   protected void handleOrDeferMessage(DistributionMessage msg) {
@@ -1068,7 +1066,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * <p>
    * It is possible to receive messages not consistent with our view. We handle this here, and
    * generate an uplevel event if necessary
-   * 
+   *
    * @param msg the message
    */
   private void dispatchMessage(DistributionMessage msg) {
@@ -1103,7 +1101,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
     }
 
     if (shunned) { // bug #41538 - shun notification must be outside synchronization to avoid
-                   // hanging
+      // hanging
       warnShun(m);
       if (logger.isTraceEnabled(LogMarker.DISTRIBUTION_VIEWS)) {
         logger.trace(LogMarker.DISTRIBUTION_VIEWS,
@@ -1118,7 +1116,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Process a new view object, or place on the startup queue
-   * 
+   *
    * @param viewArg the new view
    */
   protected void handleOrDeferViewEvent(NetView viewArg) {
@@ -1152,14 +1150,14 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   @Override
   public void memberSuspected(InternalDistributedMember initiator,
-      InternalDistributedMember suspect, String reason) {
+                              InternalDistributedMember suspect, String reason) {
     SuspectMember s = new SuspectMember(initiator, suspect, reason);
     handleOrDeferSuspect(s);
   }
 
   /**
    * Process a new view object, or place on the startup queue
-   * 
+   *
    * @param suspectInfo the suspectee and suspector
    */
   protected void handleOrDeferSuspect(SuspectMember suspectInfo) {
@@ -1196,7 +1194,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Dispatch routine for processing a single startup event
-   * 
+   *
    * @param o the startup event to handle
    */
   private void processStartupEvent(StartupEvent o) {
@@ -1350,7 +1348,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * <p>
    * If no members have partition detection enabled, there will be no lead member and this method
    * will return null.
-   * 
+   *
    * @return the lead member associated with the latest view
    */
   public DistributedMember getLeadMember() {
@@ -1368,7 +1366,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * test hook
-   * 
+   *
    * @return the current membership view coordinator
    */
   public DistributedMember getCoordinator() {
@@ -1415,7 +1413,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Ensure that the critical classes from components get loaded.
-   * 
+   *
    * @see SystemFailure#loadEmergencyClasses()
    */
   public static void loadEmergencyClasses() {
@@ -1430,7 +1428,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   /**
    * Close the receiver, avoiding all potential deadlocks and eschewing any attempts at being
    * graceful.
-   * 
+   *
    * @see SystemFailure#emergencyClose()
    */
   public void emergencyClose() {
@@ -1652,7 +1650,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Perform the grossness associated with sending a message over a DirectChannel
-   * 
+   *
    * @param destinations the list of destinations
    * @param content the message
    * @param theStats the statistics object to update
@@ -1716,18 +1714,18 @@ public class GMSMembershipManager implements MembershipManager, Manager {
         return null;
 
       List<InternalDistributedMember> members = (List<InternalDistributedMember>) ex.getMembers(); // We
-                                                                                                   // need
-                                                                                                   // to
-                                                                                                   // return
-                                                                                                   // this
-                                                                                                   // list
-                                                                                                   // of
-                                                                                                   // failures
+      // need
+      // to
+      // return
+      // this
+      // list
+      // of
+      // failures
 
       // SANITY CHECK: If we fail to send a message to an existing member
       // of the view, we have a serious error (bug36202).
       NetView view = services.getJoinLeave().getView(); // grab a recent view, excluding shunned
-                                                        // members
+      // members
 
       // Iterate through members and causes in tandem :-(
       Iterator it_mem = members.iterator();
@@ -1805,7 +1803,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   }
 
   public Set<InternalDistributedMember> send(InternalDistributedMember[] destinations,
-      DistributionMessage msg, DMStats theStats) throws NotSerializableException {
+                                             DistributionMessage msg, DMStats theStats) throws NotSerializableException {
     Set<InternalDistributedMember> result;
     boolean allDestinations = msg.forAll();
 
@@ -1866,7 +1864,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
     }
 
     boolean sendViaMessenger = isForceUDPCommunications(); // enable when bug #46438 is fixed: ||
-                                                           // msg.sendViaUDP();
+    // msg.sendViaUDP();
 
     if (useMcast || tcpDisabled || sendViaMessenger) {
       checkAddressesForUUIDs(destinations);
@@ -1933,7 +1931,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Clean up and create consistent new view with member removed. No uplevel events are generated.
-   * 
+   *
    * Must be called with the {@link #latestViewLock} held.
    */
   private void destroyMember(final InternalDistributedMember member, final String reason) {
@@ -1996,12 +1994,12 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   /**
    * Indicate whether the given member is in the zombie list (dead or dying)
-   * 
+   *
    * @param m the member in question
-   * 
+   *
    *        This also checks the time the given member was shunned, and has the side effect of
    *        removing the member from the list if it was shunned too far in the past.
-   * 
+   *
    *        Concurrency: protected by {@link #latestViewLock} ReentrantReadWriteLock
    *
    * @return true if the given member is a zombie
@@ -2049,9 +2047,9 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * during view processing.
    * <p>
    * Like isShunned, this method holds the view lock while executing
-   * 
+   *
    * Concurrency: protected by {@link #latestViewLock} ReentrantReadWriteLock
-   * 
+   *
    * @param m the member in question
    * @return true if the given member is a surprise member
    */
@@ -2072,7 +2070,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   /**
    * for testing we need to be able to inject surprise members into the view to ensure that
    * sunsetting works properly
-   * 
+   *
    * @param m the member ID to add
    * @param birthTime the millisecond clock time that the member was first seen
    */
@@ -2106,7 +2104,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
    * really old.
    * <p>
    * Must be called with {@link #latestViewLock} held and the view stable.
-   * 
+   *
    * @param m the member to add
    */
   private void addShunnedMember(InternalDistributedMember m) {

http://git-wip-us.apache.org/repos/asf/geode/blob/135bd77a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
index ccce013..5e3cf3e 100644
--- a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
@@ -17,6 +17,7 @@ package org.apache.geode.distributed.internal;
 import static org.apache.geode.distributed.ConfigurationProperties.*;
 import static org.apache.geode.test.dunit.Assert.*;
 
+import org.apache.geode.test.dunit.rules.DistributedRestoreSystemProperties;
 import org.awaitility.Awaitility;
 
 import java.net.InetAddress;
@@ -28,6 +29,7 @@ import org.apache.geode.test.junit.categories.MembershipTest;
 import org.apache.logging.log4j.Logger;
 import org.junit.Assert;
 import org.junit.Ignore;
+import org.junit.Rule;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
@@ -74,6 +76,9 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
 
   public static DistributedSystem ds;
 
+  @Rule
+  public DistributedRestoreSystemProperties restoreSystemProperties = new DistributedRestoreSystemProperties();
+
   /**
    * Clears the exceptionInThread flag in the given distribution manager.
    */
@@ -137,18 +142,14 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
     InternalDistributedMember idm = mgr.getLocalMember();
     // TODO GMS needs to have a system property allowing the bind-port to be set
     System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "jg-bind-port", "" + idm.getPort());
-    try {
-      sys.disconnect();
-      sys = getSystem();
-      mgr = MembershipManagerHelper.getMembershipManager(sys);
-      sys.disconnect();
-      InternalDistributedMember idm2 = mgr.getLocalMember();
-      org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
-          .info("original ID=" + idm + " and after connecting=" + idm2);
-      assertTrue("should not have used a different udp port", idm.getPort() == idm2.getPort());
-    } finally {
-      System.getProperties().remove(DistributionConfig.GEMFIRE_PREFIX + "jg-bind-port");
-    }
+    sys.disconnect();
+    sys = getSystem();
+    mgr = MembershipManagerHelper.getMembershipManager(sys);
+    sys.disconnect();
+    InternalDistributedMember idm2 = mgr.getLocalMember();
+    org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
+        .info("original ID=" + idm + " and after connecting=" + idm2);
+    assertTrue("should not have used a different udp port", idm.getPort() == idm2.getPort());
   }
 
   /**
@@ -158,11 +159,12 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
    * should be gone and force more view processing to have it scrubbed from the set.
    **/
   @Test
-  public void testSurpriseMemberHandling() {
-    VM vm0 = Host.getHost(0).getVM(0);
+  public void testSurpriseMemberHandling() throws Exception {
 
+    System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "surprise-member-timeout", "3000");
     InternalDistributedSystem sys = getSystem();
     MembershipManager mgr = MembershipManagerHelper.getMembershipManager(sys);
+    assertTrue(((GMSMembershipManager) mgr).isCleanupTimerStarted());
 
     try {
       InternalDistributedMember mbr =
@@ -172,7 +174,8 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
 
       // if the view number isn't being recorded correctly the test will pass but the
       // functionality is broken
-      Assert.assertTrue("expected view ID to be greater than zero", mgr.getView().getViewId() > 0);
+      Assert.assertTrue("expected view ID to be greater than zero",
+          mgr.getView().getViewId() > 0);
 
       int oldViewId = mbr.getVmViewId();
       mbr.setVmViewId((int) mgr.getView().getViewId() - 1);
@@ -180,19 +183,13 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
           .info("current membership view is " + mgr.getView());
       org.apache.geode.test.dunit.LogWriterUtils.getLogWriter()
           .info("created ID " + mbr + " with view ID " + mbr.getVmViewId());
-      sys.getLogWriter()
-          .info("<ExpectedException action=add>attempt to add old member</ExpectedException>");
-      sys.getLogWriter()
-          .info("<ExpectedException action=add>Removing shunned GemFire node</ExpectedException>");
-      try {
-        boolean accepted = mgr.addSurpriseMember(mbr);
-        Assert.assertTrue("member with old ID was not rejected (bug #44566)", !accepted);
-      } finally {
-        sys.getLogWriter()
-            .info("<ExpectedException action=remove>attempt to add old member</ExpectedException>");
-        sys.getLogWriter().info(
-            "<ExpectedException action=remove>Removing shunned GemFire node</ExpectedException>");
-      }
+
+      IgnoredException.addIgnoredException("attempt to add old member");
+      IgnoredException.addIgnoredException("Removing shunned GemFire node");
+
+      boolean accepted = mgr.addSurpriseMember(mbr);
+      Assert.assertTrue("member with old ID was not rejected (bug #44566)", !accepted);
+
       mbr.setVmViewId(oldViewId);
 
       // now forcibly add it as a surprise member and show that it is reaped
@@ -203,28 +200,15 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
       MembershipManagerHelper.addSurpriseMember(sys, mbr, birthTime);
       assertTrue("Member was not a surprise member", mgr.isSurpriseMember(mbr));
 
-      // force a real view change
-      SerializableRunnable connectDisconnect = new SerializableRunnable() {
-        public void run() {
-          getSystem().disconnect();
-        }
-      };
-      vm0.invoke(connectDisconnect);
-
       if (birthTime < (System.currentTimeMillis() - timeout)) {
         return; // machine is too busy and we didn't get enough CPU to perform more assertions
       }
       assertTrue("Member was incorrectly removed from surprise member set",
           mgr.isSurpriseMember(mbr));
 
-      try {
-        Thread.sleep(gracePeriod);
-      } catch (InterruptedException e) {
-        fail("test was interrupted", e);
-      }
-
-      vm0.invoke(connectDisconnect);
-      assertTrue("Member was not removed from surprise member set", !mgr.isSurpriseMember(mbr));
+      Awaitility.await("waiting for member to be removed")
+          .atMost((timeout / 3) + gracePeriod, TimeUnit.MILLISECONDS)
+          .until(() -> !mgr.isSurpriseMember(mbr));
 
     } finally {
       if (sys != null && sys.isConnected()) {


[3/4] geode git commit: GEODE-2497 removing spots

Posted by kl...@apache.org.
GEODE-2497 removing spots


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/8065b672
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/8065b672
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/8065b672

Branch: refs/heads/feature/GEODE-2514
Commit: 8065b67292049adf6027694d744eff43253aba92
Parents: 3474fa7
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Wed Feb 22 08:48:43 2017 -0800
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Wed Feb 22 08:48:43 2017 -0800

----------------------------------------------------------------------
 .../distributed/internal/DistributionManagerDUnitTest.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/8065b672/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
index 08e3b87..24d8a8a 100644
--- a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
@@ -200,9 +200,9 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
       MembershipManagerHelper.addSurpriseMember(sys, mbr, birthTime);
       assertTrue("Member was not a surprise member", mgr.isSurpriseMember(mbr));
 
-//      if (birthTime < (System.currentTimeMillis() - timeout)) {
-//        return; // machine is too busy and we didn't get enough CPU to perform more assertions
-//      }
+      // if (birthTime < (System.currentTimeMillis() - timeout)) {
+      // return; // machine is too busy and we didn't get enough CPU to perform more assertions
+      // }
 
       Awaitility.await("waiting for member to be removed")
           .atMost((timeout / 3) + gracePeriod, TimeUnit.MILLISECONDS)


[4/4] geode git commit: GEODE-2514: add new tests for statistics archive rolling and removal

Posted by kl...@apache.org.
GEODE-2514: add new tests for statistics archive rolling and removal

* add MainWithChildrenRollingFileHandlerIntegrationTest
* add StatArchiveHandlerIntegrationTest
* expand DiskSpaceLimitIntegrationTest


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/7e85e08e
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/7e85e08e
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/7e85e08e

Branch: refs/heads/feature/GEODE-2514
Commit: 7e85e08e0d5cc4a65c98270ea253e5e7331ca81e
Parents: 8065b67
Author: Kirk Lund <kl...@apache.org>
Authored: Tue Feb 21 13:09:24 2017 -0800
Committer: Kirk Lund <kl...@apache.org>
Committed: Wed Feb 22 09:33:22 2017 -0800

----------------------------------------------------------------------
 .../io/MainWithChildrenRollingFileHandler.java  |   6 +-
 .../internal/statistics/StatArchiveHandler.java |   6 +-
 ...ildrenRollingFileHandlerIntegrationTest.java |  96 ++++++++
 .../DiskSpaceLimitIntegrationTest.java          | 245 ++++++++++++++++++-
 .../StatArchiveHandlerIntegrationTest.java      | 243 ++++++++++++++++++
 5 files changed, 582 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/7e85e08e/geode-core/src/main/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandler.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandler.java b/geode-core/src/main/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandler.java
index 20d1c4f..11d10af 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandler.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandler.java
@@ -206,7 +206,11 @@ public class MainWithChildrenRollingFileHandler implements RollingFileHandler {
     return tmp;
   }
 
-  private Pattern getFilePattern(String name) {
+  Pattern getFilePattern(String name) {
+    if (name == null || "".equals(name.trim())) {
+      throw new IllegalArgumentException("Name must not be empty");
+    }
+
     int extIdx = name.lastIndexOf('.');
     String ext = "";
     if (extIdx != -1) {

http://git-wip-us.apache.org/repos/asf/geode/blob/7e85e08e/geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveHandler.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveHandler.java b/geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveHandler.java
index 3eb9730..03b0f89 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveHandler.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/statistics/StatArchiveHandler.java
@@ -446,7 +446,7 @@ public class StatArchiveHandler implements SampleHandler {
    * @return the modified archive file name to use; it is modified by applying mainArchiveId and
    *         archiveId to the name for supporting file rolling
    */
-  private File getRollingArchiveName(File archive, boolean archiveClosed) {
+  File getRollingArchiveName(File archive, boolean archiveClosed) {
     if (mainArchiveId != -1) {
       // leave mainArchiveId as is. Bump archiveId.
     } else {
@@ -538,7 +538,7 @@ public class StatArchiveHandler implements SampleHandler {
     return result;
   }
 
-  private void initMainArchiveId(File archive) {
+  void initMainArchiveId(File archive) {
     if (mainArchiveId != -1) {
       // already initialized
       return;
@@ -598,7 +598,7 @@ public class StatArchiveHandler implements SampleHandler {
    * @return the modified archive file name to use; it is modified by applying the next main id if
    *         any files in the dir already have a main id in the file name
    */
-  private File getRenameArchiveName(File archive) {
+  File getRenameArchiveName(File archive) {
     File dir = archive.getAbsoluteFile().getParentFile();
     int previousMainId = this.rollingFileHandler.calcNextMainId(dir, false);
     if (previousMainId == 0) {

http://git-wip-us.apache.org/repos/asf/geode/blob/7e85e08e/geode-core/src/test/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandlerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandlerIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandlerIntegrationTest.java
new file mode 100644
index 0000000..5ea77c4
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/internal/io/MainWithChildrenRollingFileHandlerIntegrationTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.io;
+
+import static org.assertj.core.api.Assertions.*;
+
+import java.util.regex.Pattern;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+@Category(IntegrationTest.class)
+public class MainWithChildrenRollingFileHandlerIntegrationTest {
+
+  private String name;
+  private MainWithChildrenRollingFileHandler handler;
+
+  @Rule
+  public TestName testName = new TestName();
+
+  @Before
+  public void before() throws Exception {
+    this.name = this.testName.getMethodName();
+    this.handler = new MainWithChildrenRollingFileHandler();
+  }
+
+  @Test
+  public void getFilePattern_matchesFilesWithBothIds() throws Exception {
+    Pattern pattern = this.handler.getFilePattern(this.name);
+
+    assertThat(pattern).isNotNull();
+    assertThat(pattern.matcher(this.name).matches()).isFalse();
+    assertThat(pattern.matcher(this.name + "-01-01").matches()).isTrue();
+    assertThat(pattern.matcher(this.name + "-01-02").matches()).isTrue();
+    assertThat(pattern.matcher(this.name + "-02-01").matches()).isTrue();
+    assertThat(pattern.matcher(this.name + "-01").matches()).isFalse();
+    assertThat(pattern.matcher(this.name + "0101").matches()).isFalse();
+    assertThat(pattern.matcher(this.name + "--").matches()).isFalse();
+
+    // TODO: revisit these to determine if behavior should change
+    assertThat(pattern.matcher(this.name + "-01-01-01").matches()).isFalse();
+    assertThat(pattern.matcher(this.name + ".01-01-01").matches()).isFalse();
+  }
+
+  @Test
+  public void getFilePattern_withNumbers_matchesFiles() throws Exception {
+    this.name = "a1s2d3f4_cache1_statistics";
+    Pattern pattern = this.handler.getFilePattern(this.name);
+
+    assertThat(pattern).isNotNull();
+    assertThat(pattern.matcher(this.name + "-01-41").matches()).isTrue();
+  }
+
+  @Test
+  public void getFilePattern_withHyphens_matchesFiles() throws Exception {
+    this.name = "a1s2d3f4_cache1-statistics";
+    Pattern pattern = this.handler.getFilePattern(this.name);
+
+    assertThat(pattern).isNotNull();
+    assertThat(pattern.matcher(this.name + "-01-41").matches()).isTrue();
+  }
+
+  @Test
+  public void getFilePattern_empty_throwsIllegalArgumentException() throws Exception {
+    this.name = "";
+
+    assertThatThrownBy(() -> this.handler.getFilePattern(this.name))
+        .isInstanceOf(IllegalArgumentException.class);
+  }
+
+  @Test
+  public void getFilePattern_null_throwsIllegalArgumentException() throws Exception {
+    this.name = null;
+
+    assertThatThrownBy(() -> this.handler.getFilePattern(this.name))
+        .isInstanceOf(IllegalArgumentException.class);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/geode/blob/7e85e08e/geode-core/src/test/java/org/apache/geode/internal/statistics/DiskSpaceLimitIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/statistics/DiskSpaceLimitIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/internal/statistics/DiskSpaceLimitIntegrationTest.java
index 1702d9a..5d94fa0 100644
--- a/geode-core/src/test/java/org/apache/geode/internal/statistics/DiskSpaceLimitIntegrationTest.java
+++ b/geode-core/src/test/java/org/apache/geode/internal/statistics/DiskSpaceLimitIntegrationTest.java
@@ -14,8 +14,10 @@
  */
 package org.apache.geode.internal.statistics;
 
+import static java.lang.String.valueOf;
 import static java.util.concurrent.TimeUnit.MINUTES;
-import static org.apache.commons.io.FileUtils.moveFileToDirectory;
+import static org.apache.commons.io.FileUtils.*;
+import static org.apache.commons.lang.StringUtils.leftPad;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
@@ -37,9 +39,14 @@ import org.junit.rules.TestName;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.TimeoutException;
 
 @Category(IntegrationTest.class)
+@SuppressWarnings("unused")
 public class DiskSpaceLimitIntegrationTest {
 
   private static final long FILE_SIZE_LIMIT = 256;
@@ -48,6 +55,8 @@ public class DiskSpaceLimitIntegrationTest {
   private File dir;
   private File dirOfDeletedFiles;
 
+  private String name;
+
   private String archiveFileName;
 
   private LocalStatisticsFactory factory;
@@ -55,9 +64,14 @@ public class DiskSpaceLimitIntegrationTest {
   private StatisticsType statisticsType;
   private Statistics statistics;
 
+  private RollingFileHandler testRollingFileHandler;
+  private MainWithChildrenRollingFileHandler mainWithChildrenRollingFileHandler;
+
   private SampleCollector sampleCollector;
   private StatArchiveHandlerConfig config;
 
+  private long initTimeStamp;
+
   private NanoTimer timer = new NanoTimer();
   private long nanosTimeStamp;
 
@@ -69,10 +83,10 @@ public class DiskSpaceLimitIntegrationTest {
   @Before
   public void setUp() throws Exception {
     this.dir = this.temporaryFolder.getRoot();
-    this.dirOfDeletedFiles = this.temporaryFolder.newFolder("deleted");
 
-    this.archiveFileName =
-        new File(this.dir, this.testName.getMethodName() + ".gfs").getAbsolutePath();
+    this.name = this.testName.getMethodName();
+
+    this.archiveFileName = new File(this.dir, this.name + ".gfs").getAbsolutePath();
 
     this.factory = new LocalStatisticsFactory(null);
     this.statisticDescriptors = new StatisticDescriptor[] {
@@ -93,11 +107,12 @@ public class DiskSpaceLimitIntegrationTest {
         .thenReturn(this.temporaryFolder.getRoot().getAbsolutePath());
     when(this.config.getProductDescription()).thenReturn(this.testName.getMethodName());
 
-    RollingFileHandler rollingFileHandler = new TestableRollingFileHandler();
+    this.testRollingFileHandler = new TestableRollingFileHandler();
+    this.mainWithChildrenRollingFileHandler = new MainWithChildrenRollingFileHandler();
 
     this.sampleCollector = new SampleCollector(sampler);
-    this.sampleCollector.initialize(this.config, NanoTimer.getTime(), rollingFileHandler);
 
+    this.initTimeStamp = NanoTimer.getTime();
     this.timer.reset();
     this.nanosTimeStamp = this.timer.getLastResetTime() - getNanoRate();
   }
@@ -109,6 +124,9 @@ public class DiskSpaceLimitIntegrationTest {
 
   @Test
   public void zeroKeepsAllFiles() throws Exception {
+    this.dirOfDeletedFiles = this.temporaryFolder.newFolder("deleted");
+    this.sampleCollector.initialize(this.config, this.initTimeStamp, this.testRollingFileHandler);
+
     when(this.config.getArchiveDiskSpaceLimit()).thenReturn(0L);
     sampleUntilFileExists(archiveFile(1));
     sampleUntilFileExists(archiveFile(2));
@@ -118,6 +136,9 @@ public class DiskSpaceLimitIntegrationTest {
 
   @Test
   public void aboveZeroDeletesOldestFile() throws Exception {
+    this.dirOfDeletedFiles = this.temporaryFolder.newFolder("deleted");
+    this.sampleCollector.initialize(this.config, this.initTimeStamp, this.testRollingFileHandler);
+
     when(this.config.getArchiveDiskSpaceLimit()).thenReturn(DISK_SPACE_LIMIT);
     sampleUntilFileExists(archiveFile(1));
     sampleUntilFileExists(archiveFile(2));
@@ -138,6 +159,118 @@ public class DiskSpaceLimitIntegrationTest {
     assertThat(everExisted(archiveFile(1))).isTrue();
   }
 
+  @Test
+  public void aboveZeroDeletesPreviousFiles() throws Exception {
+    assertThat(numberOfFiles(this.dir)).as("Unexpected files: " + listFiles(this.dir)).isEqualTo(0);
+
+    int oldMainId = 1;
+    int newMainId = 2;
+
+    int numberOfPreviousFiles = 100;
+    int numberOfLines = 100;
+    createPreviousFiles(oldMainId, numberOfPreviousFiles, numberOfLines);
+    assertThat(numberOfFiles(this.dir)).as("Missing files: " + listFiles(this.dir))
+        .isEqualTo(numberOfPreviousFiles);
+
+    for (int childId = 1; childId <= numberOfPreviousFiles; childId++) {
+      assertThat(archiveFile(oldMainId, childId)).exists();
+    }
+
+    // current archive file does not exist yet
+    assertThat(archiveFile()).doesNotExist();
+
+    // rolling files for mainId 2 do not exist yet
+    assertThat(markerFile(newMainId)).doesNotExist();
+    assertThat(archiveFile(newMainId, 1)).doesNotExist();
+
+    when(this.config.getArchiveDiskSpaceLimit())
+        .thenReturn(sizeOfDirectory(this.dir) / numberOfPreviousFiles);
+
+    this.sampleCollector.initialize(this.config, this.initTimeStamp,
+        this.mainWithChildrenRollingFileHandler);
+
+    assertThat(archiveFile()).exists().hasParent(this.dir);
+    assertThat(markerFile(newMainId)).exists().hasParent(this.dir).hasBinaryContent(new byte[0]);
+
+    assertThat(archiveFile(newMainId, 1)).doesNotExist();
+
+    sampleNumberOfTimes(1);
+
+    sampleUntilFileExists(archiveFile(newMainId, 1));
+    assertThat(archiveFile(newMainId, 1)).exists();
+
+    // this might be a brittle assertion... ok to delete if following for-block-assertion passes
+    assertThat(numberOfFiles(this.dir)).as("Unexpected files: " + listFiles(this.dir)).isEqualTo(2);
+
+    for (int childId = 1; childId <= numberOfPreviousFiles; childId++) {
+      assertThat(archiveFile(oldMainId, childId)).doesNotExist();
+    }
+  }
+
+  @Test
+  public void aboveZeroDeletesPreviousFiles_nameWithHyphen() throws Exception {
+    this.name = "psin8p724_cache1-statistics";
+    this.archiveFileName = new File(this.dir, this.name + ".gfs").getAbsolutePath();
+    when(this.config.getArchiveFileName()).thenReturn(new File(this.archiveFileName));
+
+    assertThat(numberOfFiles(this.dir)).as("Unexpected files: " + listFiles(this.dir)).isEqualTo(0);
+
+    int oldMainId = 1;
+    int newMainId = 2;
+
+    int numberOfPreviousFiles = 100;
+    int numberOfLines = 100;
+    createPreviousFiles(oldMainId, numberOfPreviousFiles, numberOfLines);
+    assertThat(numberOfFiles(this.dir)).as("Missing files: " + listFiles(this.dir))
+        .isEqualTo(numberOfPreviousFiles);
+
+    for (int childId = 1; childId <= numberOfPreviousFiles; childId++) {
+      assertThat(archiveFile(oldMainId, childId)).exists();
+    }
+
+    // current archive file does not exist yet
+    assertThat(archiveFile()).doesNotExist();
+
+    // rolling files for mainId 2 do not exist yet
+    assertThat(markerFile(newMainId)).doesNotExist();
+    assertThat(archiveFile(newMainId, 1)).doesNotExist();
+
+    when(this.config.getArchiveDiskSpaceLimit())
+        .thenReturn(sizeOfDirectory(this.dir) / numberOfPreviousFiles);
+
+    this.sampleCollector.initialize(this.config, this.initTimeStamp,
+        this.mainWithChildrenRollingFileHandler);
+
+    assertThat(archiveFile()).exists().hasParent(this.dir);
+    assertThat(markerFile(newMainId)).exists().hasParent(this.dir).hasBinaryContent(new byte[0]);
+
+    assertThat(archiveFile(newMainId, 1)).doesNotExist();
+
+    sampleNumberOfTimes(1);
+
+    sampleUntilFileExists(archiveFile(newMainId, 1));
+    assertThat(archiveFile(newMainId, 1)).exists();
+
+    // this might be a brittle assertion... ok to delete if following for-block-assertion passes
+    assertThat(numberOfFiles(this.dir)).as("Unexpected files: " + listFiles(this.dir)).isEqualTo(2);
+
+    for (int childId = 1; childId <= numberOfPreviousFiles; childId++) {
+      assertThat(archiveFile(oldMainId, childId)).doesNotExist();
+    }
+  }
+
+  private void sampleNumberOfTimes(final int value) throws InterruptedException {
+    long minutes = 1;
+    long timeout = System.nanoTime() + MINUTES.toNanos(minutes);
+    int count = 0;
+    do {
+      sample(advanceNanosTimeStamp());
+      count++;
+      Thread.sleep(10);
+    } while (count < value && System.nanoTime() < timeout);
+    System.out.println("Sampled " + count + " times.");
+  }
+
   private void sampleUntilFileExists(final File file)
       throws InterruptedException, TimeoutException {
     long minutes = 1;
@@ -203,9 +336,102 @@ public class DiskSpaceLimitIntegrationTest {
     return 1000; // 1 second
   }
 
-  private File archiveFile(final int child) {
-    return new File(this.dir,
-        this.testName.getMethodName() + "-01-" + String.format("%02d", child) + ".gfs");
+  private File archiveFile() {
+    return archiveFile(archiveName());
+  }
+
+  private String archiveName() {
+    return this.name;
+  }
+
+  private File archiveFile(final String name) {
+    return new File(this.dir, name + ".gfs");
+  }
+
+  private File archiveFile(final int childId) {
+    return archiveFile(1, childId);
+  }
+
+  private File archiveFile(final int mainId, final int childId) {
+    return archiveFile(archiveName(), mainId, childId);
+  }
+
+  private File archiveFile(final String name, final int mainId, final int childId) {
+    return new File(this.dir, archiveName(name, mainId, childId) + ".gfs");
+  }
+
+  private String archiveName(final int mainId, final int childId) {
+    return archiveName(archiveName(), mainId, childId);
+  }
+
+  private String archiveName(final String name, final int mainId, final int childId) {
+    return name + "-" + formatId(mainId) + "-" + formatId(childId);
+  }
+
+  private File markerFile(final int mainId) {
+    return markerFile(archiveName(), mainId);
+  }
+
+  private File markerFile(final String name, final int mainId) {
+    return new File(this.dir, markerName(name, mainId));
+  }
+
+  private String markerName(final int mainId) {
+    return markerName(archiveName(), mainId);
+  }
+
+  private String markerName(final String name, final int mainId) {
+    return archiveName(name, mainId, 0) + ".marker";
+  }
+
+  private String formatId(final int id) {
+    return String.format("%02d", id);
+  }
+
+  private List<File> listFiles(final File dir) {
+    return Arrays.asList(dir.listFiles());
+  }
+
+  private int numberOfFiles(final File dir) {
+    return dir.listFiles().length;
+  }
+
+  private void createPreviousFiles(final int mainId, final int fileCount, final int lineCount)
+      throws IOException {
+    createPreviousFiles(this.name, mainId, fileCount, lineCount);
+  }
+
+  private void createPreviousFiles(final String name, final int mainId, final int fileCount,
+      final int lineCount) throws IOException {
+    int childId = 1;
+    List<String> lines = lines(lineCount);
+    for (int i = 0; i < fileCount; i++) {
+      File file = createFile(name, mainId, childId);
+      writeFile(file, lines);
+      childId++;
+    }
+  }
+
+  private File createFile(final String name, final int mainId, final int childId) {
+    File file = new File(this.dir, name + "-" + leftPad(valueOf(mainId), 2, "0") + "-"
+        + leftPad(valueOf(childId), 2, "0") + ".gfs");
+    return file;
+  }
+
+  private void writeFile(final File file, final List<String> lines) throws IOException {
+    PrintWriter writer = new PrintWriter(file, "UTF-8");
+    for (String line : lines) {
+      writer.println(line);
+    }
+    writer.close();
+  }
+
+  private List<String> lines(final int lineCount) {
+    List<String> lines = new ArrayList<>();
+    for (int i = 0; i < lineCount; i++) {
+      lines.add(this.testName.getMethodName());
+    }
+    return lines;
   }
 
   /**
@@ -222,5 +448,4 @@ public class DiskSpaceLimitIntegrationTest {
       }
     }
   }
-
 }

http://git-wip-us.apache.org/repos/asf/geode/blob/7e85e08e/geode-core/src/test/java/org/apache/geode/internal/statistics/StatArchiveHandlerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/statistics/StatArchiveHandlerIntegrationTest.java b/geode-core/src/test/java/org/apache/geode/internal/statistics/StatArchiveHandlerIntegrationTest.java
new file mode 100644
index 0000000..ca5993c
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/internal/statistics/StatArchiveHandlerIntegrationTest.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.statistics;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.IOException;
+
+import junitparams.JUnitParamsRunner;
+import junitparams.Parameters;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+
+import org.apache.geode.internal.io.MainWithChildrenRollingFileHandler;
+import org.apache.geode.internal.io.RollingFileHandler;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+/**
+ * Tests behavior that interacts with file system.
+ */
+@Category(IntegrationTest.class)
+@RunWith(JUnitParamsRunner.class)
+public class StatArchiveHandlerIntegrationTest {
+
+  private File dir;
+  private String ext;
+  private String name;
+  private File archive;
+
+  private StatArchiveHandlerConfig mockConfig;
+  private SampleCollector mockCollector;
+  private RollingFileHandler rollingFileHandler;
+
+  @Rule
+  public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+  @Rule
+  public TestName testName = new TestName();
+
+  @Before
+  public void setUp() throws Exception {
+    this.dir = this.temporaryFolder.getRoot();
+
+    this.ext = ".gfs";
+    this.name = this.testName.getMethodName();
+    this.archive = new File(this.dir, this.name + this.ext);
+
+    this.mockConfig = mock(StatArchiveHandlerConfig.class);
+    this.mockCollector = mock(SampleCollector.class);
+    this.rollingFileHandler = new MainWithChildrenRollingFileHandler();
+  }
+
+  @Test
+  @Parameters({"false,false,false", "false,false,true", "false,true,false", "false,true,true",
+      "true,false,false", "true,false,true", "true,true,false", "true,true,true"})
+  public void getRollingArchiveName_withEmptyDir_createsFirstIds(final boolean archiveExists,
+      final boolean archiveClosed, final boolean initMainId) throws Exception {
+    if (archiveExists) {
+      this.archive.createNewFile();
+    }
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    if (initMainId) {
+      handler.initMainArchiveId(this.archive);
+    }
+
+    File file = handler.getRollingArchiveName(this.archive, archiveClosed);
+
+    assertThat(file).hasParent(this.dir).hasName(this.name + formatIds(1, 1) + this.ext);
+  }
+
+  @Test
+  @Parameters({"1,1,false,false", "1,1,false,true", "1,1,true,false", "1,1,true,true",
+      "1,10,false,false", "1,10,false,true", "1,10,true,false", "1,10,true,true",
+      "10,1,false,false", "10,1,false,true", "10,1,true,false", "10,1,true,true",
+      "10,10,false,false", "10,10,false,true", "10,10,true,false", "10,10,true,true"})
+  public void getRollingArchiveName_withOldArchives_rollsChildId(final int mainCount,
+      final int childCount, final boolean archiveExists, final boolean archiveClosed)
+      throws Exception {
+    createEmptyArchiveFiles(this.dir, this.name, this.ext, mainCount, childCount);
+    if (archiveExists) {
+      this.archive.createNewFile();
+    }
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+
+    File file = handler.getRollingArchiveName(this.archive, archiveClosed);
+
+    assertThat(file).hasParent(this.dir)
+        .hasName(this.name + formatIds(mainCount, childCount + 1) + this.ext);
+  }
+
+  @Test
+  public void initMainArchiveId_withEmptyDir_createsMainId_createsFirstMarker() throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+
+    handler.initMainArchiveId(this.archive);
+
+    assertThat(new File(this.dir, this.name + formatIds(1, 0) + ".marker")).exists();
+  }
+
+  @Test
+  @Parameters({"1,1", "1,10", "10,1", "10,10"})
+  public void initMainArchiveId_withOldArchives_rollsMainId_rollsMarker(final int mainCount,
+      final int childCount) throws Exception {
+    createEmptyArchiveFiles(this.dir, this.name, this.ext, mainCount, childCount);
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+
+    handler.initMainArchiveId(this.archive);
+
+    assertThat(new File(this.dir, this.name + formatIds(mainCount + 1, 0) + ".marker")).exists();
+  }
+
+  @Test
+  public void getRenameArchiveName_withEmptyDir_createsFirstIds() throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).isNotNull().isEqualTo(new File(this.dir, this.name + "-01-01" + this.ext));
+  }
+
+  @Test
+  public void getRenameArchiveName_withExtraneousIds_withEmptyDir_appendsIds() throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    this.archive = new File(this.dir, this.name + "-01-01" + this.ext);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).hasParent(this.dir).hasName(this.name + "-01-01-01-01" + this.ext);
+  }
+
+  @Test
+  public void getRenameArchiveName_withExtraneousDots_withEmptyDir_appendsIds() throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    this.archive = new File(this.dir, this.name + ".test.test" + this.ext);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).hasParent(this.dir).hasName(this.name + ".test.test-01-01" + this.ext);
+  }
+
+  @Test
+  public void getRenameArchiveName_withExtraneousUnderscores_withEptyDir_appendsIds()
+      throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    this.archive = new File(this.dir, this.name + "_test_test" + this.ext);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).hasParent(this.dir).hasName(this.name + "_test_test-01-01" + this.ext);
+  }
+
+  @Test
+  public void getRenameArchiveName_withExtraneousHyphens_withEmptyDir_appendsIds()
+      throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    this.archive = new File(this.dir, this.name + "-test-test" + this.ext);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).hasParent(this.dir).hasName(this.name + "-test-test-01-01" + this.ext);
+  }
+
+  @Test
+  @Parameters({"1,1", "1,10", "10,1", "10,10"})
+  public void getRenameArchiveName_withOldArchives_rollsMainId(final int mainCount,
+      final int childCount) throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    createEmptyArchiveFiles(this.dir, this.name, this.ext, mainCount, childCount);
+
+    File renamed = handler.getRenameArchiveName(this.archive);
+
+    assertThat(renamed).doesNotExist().hasParent(this.dir)
+        .hasName(this.name + formatIds(mainCount + 1, 1) + this.ext);
+  }
+
+  @Test
+  public void getRenameArchiveName_withNullArchive_throwsNullPointerException() throws Exception {
+    StatArchiveHandler handler =
+        new StatArchiveHandler(this.mockConfig, this.mockCollector, this.rollingFileHandler);
+    File archive = null;
+
+    assertThatThrownBy(() -> handler.getRenameArchiveName(archive))
+        .isInstanceOf(NullPointerException.class);
+  }
+
+  /**
+   * Returns mainId and childId formatted like "-01-01"
+   */
+  private String formatIds(final int mainId, final int childId) {
+    return "-" + formatId(mainId) + "-" + formatId(childId);
+  }
+
+  /**
+   * Returns id formatted like "01"
+   */
+  private String formatId(final int id) {
+    return String.format("%02d", id);
+  }
+
+  /**
+   * Creates empty archive files like dir/name-mainId-childId.ext. Returns greatest mainId.
+   */
+  private int createEmptyArchiveFiles(final File dir, final String name, final String ext,
+      final int mainCount, final int childCount) throws IOException {
+    int mainId = 1;
+    for (; mainId <= mainCount; mainId++) {
+      for (int childId = 1; childId <= childCount; childId++) {
+        File existing = new File(dir, name + formatIds(mainId, childId) + ext);
+        existing.createNewFile();
+      }
+    }
+    return mainId - 1;
+  }
+}


[2/4] geode git commit: GEODE-2497 surprise member not removed during startup

Posted by kl...@apache.org.
GEODE-2497 surprise member not removed during startup

addressing Galen's comments on the pull request


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/3474fa7a
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/3474fa7a
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/3474fa7a

Branch: refs/heads/feature/GEODE-2514
Commit: 3474fa7ab4bccbe7a08f70e251ac778eebd6c453
Parents: 135bd77
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Tue Feb 21 16:36:39 2017 -0800
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Tue Feb 21 16:36:39 2017 -0800

----------------------------------------------------------------------
 .../membership/gms/mgr/GMSMembershipManager.java      |  6 +++---
 .../internal/DistributionManagerDUnitTest.java        | 14 ++++++--------
 2 files changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/3474fa7a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
index 050e201..d0a0cbe 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/membership/gms/mgr/GMSMembershipManager.java
@@ -576,7 +576,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
       // expire surprise members, add others to view
       long oldestAllowed = System.currentTimeMillis() - this.surpriseMemberTimeout;
       for (Iterator<Map.Entry<InternalDistributedMember, Long>> it =
-           surpriseMembers.entrySet().iterator(); it.hasNext();) {
+          surpriseMembers.entrySet().iterator(); it.hasNext();) {
         Map.Entry<InternalDistributedMember, Long> entry = it.next();
         Long birthtime = entry.getValue();
         if (birthtime.longValue() < oldestAllowed) {
@@ -1150,7 +1150,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
 
   @Override
   public void memberSuspected(InternalDistributedMember initiator,
-                              InternalDistributedMember suspect, String reason) {
+      InternalDistributedMember suspect, String reason) {
     SuspectMember s = new SuspectMember(initiator, suspect, reason);
     handleOrDeferSuspect(s);
   }
@@ -1803,7 +1803,7 @@ public class GMSMembershipManager implements MembershipManager, Manager {
   }
 
   public Set<InternalDistributedMember> send(InternalDistributedMember[] destinations,
-                                             DistributionMessage msg, DMStats theStats) throws NotSerializableException {
+      DistributionMessage msg, DMStats theStats) throws NotSerializableException {
     Set<InternalDistributedMember> result;
     boolean allDestinations = msg.forAll();
 

http://git-wip-us.apache.org/repos/asf/geode/blob/3474fa7a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
index 5e3cf3e..08e3b87 100644
--- a/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/DistributionManagerDUnitTest.java
@@ -77,7 +77,8 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
   public static DistributedSystem ds;
 
   @Rule
-  public DistributedRestoreSystemProperties restoreSystemProperties = new DistributedRestoreSystemProperties();
+  public DistributedRestoreSystemProperties restoreSystemProperties =
+      new DistributedRestoreSystemProperties();
 
   /**
    * Clears the exceptionInThread flag in the given distribution manager.
@@ -174,8 +175,7 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
 
       // if the view number isn't being recorded correctly the test will pass but the
       // functionality is broken
-      Assert.assertTrue("expected view ID to be greater than zero",
-          mgr.getView().getViewId() > 0);
+      Assert.assertTrue("expected view ID to be greater than zero", mgr.getView().getViewId() > 0);
 
       int oldViewId = mbr.getVmViewId();
       mbr.setVmViewId((int) mgr.getView().getViewId() - 1);
@@ -200,11 +200,9 @@ public class DistributionManagerDUnitTest extends JUnit4DistributedTestCase {
       MembershipManagerHelper.addSurpriseMember(sys, mbr, birthTime);
       assertTrue("Member was not a surprise member", mgr.isSurpriseMember(mbr));
 
-      if (birthTime < (System.currentTimeMillis() - timeout)) {
-        return; // machine is too busy and we didn't get enough CPU to perform more assertions
-      }
-      assertTrue("Member was incorrectly removed from surprise member set",
-          mgr.isSurpriseMember(mbr));
+//      if (birthTime < (System.currentTimeMillis() - timeout)) {
+//        return; // machine is too busy and we didn't get enough CPU to perform more assertions
+//      }
 
       Awaitility.await("waiting for member to be removed")
           .atMost((timeout / 3) + gracePeriod, TimeUnit.MILLISECONDS)