You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bo...@apache.org on 2018/06/29 19:43:43 UTC

[geode] 05/05: GEM-1772: Added additional debugging

This is an automated email from the ASF dual-hosted git repository.

boglesby pushed a commit to branch feature/GEM-1772
in repository https://gitbox.apache.org/repos/asf/geode.git

commit 7c17f1c97136267a77ea1f7660b5f5658439dee4
Author: Barry Oglesby <bo...@pivotal.io>
AuthorDate: Fri Jun 29 12:42:01 2018 -0700

    GEM-1772: Added additional debugging
---
 .../distributed/internal/DistributionAdvisor.java  | 40 ++++++++++++++++++++--
 .../internal/cache/AbstractUpdateOperation.java    |  5 +--
 .../apache/geode/internal/cache/BucketRegion.java  |  2 +-
 .../internal/cache/DistributedCacheOperation.java  | 19 ++++++++--
 .../apache/geode/internal/cache/LocalRegion.java   |  2 +-
 .../geode/internal/cache/PartitionedRegion.java    |  2 +-
 .../geode/internal/cache/StateFlushOperation.java  |  5 +--
 .../org/apache/geode/internal/cache/TXState.java   |  2 +-
 .../internal/cache/tx/RemotePutAllMessage.java     |  2 +-
 9 files changed, 66 insertions(+), 13 deletions(-)

diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/DistributionAdvisor.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/DistributionAdvisor.java
index 5ce42ed..ce9ed7c 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/DistributionAdvisor.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/DistributionAdvisor.java
@@ -14,6 +14,8 @@
  */
 package org.apache.geode.distributed.internal;
 
+import static org.apache.geode.internal.cache.DistributedCacheOperation.DISTRIBUTED_PUT_ALL_IN_PROGRESS;
+
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
@@ -40,6 +42,7 @@ import org.apache.geode.internal.InternalDataSerializer;
 import org.apache.geode.internal.Version;
 import org.apache.geode.internal.cache.CacheDistributionAdvisor.CacheProfile;
 import org.apache.geode.internal.cache.DistributedRegion;
+import org.apache.geode.internal.cache.StateFlushOperation;
 import org.apache.geode.internal.cache.UpdateAttributesProcessor;
 import org.apache.geode.internal.cache.persistence.PersistentMemberID;
 import org.apache.geode.internal.cache.versions.VersionSource;
@@ -626,6 +629,13 @@ public class DistributionAdvisor {
           synchronized (this.opCountLock) {
             previousVersionOpCount += currentVersionOpCount;
             currentVersionOpCount = 0;
+            logger.warn(
+                "XXX DistributionAdvisor doPutProfile advisee="
+                    + this.advisee.getFullPath() + "; membershipVersion=" + membershipVersion
+                    + "; previousVersionOpCount=" + previousVersionOpCount
+                    + "; currentVersionOpCount="
+                    + currentVersionOpCount,
+                new Exception());
           }
         }
       }
@@ -730,6 +740,12 @@ public class DistributionAdvisor {
       synchronized (this.opCountLock) {
         previousVersionOpCount += currentVersionOpCount;
         currentVersionOpCount = 0;
+        logger.warn(
+            "XXX DistributionAdvisor forceNewMembershipVersion advisee="
+                + this.advisee.getFullPath() + "; membershipVersion=" + membershipVersion
+                + "; previousVersionOpCount=" + previousVersionOpCount + "; currentVersionOpCount="
+                + currentVersionOpCount,
+            new Exception());
         if (logger.isTraceEnabled(LogMarker.DISTRIBUTION_STATE_FLUSH_VERBOSE)) {
           logger.trace(LogMarker.DISTRIBUTION_STATE_FLUSH_VERBOSE,
               "advisor for {} forced new membership version to {} previousOpCount={}", getAdvisee(),
@@ -759,6 +775,12 @@ public class DistributionAdvisor {
         logger.trace(LogMarker.STATE_FLUSH_OP_VERBOSE, "StateFlush current opcount incremented: {}",
             currentVersionOpCount);
       }
+      if (DISTRIBUTED_PUT_ALL_IN_PROGRESS.get()) {
+        logger.warn(
+            "XXX DistributionAdvisor startOperation advisee=" + this.advisee.getFullPath()
+                + "; membershipVersion=" + membershipVersion + "; currentVersionOpCount="
+                + currentVersionOpCount);
+      }
     }
     return membershipVersion;
   }
@@ -776,13 +798,25 @@ public class DistributionAdvisor {
         currentVersionOpCount--;
         if (logger.isTraceEnabled(LogMarker.STATE_FLUSH_OP_VERBOSE)) {
           logger.trace(LogMarker.STATE_FLUSH_OP_VERBOSE,
-              "StateFlush current opcount deccremented: {}", currentVersionOpCount);
+              "StateFlush current opcount decremented: {}", currentVersionOpCount);
+        }
+        if (DISTRIBUTED_PUT_ALL_IN_PROGRESS.get()) {
+          logger.warn(
+              "XXX DistributionAdvisor endOperation advisee=" + this.advisee.getFullPath()
+                  + "; version=" + version + "; membershipVersion=" + membershipVersion
+                  + "; currentVersionOpCount=" + currentVersionOpCount);
         }
       } else {
         previousVersionOpCount--;
         if (logger.isTraceEnabled(LogMarker.STATE_FLUSH_OP_VERBOSE)) {
           logger.trace(LogMarker.STATE_FLUSH_OP_VERBOSE,
-              "StateFlush previous opcount incremented: {}", previousVersionOpCount);
+              "StateFlush previous opcount decremented: {}", previousVersionOpCount);
+        }
+        if (DISTRIBUTED_PUT_ALL_IN_PROGRESS.get()) {
+          logger.warn(
+              "XXX DistributionAdvisor endOperation advisee=" + this.advisee.getFullPath()
+                  + "; version=" + version + "; membershipVersion=" + membershipVersion
+                  + "; previousVersionOpCount=" + previousVersionOpCount);
         }
       }
     }
@@ -818,6 +852,8 @@ public class DistributionAdvisor {
       synchronized (this.opCountLock) {
         opCount = this.previousVersionOpCount;
       }
+      StateFlushOperation
+          .log("DistributionAdvisor.waitForCurrentOperations waiting opCount=" + opCount);
       if (opCount <= 0) {
         break;
       }
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractUpdateOperation.java b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractUpdateOperation.java
index 6ec4453..0da39a1 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractUpdateOperation.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractUpdateOperation.java
@@ -42,7 +42,7 @@ import org.apache.geode.internal.i18n.LocalizedStrings;
 import org.apache.geode.internal.logging.LogService;
 
 /**
- * Common code for both UpdateOperation and DistributedPutAllOperation.
+ * Common code for both UpdateOperation and DISTRIBUTED_PUT_ALL_OPERATION.
  *
  */
 public abstract class AbstractUpdateOperation extends DistributedCacheOperation {
@@ -262,7 +262,8 @@ public abstract class AbstractUpdateOperation extends DistributedCacheOperation
     // @todo darrel: make this method static?
     /**
      * Do the actual update after operationOnRegion has confirmed work needs to be done Note this is
-     * the default implementation used by UpdateOperation. DistributedPutAllOperation overrides and
+     * the default implementation used by UpdateOperation. DISTRIBUTED_PUT_ALL_OPERATION overrides
+     * and
      * then calls back using super to this implementation. NOTE: be careful to not use methods like
      * getEvent(); defer to the ev passed as a parameter instead.
      */
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/BucketRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/BucketRegion.java
index 0fb745d..caeb95c 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/BucketRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/BucketRegion.java
@@ -1844,7 +1844,7 @@ public class BucketRegion extends DistributedRegion implements Bucket {
    * create a PutAllPRMessage for notify-only and send it to all adjunct nodes. return a set of
    * members that should be attached to the operation's reply processor (if any)
    *
-   * @param dpao DistributedPutAllOperation object for PutAllMessage
+   * @param dpao DISTRIBUTED_PUT_ALL_OPERATION object for PutAllMessage
    * @param cacheOpRecipients set of receiver which got cacheUpdateOperation.
    * @param adjunctRecipients recipients that must unconditionally get the event
    * @param filterRoutingInfo routing information for all members having the region
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/DistributedCacheOperation.java b/geode-core/src/main/java/org/apache/geode/internal/cache/DistributedCacheOperation.java
index 86ad5e0..cd8c41e 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/DistributedCacheOperation.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/DistributedCacheOperation.java
@@ -254,7 +254,13 @@ public abstract class DistributedCacheOperation {
     DistributedRegion region = getRegion();
     long viewVersion = -1;
     if (this.containsRegionContentChange()) {
+      if (this instanceof DistributedPutAllOperation) {
+        DISTRIBUTED_PUT_ALL_IN_PROGRESS.set(true);
+        logger.warn(
+            "XXX DistributedCacheOperation startOperation about to start operation msg=" + this);
+      }
       viewVersion = region.getDistributionAdvisor().startOperation();
+      DISTRIBUTED_PUT_ALL_IN_PROGRESS.set(false);
     }
     if (logger.isTraceEnabled(LogMarker.STATE_FLUSH_OP_VERBOSE)) {
       logger.trace(LogMarker.STATE_FLUSH_OP_VERBOSE, "dispatching operation in view version {}",
@@ -276,6 +282,9 @@ public abstract class DistributedCacheOperation {
     return viewVersion;
   }
 
+  public static ThreadLocal<Boolean> DISTRIBUTED_PUT_ALL_IN_PROGRESS =
+      ThreadLocal.withInitial(() -> Boolean.FALSE);
+
   /**
    * region's distribution advisor marked that a distribution is ended. This method should pair with
    * startOperation in try/finally block.
@@ -283,7 +292,13 @@ public abstract class DistributedCacheOperation {
   public void endOperation(long viewVersion) {
     DistributedRegion region = getRegion();
     if (viewVersion != -1) {
+      if (this instanceof DistributedPutAllOperation) {
+        DISTRIBUTED_PUT_ALL_IN_PROGRESS.set(true);
+      }
       region.getDistributionAdvisor().endOperation(viewVersion);
+      logger.warn(
+          "XXX DistributedCacheOperation endOperation operation msg=" + this);
+      DISTRIBUTED_PUT_ALL_IN_PROGRESS.set(false);
       if (logger.isTraceEnabled()) {
         logger.trace(LogMarker.STATE_FLUSH_OP_VERBOSE,
             "done dispatching operation in view version {}", viewVersion);
@@ -455,7 +470,7 @@ public abstract class DistributedCacheOperation {
         if (this instanceof DistributedPutAllOperation) {
           // if (logger.isDebugEnabled()) {
           logger.warn(
-              "XXX DistributedPutAllOperation _distribute recipients for {}: {} with adjunct messages to: {}",
+              "XXX DISTRIBUTED_PUT_ALL_OPERATION _distribute recipients for {}: {} with adjunct messages to: {}",
               this, recipients,
               adjunctRecipients);
           // }
@@ -610,7 +625,7 @@ public abstract class DistributedCacheOperation {
 
         if (failures != null && !failures.isEmpty()) {
           logger.warn(
-              "XXX DistributedPutAllOperation _distribute Failed sending ({}) to {} while processing event:{}",
+              "XXX DISTRIBUTED_PUT_ALL_OPERATION _distribute Failed sending ({}) to {} while processing event:{}",
               msg, failures, event);
         }
 
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
index 1642a8a..61aabd1 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
@@ -10061,7 +10061,7 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
    *
    * @param key the cache key
    * @param value the cache value
-   * @param putallOp the DistributedPutAllOperation associated with the event
+   * @param putallOp the DISTRIBUTED_PUT_ALL_OPERATION associated with the event
    * @param tagHolder holder for version tag
    * @throws TimeoutException if the operation times out
    * @throws CacheWriterException if a cache writer objects to the update
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegion.java
index 99aa3f0..59e762c 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegion.java
@@ -2211,7 +2211,7 @@ public class PartitionedRegion extends LocalRegion
   /**
    * Create PutAllPRMsgs for each bucket, and send them.
    *
-   * @param putAllOp DistributedPutAllOperation object.
+   * @param putAllOp DISTRIBUTED_PUT_ALL_OPERATION object.
    * @param successfulPuts not used in PartitionedRegion.
    */
   @Override
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/StateFlushOperation.java b/geode-core/src/main/java/org/apache/geode/internal/cache/StateFlushOperation.java
index 11e2e2d..5877617 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/StateFlushOperation.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/StateFlushOperation.java
@@ -98,7 +98,7 @@ public class StateFlushOperation {
 
   private DistributionManager dm;
 
-  private static void log(String message) {
+  public static void log(String message) {
     // System.out.println(Thread.currentThread().getName() + ": " + message);
     logger.warn("XXX " + message);
   }
@@ -506,7 +506,8 @@ public class StateFlushOperation {
     @Override
     public String toString() {
       return "StateMarkerMessage(requestingMember=" + this.getSender() + ",processorId="
-          + processorId + ",target=" + relayRecipient + ",region=" + regionPath + ")";
+          + processorId + ",target=" + relayRecipient + ",region=" + regionPath + ",flushNewOps="
+          + flushNewOps + ")";
     }
 
     @Override
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/TXState.java b/geode-core/src/main/java/org/apache/geode/internal/cache/TXState.java
index 9768fb8..ef1a6f2 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/TXState.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/TXState.java
@@ -1928,7 +1928,7 @@ public class TXState implements TXStateInterface {
    *
    * @see
    * org.apache.geode.internal.cache.InternalDataView#postPutAll(org.apache.geode.internal.cache.
-   * DistributedPutAllOperation, java.util.Map, org.apache.geode.internal.cache.LocalRegion)
+   * DISTRIBUTED_PUT_ALL_OPERATION, java.util.Map, org.apache.geode.internal.cache.LocalRegion)
    */
   @Override
   public void postPutAll(final DistributedPutAllOperation putallOp,
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/tx/RemotePutAllMessage.java b/geode-core/src/main/java/org/apache/geode/internal/cache/tx/RemotePutAllMessage.java
index f4a68b0..09f759a 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/tx/RemotePutAllMessage.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/tx/RemotePutAllMessage.java
@@ -68,7 +68,7 @@ import org.apache.geode.internal.offheap.annotations.Released;
 /**
  * A Replicate Region putAll message. Meant to be sent only to the peer who hosts transactional
  * data. It is also used to implement non-transactional putAlls, see:
- * DistributedPutAllOperation.initMessage
+ * DISTRIBUTED_PUT_ALL_OPERATION.initMessage
  *
  * @since GemFire 6.5
  */