You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ag...@apache.org on 2016/04/15 02:47:30 UTC

incubator-geode git commit: GEODE-1209: Added new attribute to forward eviction/expiration to AEQ. Tested with manual testing.

Repository: incubator-geode
Updated Branches:
  refs/heads/feature/GEODE-1209 [created] 87f8e4bd7


GEODE-1209: Added new attribute to forward eviction/expiration to AEQ. Tested with manual testing.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/87f8e4bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/87f8e4bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/87f8e4bd

Branch: refs/heads/feature/GEODE-1209
Commit: 87f8e4bd7e5a0d2e81db02d0d68733e967342817
Parents: 2622510
Author: Anil <ag...@pivotal.io>
Authored: Thu Apr 14 17:47:16 2016 -0700
Committer: Anil <ag...@pivotal.io>
Committed: Thu Apr 14 17:47:16 2016 -0700

----------------------------------------------------------------------
 .../cache/asyncqueue/AsyncEventQueue.java       |  9 ++++
 .../asyncqueue/AsyncEventQueueFactory.java      |  6 +++
 .../internal/AsyncEventQueueFactoryImpl.java    |  8 ++-
 .../internal/AsyncEventQueueImpl.java           | 12 ++++-
 .../gemfire/internal/cache/LocalRegion.java     | 48 +++++++++---------
 .../cache/wan/AbstractGatewaySender.java        | 51 ++++++++++++++++++--
 .../cache/wan/GatewaySenderAttributes.java      |  7 +++
 .../cache/xmlcache/AsyncEventQueueCreation.java |  7 +++
 8 files changed, 115 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueue.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueue.java b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueue.java
index a2b8b0f..d024a92 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueue.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueue.java
@@ -147,4 +147,13 @@ public interface AsyncEventQueue {
    *         <code>AsyncEventQueue</code>
    */
   public GatewayEventSubstitutionFilter getGatewayEventSubstitutionFilter();
+ 
+  /**
+   * Represents if eviction and expiration events/operations are passed to
+   * <code>AsyncEventListener</code>.
+   * 
+   * @return boolen True if eviction and expiration operations are passed.
+   */
+  public boolean isEvictionAndExpirationEventsAllowed();
+  
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueueFactory.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueueFactory.java b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueueFactory.java
index 3e30b38..6afd55c 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueueFactory.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/AsyncEventQueueFactory.java
@@ -170,7 +170,13 @@ public interface AsyncEventQueueFactory {
   public AsyncEventQueueFactory setGatewayEventSubstitutionListener(
       GatewayEventSubstitutionFilter filter);
 
+  /**
+   * Allows eviction and expiration events/operation to be propagated
+   * to <code>AsyncEventListener</code>. 
+   */
+  public AsyncEventQueueFactory forwardEvictionAndExpirationEvents();
 
+  
   /**
    * Creates the <code>AsyncEventQueue</code>. It accepts Id of AsyncEventQueue
    * and instance of AsyncEventListener. Multiple queues can be created using

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueFactoryImpl.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueFactoryImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueFactoryImpl.java
index d15222b..0f839e3 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueFactoryImpl.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueFactoryImpl.java
@@ -193,7 +193,7 @@ public class AsyncEventQueueFactoryImpl implements AsyncEventQueueFactory {
           //  AsyncEventQueueImpl.getSenderIdFromAsyncEventQueueId(asyncQueueId));
       addAsyncEventListener(listener);
       GatewaySender sender = create(AsyncEventQueueImpl.getSenderIdFromAsyncEventQueueId(asyncQueueId));
-      AsyncEventQueueImpl queue = new AsyncEventQueueImpl(sender, listener);
+      AsyncEventQueueImpl queue = new AsyncEventQueueImpl(sender, listener, this.attrs.isEvictionAndExpirationEventsAllowed());
       asyncEventQueue = queue;
       ((GemFireCacheImpl) cache).addAsyncEventQueue(queue);
     } else if (this.cache instanceof CacheCreation) {
@@ -297,4 +297,10 @@ public class AsyncEventQueueFactoryImpl implements AsyncEventQueueFactory {
     this.attrs.isMetaQueue = isMetaQueue;
     return this;
   }
+
+  @Override
+  public AsyncEventQueueFactory forwardEvictionAndExpirationEvents() {
+    this.attrs.forwardEvictionAndExpirationEvents = true;
+    return this;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueImpl.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueImpl.java b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueImpl.java
index 6b3eb4a..88436ad 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueImpl.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/cache/asyncqueue/internal/AsyncEventQueueImpl.java
@@ -36,12 +36,15 @@ public class AsyncEventQueueImpl implements AsyncEventQueue {
   private GatewaySender sender = null;
   
   private AsyncEventListener asyncEventListener = null;
+    
+  private final boolean allowEvictionAndExpirationEvents;
   
   public static final String ASYNC_EVENT_QUEUE_PREFIX = "AsyncEventQueue_";
   
-  public AsyncEventQueueImpl(GatewaySender sender, AsyncEventListener eventListener) {
+  public AsyncEventQueueImpl(GatewaySender sender, AsyncEventListener eventListener, boolean allowEvictionAndExpirationEvents) {
     this.sender = sender;
     this.asyncEventListener = eventListener;
+    this.allowEvictionAndExpirationEvents = allowEvictionAndExpirationEvents;
   }
  
   @Override
@@ -201,5 +204,10 @@ public class AsyncEventQueueImpl implements AsyncEventQueue {
     // TODO Auto-generated method stub
     return false;
   }
-  
+
+  @Override
+  public boolean isEvictionAndExpirationEventsAllowed() {
+    return this.allowEvictionAndExpirationEvents;
+  }
+     
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
index 3ff48bb..cfd385d 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
@@ -6724,7 +6724,8 @@ public class LocalRegion extends AbstractRegion
   protected void notifyGatewaySender(EnumListenerEvent operation,
       EntryEventImpl event) {
     
-    if (event.isConcurrencyConflict()) { // usually concurrent cache modification problem
+    if (this.isInternalRegion() || isPdxTypesRegion() || 
+        event.isConcurrencyConflict() /* usually concurrent cache modification problem */) { 
       return;
     }
     
@@ -6736,34 +6737,31 @@ public class LocalRegion extends AbstractRegion
       return;
     }
     
-    if (!event.getOperation().isLocal()) {
-      Set<String> allGatewaySenderIds = null;
-      checkSameSenderIdsAvailableOnAllNodes();
-      if (event.getOperation() == Operation.UPDATE_VERSION_STAMP) {
-        allGatewaySenderIds = getGatewaySenderIds();
-      } else {
-        allGatewaySenderIds = getAllGatewaySenderIds();
-      }
-      
-      List<Integer> allRemoteDSIds = getRemoteDsIds(allGatewaySenderIds);
+    
+    Set<String> allGatewaySenderIds = null;
+    checkSameSenderIdsAvailableOnAllNodes();
+    if (event.getOperation() == Operation.UPDATE_VERSION_STAMP) {
+      allGatewaySenderIds = getGatewaySenderIds();
+    } else {
+      allGatewaySenderIds = getAllGatewaySenderIds();
+    }
 
-      if (allRemoteDSIds != null) {
-        for (GatewaySender sender : getCache().getAllGatewaySenders()) {
-          if (!isPdxTypesRegion()) {
-            if (allGatewaySenderIds.contains(sender.getId())) {
-              //TODO: This is a BUG. Why return and not continue?
-              if((!this.getDataPolicy().withStorage()) && sender.isParallel()){
-                return;
-              }
-              if(logger.isDebugEnabled()) {
-                logger.debug("Notifying the GatewaySender : {}", sender.getId());
-              }
-              ((AbstractGatewaySender)sender).distribute(operation, event,
-                  allRemoteDSIds);
-            }
+    List<Integer> allRemoteDSIds = getRemoteDsIds(allGatewaySenderIds);
+    if (allRemoteDSIds != null) {
+      for (GatewaySender sender : getCache().getAllGatewaySenders()) {
+        if (allGatewaySenderIds.contains(sender.getId())) {
+          //TODO: This is a BUG. Why return and not continue?
+          if((!this.getDataPolicy().withStorage()) && sender.isParallel()){
+            return;
+          }
+          if(logger.isDebugEnabled()) {
+            logger.debug("Notifying the GatewaySender : {}", sender.getId());
           }
+          ((AbstractGatewaySender)sender).distribute(operation, event,
+              allRemoteDSIds);
         }
       }
+
       
 //      if (shouldNotifyGatewaySender()) {
 //        // Get All WAN site DSID's to be sent to each WAN site so that they

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/AbstractGatewaySender.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/AbstractGatewaySender.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/AbstractGatewaySender.java
index 4f3488b..5c2f959 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/AbstractGatewaySender.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/AbstractGatewaySender.java
@@ -34,6 +34,7 @@ import com.gemstone.gemfire.cache.AttributesFactory;
 import com.gemstone.gemfire.cache.Cache;
 import com.gemstone.gemfire.cache.CacheException;
 import com.gemstone.gemfire.cache.DataPolicy;
+import com.gemstone.gemfire.cache.Operation;
 import com.gemstone.gemfire.cache.Region;
 import com.gemstone.gemfire.cache.RegionAttributes;
 import com.gemstone.gemfire.cache.RegionDestroyedException;
@@ -136,6 +137,8 @@ public abstract class AbstractGatewaySender implements GatewaySender,
 
   protected List<AsyncEventListener> listeners;
   
+  protected boolean allowEvictionAndExpirationEvents;
+  
   protected GatewayEventSubstitutionFilter substitutionFilter;
   
   protected LocatorDiscoveryCallback locatorDiscoveryCallback;
@@ -272,6 +275,7 @@ public abstract class AbstractGatewaySender implements GatewaySender,
         initializeEventIdIndex();
     }
     this.isBucketSorted = attrs.isBucketSorted();
+    this.allowEvictionAndExpirationEvents = attrs.isEvictionAndExpirationEventsAllowed();
   }
   
   public void createSender(Cache cache, GatewaySenderAttributes attrs){
@@ -322,7 +326,7 @@ public abstract class AbstractGatewaySender implements GatewaySender,
     }
     this.isBucketSorted = attrs.isBucketSorted();
     this.isHDFSQueue = attrs.isHDFSQueue();
-   
+    this.allowEvictionAndExpirationEvents = attrs.isEvictionAndExpirationEventsAllowed();
   }
   
   public GatewaySenderAdvisor getSenderAdvisor() {
@@ -848,16 +852,50 @@ public abstract class AbstractGatewaySender implements GatewaySender,
     return this.eventProcessor;
   }
 
+  /**
+   * Check if this event can be distributed by senders.
+   * @param event
+   * @param stats
+   * @return boolean True if the event is allowed.
+   */
+  private boolean checkForDistribution(EntryEventImpl event, GatewaySenderStats stats) {
+    
+    if (event.getRegion().getDataPolicy().equals(DataPolicy.NORMAL))
+    {
+      return false;
+    }
+    
+    // Eviction and expirations are not passed to WAN.
+    // Eviction and Expiration are passed to AEQ based on its configuration.
+    if (event.getOperation().isLocal() || event.getOperation().isExpiration()) {
+      // Check if its AEQ and AEQ is configured to forward eviction/expiration events.
+      if (this.isAsyncEventQueue() && this.allowEvictionAndExpirationEvents) {
+        return true;
+      }
+      return false;
+    }
+    
+    return true;
+  }
+  
+  
   public void distribute(EnumListenerEvent operation, EntryEventImpl event,
       List<Integer> allRemoteDSIds) {
+    
     final boolean isDebugEnabled = logger.isDebugEnabled();
     
+    // If this gateway is not running, return
+    if (!isRunning()) {
+      if (isDebugEnabled) {
+        logger.debug("Returning back without putting into the gateway sender queue");
+      }
+      return;
+    }
+    
     final GatewaySenderStats stats = getStatistics();
     stats.incEventsReceived();
-    // If the event is local (see bug 35831) or an expiration ignore it.
-    //removed the check of isLocal as in notifyGAtewayHub this has been taken care
-    if (/*event.getOperation().isLocal() || */event.getOperation().isExpiration()
-        || event.getRegion().getDataPolicy().equals(DataPolicy.NORMAL)) {
+   
+    if (!checkForDistribution(event, stats)) {
       getStatistics().incEventsNotQueued();
       return;
     }
@@ -868,6 +906,7 @@ public abstract class AbstractGatewaySender implements GatewaySender,
       stats.incEventsNotQueued();
       return;
     }
+    
     // this filter is defined by Asif which exist in old wan too. new wan has
     // other GatewaEventFilter. Do we need to get rid of this filter. Cheetah is
     // not cinsidering this filter
@@ -956,6 +995,7 @@ public abstract class AbstractGatewaySender implements GatewaySender,
     }
     try {
       // If this gateway is not running, return
+      // The sender may have stopped, after we have checked the status in the beginning. 
       if (!isRunning()) {
         if (isDebugEnabled) {
           logger.debug("Returning back without putting into the gateway sender queue");
@@ -1003,6 +1043,7 @@ public abstract class AbstractGatewaySender implements GatewaySender,
     }
   }
   
+
   /**
    * During sender is getting started, if there are any cache operation on queue then that event will be stored in temp queue. 
    * Once sender is started, these event from tmp queue will be added to sender queue.

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderAttributes.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderAttributes.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderAttributes.java
index 025616d..b07eb79 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderAttributes.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/wan/GatewaySenderAttributes.java
@@ -85,6 +85,8 @@ public class GatewaySenderAttributes {
   public boolean isHDFSQueue = GatewaySenderAttributes.DEFAULT_IS_HDFSQUEUE;
   public boolean isMetaQueue = GatewaySenderAttributes.DEFAULT_IS_META_QUEUE;
   
+  public boolean forwardEvictionAndExpirationEvents = false;
+  
   public int getSocketBufferSize() {
     return this.socketBufferSize;
   }
@@ -197,4 +199,9 @@ public class GatewaySenderAttributes {
   public boolean isMetaQueue() {
     return this.isMetaQueue;
   }
+  
+  public boolean isEvictionAndExpirationEventsAllowed() {
+    return this.forwardEvictionAndExpirationEvents;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/87f8e4bd/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/xmlcache/AsyncEventQueueCreation.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/xmlcache/AsyncEventQueueCreation.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/xmlcache/AsyncEventQueueCreation.java
index 77f9596..9c027ac 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/xmlcache/AsyncEventQueueCreation.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/xmlcache/AsyncEventQueueCreation.java
@@ -44,6 +44,7 @@ public class AsyncEventQueueCreation implements AsyncEventQueue {
   private boolean isHDFSQueue = false;
   private int dispatcherThreads = 1;
   private OrderPolicy orderPolicy = OrderPolicy.KEY;
+  private boolean allowEvictionAndExpirationEvents = false;
   
   public AsyncEventQueueCreation() {
   }
@@ -64,6 +65,7 @@ public class AsyncEventQueueCreation implements AsyncEventQueue {
     this.isBucketSorted = senderAttrs.isBucketSorted; 
     this.isHDFSQueue = senderAttrs.isHDFSQueue;
     this.gatewayEventSubstitutionFilter = senderAttrs.eventSubstitutionFilter;
+    this.allowEvictionAndExpirationEvents = senderAttrs.forwardEvictionAndExpirationEvents;
   }
   
   @Override
@@ -220,4 +222,9 @@ public class AsyncEventQueueCreation implements AsyncEventQueue {
   public void setIsHDFSQueue(boolean isHDFSQueue) {
     this.isHDFSQueue = isHDFSQueue;
   }
+
+  @Override
+  public boolean isEvictionAndExpirationEventsAllowed() {
+    return this.allowEvictionAndExpirationEvents;
+  }
 }