You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2015/10/06 19:16:35 UTC

incubator-geode git commit: GEODE-362: fix intermittent failures in TXExpiryJUnitTest

Repository: incubator-geode
Updated Branches:
  refs/heads/develop 55cd9246f -> 1673e2461


GEODE-362: fix intermittent failures in TXExpiryJUnitTest

The TXExpiryJUnitTest now uses a test hook so it can be notified
when an ExpiryTask has expired. Note that when a tx is in progress
ExpiryTasks expire but do not perform their expiration action.
When the tx completes the expiration is rescheduled.
TxExpiryJUnitTest now also configures millisecond expiration units
so it can run faster. And the code has been cleaned up to remove
all eclipse warnings.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/1673e246
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/1673e246
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/1673e246

Branch: refs/heads/develop
Commit: 1673e2461524f97b93b413d4d1b60486c92b5752
Parents: 55cd924
Author: Darrel Schneider <ds...@pivotal.io>
Authored: Mon Oct 5 16:12:35 2015 -0700
Committer: Darrel Schneider <ds...@pivotal.io>
Committed: Tue Oct 6 09:50:53 2015 -0700

----------------------------------------------------------------------
 .../gemfire/internal/cache/ExpiryTask.java      |  20 +-
 .../gemfire/internal/cache/LocalRegion.java     |   7 +
 .../com/gemstone/gemfire/TXExpiryJUnitTest.java | 510 +++++++++----------
 3 files changed, 270 insertions(+), 267 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1673e246/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExpiryTask.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExpiryTask.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExpiryTask.java
index d5dc5ee..5c428b2 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExpiryTask.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/ExpiryTask.java
@@ -222,7 +222,6 @@ public abstract class ExpiryTask extends SystemTimer.SystemTimerTask {
   
   protected final boolean expire(boolean isPending) throws CacheException 
   {
-    waitOnExpirationSuspension();
     ExpirationAction action = getAction();
     if (action == null) return false;
     return expire(action, isPending);
@@ -351,6 +350,7 @@ public abstract class ExpiryTask extends SystemTimer.SystemTimerTask {
           getLocalRegion().isDestroyed()) {
         return;
       }
+      waitOnExpirationSuspension();
       if (logger.isTraceEnabled()) {
         logger.trace("{} is fired", this);
       }
@@ -396,6 +396,10 @@ public abstract class ExpiryTask extends SystemTimer.SystemTimerTask {
        // is still usable:
        SystemFailure.checkFailure();
        logger.fatal(LocalizedMessage.create(LocalizedStrings.ExpiryTask_EXCEPTION_IN_EXPIRATION_TASK), ex);
+    } finally {
+      if (expiryTaskListener != null) {
+        expiryTaskListener.afterExpire(this);
+      }
     }
   }
 
@@ -508,4 +512,18 @@ public abstract class ExpiryTask extends SystemTimer.SystemTimerTask {
     return result;
   }
 
+  // Should only be set by unit tests
+  public static ExpiryTaskListener expiryTaskListener;
+  
+  /**
+   * Used by tests to determine if events related
+   * to an ExpiryTask have happened.
+   */
+  public interface ExpiryTaskListener {
+    /**
+     * Called after the given expiry task has expired.
+     */
+    public void afterExpire(ExpiryTask et);
+    
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1673e246/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
index 617a7ec..7c1ec89 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/cache/LocalRegion.java
@@ -8822,6 +8822,13 @@ public class LocalRegion extends AbstractRegion
   public RegionIdleExpiryTask getRegionIdleExpiryTask() {
     return this.regionIdleExpiryTask;
   }
+  /**
+   * Used by unit tests to get access to the RegionTTLExpiryTask
+   * of this region. Returns null if no task exists.
+   */
+  public RegionTTLExpiryTask getRegionTTLExpiryTask() {
+    return this.regionTTLExpiryTask;
+  }
   
   private void addExpiryTask(RegionEntry re, boolean ifAbsent)
   {

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/1673e246/gemfire-core/src/test/java/com/gemstone/gemfire/TXExpiryJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/TXExpiryJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/TXExpiryJUnitTest.java
index da9623b..f8dffcc 100644
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/TXExpiryJUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/TXExpiryJUnitTest.java
@@ -7,18 +7,13 @@
  */
 package com.gemstone.gemfire;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
-import com.gemstone.gemfire.cache.*;
-import com.gemstone.gemfire.cache.util.*;
-import com.gemstone.gemfire.internal.OSProcess;
-import com.gemstone.gemfire.internal.cache.*;
-import com.gemstone.gemfire.internal.util.StopWatch;
-import com.gemstone.gemfire.distributed.*;
-import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
-import com.gemstone.gemfire.test.junit.categories.IntegrationTest;
-
-import java.util.*;
+import java.util.Properties;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.junit.After;
@@ -26,6 +21,32 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.gemstone.gemfire.cache.AttributesMutator;
+import com.gemstone.gemfire.cache.Cache;
+import com.gemstone.gemfire.cache.CacheException;
+import com.gemstone.gemfire.cache.CacheFactory;
+import com.gemstone.gemfire.cache.CacheListener;
+import com.gemstone.gemfire.cache.CacheTransactionManager;
+import com.gemstone.gemfire.cache.CommitConflictException;
+import com.gemstone.gemfire.cache.EntryEvent;
+import com.gemstone.gemfire.cache.ExpirationAction;
+import com.gemstone.gemfire.cache.ExpirationAttributes;
+import com.gemstone.gemfire.cache.Region;
+import com.gemstone.gemfire.cache.RegionEvent;
+import com.gemstone.gemfire.cache.RegionFactory;
+import com.gemstone.gemfire.cache.Scope;
+import com.gemstone.gemfire.cache.util.CacheListenerAdapter;
+import com.gemstone.gemfire.internal.cache.ExpiryTask;
+import com.gemstone.gemfire.internal.cache.ExpiryTask.ExpiryTaskListener;
+import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
+import com.gemstone.gemfire.internal.cache.LocalRegion;
+import com.gemstone.gemfire.internal.cache.TXManagerImpl;
+import com.gemstone.gemfire.internal.cache.TXStateProxy;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+import dunit.DistributedTestCase;
+import dunit.DistributedTestCase.WaitCriterion;
+
 /**
  * Tests transaction expiration functionality
  *
@@ -33,8 +54,7 @@ import org.junit.experimental.categories.Category;
  * @since 4.0
  *
  */
-@SuppressWarnings("deprecation")
-@Category(IntegrationTest.class)
+@Category(UnitTest.class)
 public class TXExpiryJUnitTest {
 
   protected GemFireCacheImpl cache;
@@ -43,9 +63,7 @@ public class TXExpiryJUnitTest {
   protected void createCache() throws CacheException {
     Properties p = new Properties();
     p.setProperty("mcast-port", "0"); // loner
-    this.cache = (GemFireCacheImpl)CacheFactory.create(DistributedSystem.connect(p));
-    AttributesFactory af = new AttributesFactory();
-    af.setScope(Scope.DISTRIBUTED_NO_ACK);
+    this.cache = (GemFireCacheImpl) (new CacheFactory(p)).create();
     this.txMgr = this.cache.getCacheTransactionManager();
   }
   private void closeCache() {
@@ -56,7 +74,6 @@ public class TXExpiryJUnitTest {
         } catch (IllegalStateException ignore) {
         }
       }
-      // this.region = null;
       this.txMgr = null;
       Cache c = this.cache;
       this.cache = null;
@@ -72,319 +89,280 @@ public class TXExpiryJUnitTest {
   @After
   public void tearDown() throws Exception {
     closeCache();
-    InternalDistributedSystem ids = InternalDistributedSystem.getAnyInstance();
-    if (ids != null) {
-      ids.disconnect();
-    }
   }
 
   @Test
   public void testEntryTTLExpiration() throws CacheException {
-    AttributesFactory af = new AttributesFactory();
-    af.setScope(Scope.DISTRIBUTED_NO_ACK);
-    af.setStatisticsEnabled(true);
-    af.setEntryTimeToLive(new ExpirationAttributes(2, ExpirationAction.DESTROY));
-    Region exprReg = this.cache.createRegion("TXEntryTTL", af.create());
-    generalEntryExpirationTest(exprReg, 2);
-    AttributesMutator mutator = exprReg.getAttributesMutator();
-    mutator.setEntryTimeToLive(new ExpirationAttributes(1, ExpirationAction.DESTROY));
-    generalEntryExpirationTest(exprReg, 1);
+    generalEntryExpirationTest(createRegion("TXEntryTTL"), new ExpirationAttributes(1, ExpirationAction.DESTROY), true);
   } 
 
   @Test
   public void testEntryIdleExpiration() throws CacheException {
-    AttributesFactory af = new AttributesFactory();
-    af.setScope(Scope.DISTRIBUTED_NO_ACK);
-    af.setStatisticsEnabled(true);
-    af.setEntryIdleTimeout(new ExpirationAttributes(2, ExpirationAction.DESTROY));
-    Region exprReg = this.cache.createRegion("TXEntryIdle", af.create());
-//    exprReg.getCache().getLogger().info("invoking expiration test with 2");
-    generalEntryExpirationTest(exprReg, 2);
-    AttributesMutator mutator = exprReg.getAttributesMutator();
-    mutator.setEntryIdleTimeout(new ExpirationAttributes(1, ExpirationAction.DESTROY));
-//    exprReg.getCache().getLogger().info("invoking expiration test with 1 after setting idle timeout of 1 second");
-    generalEntryExpirationTest(exprReg, 1);
+    generalEntryExpirationTest(createRegion("TXEntryIdle"), new ExpirationAttributes(1, ExpirationAction.DESTROY), false);
   } 
   
-  private void waitDance(boolean list[], int waitMs) {
-    synchronized(list) {
-      if (!list[0]) {
-        try {
-          list.wait(waitMs);
-        }
-        catch (InterruptedException e) {
-          fail("Interrupted");
-        }
-        if (list[0]) {
-          fail("Cache listener detected a destroy... bad!");
-        }
-      } else {
-        fail("Cache listener detected a destroy oh man that is bad!");
-      }
-    }  
+  private Region<String, String> createRegion(String name) {
+    RegionFactory<String, String> rf = this.cache.createRegionFactory();
+    rf.setScope(Scope.DISTRIBUTED_NO_ACK);
+    rf.setStatisticsEnabled(true);
+    System.setProperty(LocalRegion.EXPIRY_MS_PROPERTY, "true");
+    try {
+      return rf.create(name);
+    } 
+    finally {
+      System.getProperties().remove(LocalRegion.EXPIRY_MS_PROPERTY);
+    }
   }
   
-  @SuppressWarnings("deprecation")
-  public void generalEntryExpirationTest(final Region exprReg, 
-                                         final int exprTime) 
+  public void generalEntryExpirationTest(final Region<String, String> exprReg, 
+                                         ExpirationAttributes exprAtt,
+                                         boolean useTTL) 
     throws CacheException 
   {
-    final int waitMs = exprTime * 1500;
-    final int patientWaitMs = exprTime * 90000;
+    final LocalRegion lr = (LocalRegion) exprReg;
     final boolean wasDestroyed[] = {false};
-    AttributesMutator mutator = exprReg.getAttributesMutator();
+    AttributesMutator<String, String> mutator = exprReg.getAttributesMutator();
     final AtomicInteger ac = new AtomicInteger();
     final AtomicInteger au = new AtomicInteger();
     final AtomicInteger ai = new AtomicInteger();
     final AtomicInteger ad = new AtomicInteger();
     
-//    exprReg.getCache().getLogger().info("generalEntryExpirationTest invoked with exprTime " + exprTime);
-
-    mutator.setCacheListener(new CacheListenerAdapter() {
-        public void close() {}
-        public void afterCreate(EntryEvent e) {
-//          e.getRegion().getCache().getLogger().info("invoked afterCreate for " + e);
-          ac.incrementAndGet();
-        }
-        public void afterUpdate(EntryEvent e) {
-//          e.getRegion().getCache().getLogger().info("invoked afterUpdate for " + e);
-          au.incrementAndGet();
-        }
-        public void afterInvalidate(EntryEvent e) {
-//          e.getRegion().getCache().getLogger().info("invoked afterInvalidate for " + e);
-          ai.incrementAndGet();
-        }
-        public void afterDestroy(EntryEvent e) {
-//          e.getRegion().getCache().getLogger().info("invoked afterDestroy for " + e);
-          ad.incrementAndGet();
-          if (e.getKey().equals("key0")) {
-            synchronized(wasDestroyed) {
-              wasDestroyed[0] = true;
-              wasDestroyed.notifyAll();
-            }
-          }
-        }
-        public void afterRegionInvalidate(RegionEvent event) {
-          fail("Unexpected invokation of afterRegionInvalidate");
-        }
-        public void afterRegionDestroy(RegionEvent event) {
-          if (!event.getOperation().isClose()) {
-            fail("Unexpected invokation of afterRegionDestroy");
-          }
-        }
-      });
-    
-    // Test to ensure an expriation does not cause a conflict
-    for(int i=0; i<2; i++) {
-      exprReg.put("key" + i, "value" + i);
-    }
-    try {  Thread.sleep(500); } catch (InterruptedException ie) {fail("interrupted");}
-    this.txMgr.begin();
-//    exprReg.getCache().getLogger().info("transactional update of key0");
-    exprReg.put("key0", "value");
-//    exprReg.getCache().getLogger().info("waiting for " + waitMs);
-    waitDance(wasDestroyed, waitMs);
-    assertEquals("value", exprReg.getEntry("key0").getValue());
-//    exprReg.getCache().getLogger().info("committing transaction");
-    try {
-      this.txMgr.commit();
-    } catch (CommitConflictException error) {
-      fail("Expiration should not cause commit to fail");
+    if (useTTL) {
+      mutator.setEntryTimeToLive(exprAtt);
+    } else {
+      mutator.setEntryIdleTimeout(exprAtt);
     }
-    assertEquals("value", exprReg.getEntry("key0").getValue());
-    try {
-      synchronized(wasDestroyed) {
-        if (!wasDestroyed[0]) {
-//          exprReg.getCache().getLogger().info("waiting for wasDestroyed to be set by listener");
-          long start = System.currentTimeMillis();
-          wasDestroyed.wait(patientWaitMs);
-          long took = System.currentTimeMillis()-start;
-          if (!wasDestroyed[0]) {
-//            exprReg.getCache().getLogger().info("wasDestroyed was never set by the listener");
-            OSProcess.printStacks(0, false);
-            fail("Cache listener did not detect a destroy in " + patientWaitMs + " ms! actuallyWaited "+took+"ms ac="+ac.get()+" au:"+au.get()+" ai:"+ai.get()+" ad:"+ad.get());
+    final CacheListener<String, String> cl = new CacheListenerAdapter<String, String>() {
+      public void afterCreate(EntryEvent<String, String> e) {
+        ac.incrementAndGet();
+      }
+      public void afterUpdate(EntryEvent<String, String> e) {
+        au.incrementAndGet();
+      }
+      public void afterInvalidate(EntryEvent<String, String> e) {
+        ai.incrementAndGet();
+      }
+      public void afterDestroy(EntryEvent<String, String> e) {
+        ad.incrementAndGet();
+        if (e.getKey().equals("key0")) {
+          synchronized(wasDestroyed) {
+            wasDestroyed[0] = true;
+            wasDestroyed.notifyAll();
           }
         }
       }
-    } catch (InterruptedException ie) {
-      fail("Caught InterruptedException while waiting for eviction");
-    }
-    assertTrue(!exprReg.containsKey("key0"));
-    // key1 is the canary for the rest of the entries
-    assertTrue(!exprReg.containsKey("key1"));
-
-    // rollback and failed commit test, ensure expiration continues
-    for(int j=0; j<2; j++) {
-      synchronized(wasDestroyed) {
-        wasDestroyed[0] = false;
+      public void afterRegionInvalidate(RegionEvent<String, String> event) {
+        fail("Unexpected invocation of afterRegionInvalidate");
       }
+      public void afterRegionDestroy(RegionEvent<String, String> event) {
+        if (!event.getOperation().isClose()) {
+          fail("Unexpected invocation of afterRegionDestroy");
+        }
+      }
+    };
+    mutator.addCacheListener(cl);
+    try {
+
+      ExpiryTask.suspendExpiration();
+      // Test to ensure an expiration does not cause a conflict
       for(int i=0; i<2; i++) {
         exprReg.put("key" + i, "value" + i);
       }
-      try {  Thread.sleep(500); } catch (InterruptedException ie) {fail("interrupted");}
       this.txMgr.begin();
       exprReg.put("key0", "value");
-      waitDance(wasDestroyed, waitMs);
+      waitForEntryExpiration(lr, "key0");
       assertEquals("value", exprReg.getEntry("key0").getValue());
-      String checkVal;
-      if (j==0) {
-        checkVal = "value0";
-        this.txMgr.rollback();
-      } else {
-        checkVal = "conflictVal";
-        final TXManagerImpl txMgrImpl = (TXManagerImpl)this.txMgr;
-        TXStateProxy tx = txMgrImpl.internalSuspend();
-        exprReg.put("key0", checkVal);
-        txMgrImpl.resume(tx);
-        try {
-          this.txMgr.commit();
-          fail("Expected CommitConflictException!");
-        } catch (CommitConflictException expected) {}
-      }
-//      exprReg.getCache().getLogger().info("waiting for listener to be invoked.  iteration = " + j);
       try {
-        synchronized(wasDestroyed) {
-          if (!wasDestroyed[0]) {
-            Object value = exprReg.get("key0");
-            if (value == null) { // destroy in progress?
-              wasDestroyed.wait(waitMs);
-              assertTrue(wasDestroyed[0]);
-            }
-            else {
-              assertEquals(checkVal, value);
-              long start = System.currentTimeMillis();
-              wasDestroyed.wait(patientWaitMs);
-              long took = System.currentTimeMillis()-start;
-              if (!wasDestroyed[0]) {
-                Map m = new HashMap(exprReg);
-                fail("Cache listener did not detect a destroy in " + patientWaitMs + " ms! actuallyWaited:"+took+"ms ac="+ac.get()+" au:"+au.get()+" ai:"+ai.get()+" ad:"+ad.get()+" j="+j+" region="+m);
-              }
-            }
-          } 
-        }
-      } catch (InterruptedException ie) {
-        fail("Caught InterruptedException while waiting for expiration");
+        ExpiryTask.suspendExpiration();
+        this.txMgr.commit();
+      } catch (CommitConflictException error) {
+        fail("Expiration should not cause commit to fail");
+      }
+      assertEquals("value", exprReg.getEntry("key0").getValue());
+      waitForEntryExpiration(lr, "key0");
+      synchronized(wasDestroyed) {
+        assertEquals(true, wasDestroyed[0]);
       }
       assertTrue(!exprReg.containsKey("key0"));
       // key1 is the canary for the rest of the entries
       assertTrue(!exprReg.containsKey("key1"));
+
+      // rollback and failed commit test, ensure expiration continues
+      for(int j=0; j<2; j++) {
+        synchronized(wasDestroyed) {
+          wasDestroyed[0] = false;
+        }
+        ExpiryTask.suspendExpiration();
+        for(int i=0; i<2; i++) {
+          exprReg.put("key" + i, "value" + i);
+        }
+        this.txMgr.begin();
+        exprReg.put("key0", "value");
+        waitForEntryExpiration(lr, "key0");
+        assertEquals("value", exprReg.getEntry("key0").getValue());
+        String checkVal;
+        ExpiryTask.suspendExpiration();
+        if (j==0) {
+          checkVal = "value0";
+          this.txMgr.rollback();
+        } else {
+          checkVal = "conflictVal";
+          final TXManagerImpl txMgrImpl = (TXManagerImpl)this.txMgr;
+          TXStateProxy tx = txMgrImpl.internalSuspend();
+          exprReg.put("key0", checkVal);
+          txMgrImpl.resume(tx);
+          try {
+            this.txMgr.commit();
+            fail("Expected CommitConflictException!");
+          } catch (CommitConflictException expected) {}
+        }
+        waitForEntryExpiration(lr, "key0");
+        synchronized(wasDestroyed) {
+          assertEquals(true, wasDestroyed[0]);
+        }
+        assertTrue(!exprReg.containsKey("key0"));
+        // key1 is the canary for the rest of the entries
+        assertTrue(!exprReg.containsKey("key1"));
+      }
+    } finally {
+      mutator.removeCacheListener(cl);
+      ExpiryTask.permitExpiration();
+    }
+  }
+  
+  private void waitForEntryExpiration(LocalRegion lr, String key) {
+    ExpirationDetector detector = new ExpirationDetector(lr.getEntryExpiryTask(key));
+    ExpiryTask.expiryTaskListener = detector;
+    try {
+      ExpiryTask.permitExpiration();
+      DistributedTestCase.waitForCriterion(detector, 3000, 2, true);
+    } finally {
+      ExpiryTask.expiryTaskListener = null;
+    }
+  }
+  private void waitForRegionExpiration(LocalRegion lr, boolean ttl) {
+    ExpirationDetector detector = new ExpirationDetector(ttl ? lr.getRegionTTLExpiryTask() : lr.getRegionIdleExpiryTask());
+    ExpiryTask.expiryTaskListener = detector;
+    try {
+      ExpiryTask.permitExpiration();
+      DistributedTestCase.waitForCriterion(detector, 3000, 2, true);
+    } finally {
+      ExpiryTask.expiryTaskListener = null;
+    }
+  }
+
+
+  /**
+   * Used to detect that a particular ExpiryTask has expired.
+   */
+  public static class ExpirationDetector implements ExpiryTaskListener, WaitCriterion {
+    private volatile boolean expired = false;
+    private final ExpiryTask et;
+    public ExpirationDetector(ExpiryTask et) {
+      assertNotNull(et);
+      this.et = et;
+    }
+    @Override
+    public void afterExpire(ExpiryTask et) {
+      if (et == this.et) {
+        this.expired = true;
+      }
+    }
+    @Override
+    public boolean done() {
+      return this.expired;
+    }
+    @Override
+    public String description() {
+      return "the expiry task " + this.et + " did not expire";
     }
   }
 
   @Test
   public void testRegionIdleExpiration() throws CacheException {
-    AttributesFactory af = new AttributesFactory();
-    af.setScope(Scope.DISTRIBUTED_NO_ACK);
-    af.setStatisticsEnabled(true);
-    af.setRegionIdleTimeout(new ExpirationAttributes(2, ExpirationAction.INVALIDATE));
-    Region exprReg = this.cache.createRegion("TXRegionIdle", af.create());
-    generalRegionExpirationTest(exprReg, 2, null, false);
-    generalRegionExpirationTest(exprReg, 1, new ExpirationAttributes(1, ExpirationAction.INVALIDATE), false);
-    generalRegionExpirationTest(exprReg, 2, new ExpirationAttributes(2, ExpirationAction.INVALIDATE), false);
-    generalRegionExpirationTest(exprReg, 1, new ExpirationAttributes(1, ExpirationAction.DESTROY), false);
+    Region<String, String> exprReg = createRegion("TXRegionIdle");
+    generalRegionExpirationTest(exprReg, new ExpirationAttributes(1, ExpirationAction.INVALIDATE), false);
+    generalRegionExpirationTest(exprReg, new ExpirationAttributes(1, ExpirationAction.DESTROY), false);
   } 
 
   @Test
   public void testRegionTTLExpiration() throws CacheException {
-    AttributesFactory af = new AttributesFactory();
-    af.setScope(Scope.DISTRIBUTED_NO_ACK);
-    af.setStatisticsEnabled(true);
-    af.setRegionTimeToLive(new ExpirationAttributes(1, ExpirationAction.INVALIDATE));
-    Region exprReg = this.cache.createRegion("TXRegionTTL", af.create());
-    generalRegionExpirationTest(exprReg, 1, null, true);
-    generalRegionExpirationTest(exprReg, 2, new ExpirationAttributes(2, ExpirationAction.INVALIDATE), true);
-    generalRegionExpirationTest(exprReg, 1, new ExpirationAttributes(1, ExpirationAction.INVALIDATE), true);
-    generalRegionExpirationTest(exprReg, 1, new ExpirationAttributes(1, ExpirationAction.DESTROY), true);
+    Region<String, String> exprReg = createRegion("TXRegionTTL");
+    generalRegionExpirationTest(exprReg, new ExpirationAttributes(1, ExpirationAction.INVALIDATE), true);
+    generalRegionExpirationTest(exprReg, new ExpirationAttributes(1, ExpirationAction.DESTROY), true);
   } 
   
-  private void generalRegionExpirationTest(final Region exprReg, 
-                                          final int exprTime, 
+  private void generalRegionExpirationTest(final Region<String, String> exprReg, 
                                           ExpirationAttributes exprAtt,
                                           boolean useTTL) 
     throws CacheException 
   {
-    final int waitMs = exprTime * 1500;
-    final int patientWaitMs = exprTime * 90000;
+    final LocalRegion lr = (LocalRegion) exprReg;
+    final ExpirationAction action = exprAtt.getAction();
     final boolean regionExpiry[] = {false};
-    AttributesMutator mutator = exprReg.getAttributesMutator();
-    mutator.setCacheListener(new CacheListenerAdapter() {
-        public void close() {}
-        public void afterCreate(EntryEvent e) {}
-        public void afterUpdate(EntryEvent e) {}
-        public void afterInvalidate(EntryEvent e) {}
-        public void afterDestroy(EntryEvent e) {}
-        public void afterRegionInvalidate(RegionEvent event) {
+    AttributesMutator<String, String> mutator = exprReg.getAttributesMutator();
+    final CacheListener<String, String> cl = new CacheListenerAdapter<String, String>() {
+      public void afterRegionInvalidate(RegionEvent<String, String> event) {
+        synchronized(regionExpiry) {
+          regionExpiry[0] = true;
+          regionExpiry.notifyAll();
+        }
+      }
+      public void afterRegionDestroy(RegionEvent<String, String> event) {
+        if (!event.getOperation().isClose()) {
           synchronized(regionExpiry) {
             regionExpiry[0] = true;
             regionExpiry.notifyAll();
           }
         }
-        public void afterRegionDestroy(RegionEvent event) {
-          if (!event.getOperation().isClose()) {
-            synchronized(regionExpiry) {
-              regionExpiry[0] = true;
-              regionExpiry.notifyAll();
-            }
-          }
-        }
-      });
-
-    // Create some keys and age them, I wish we could fake/force the age
-    // instead of having to actually wait
-    for(int i=0; i<2; i++) {
-      exprReg.put("key" + i, "value" + i);
-    }
-
-    ExpirationAction action;
-    if (exprAtt!=null) {
-      action = exprAtt.getAction();
-      if (useTTL) {
-        mutator.setRegionTimeToLive(exprAtt);
-      } else {
-        mutator.setRegionIdleTimeout(exprAtt);
       }
+    };
+    mutator.addCacheListener(cl);
+    if (useTTL) {
+      mutator.setRegionTimeToLive(exprAtt);
     } else {
-      if (useTTL) {
-        action = exprReg.getAttributes().getRegionTimeToLive().getAction();
-      } else {
-        action = exprReg.getAttributes().getRegionIdleTimeout().getAction();
-      }
+      mutator.setRegionIdleTimeout(exprAtt);
     }
 
-    // Potential race condition at this point if the Region operation
-    // is destroy i.e. we may not get to the transaction block
-    // before the destroy timer fires.
-
-    String regName = exprReg.getName();
-    // Test to ensure a region expriation does not cause a conflict    
-    this.txMgr.begin();
-    exprReg.put("key0", "value");
-    waitDance(regionExpiry, waitMs);
-    assertEquals("value", exprReg.getEntry("key0").getValue());
-    try {
-      this.txMgr.commit();
-    } catch (CommitConflictException error) {
-      fail("Expiration should not cause commit to fail");
-    }
     try {
+      ExpiryTask.suspendExpiration();
+
+      // Create some keys and age them, I wish we could fake/force the age
+      // instead of having to actually wait
+      for(int i=0; i<2; i++) {
+        exprReg.put("key" + i, "value" + i);
+      }
+
+      String regName = exprReg.getName();
+      // Test to ensure a region expiration does not cause a conflict    
+      this.txMgr.begin();
+      exprReg.put("key0", "value");
+      waitForRegionExpiration(lr, useTTL);
+      assertEquals("value", exprReg.getEntry("key0").getValue());
+      try {
+        ExpiryTask.suspendExpiration();
+        this.txMgr.commit();
+      } catch (CommitConflictException error) {
+        fail("Expiration should not cause commit to fail");
+      }
+      assertEquals("value", exprReg.getEntry("key0").getValue());
+      waitForRegionExpiration(lr, useTTL);
       synchronized(regionExpiry) {
-        if (!regionExpiry[0]) {
-          assertEquals("value", exprReg.getEntry("key0").getValue());
-          regionExpiry.wait(patientWaitMs);
-          if (!regionExpiry[0]) {
-            fail("Cache listener did not detect a region expiration in " + patientWaitMs + " ms!");
-          }
-        }
+        assertEquals(true, regionExpiry[0]);
       }
-    } catch (InterruptedException ie) {
-      fail("Caught InterruptedException while waiting for eviction");
-    }
-    if (action == ExpirationAction.DESTROY) {
-      assertNull("listener saw Region expiration, expected a destroy operation!", 
-                 this.cache.getRegion(regName));
-    } else {
-      assertTrue("listener saw Region expriation, expected invalidation", 
-                 !exprReg.containsValueForKey("key0"));
+      if (action == ExpirationAction.DESTROY) {
+        assertNull("listener saw Region expiration, expected a destroy operation!", 
+            this.cache.getRegion(regName));
+      } else {
+        assertTrue("listener saw Region expiration, expected invalidation", 
+            !exprReg.containsValueForKey("key0"));
+      }
+
+    } finally {
+      if (!exprReg.isDestroyed()) {
+        mutator.removeCacheListener(cl);
+      }
+      ExpiryTask.permitExpiration();
     }
 
     // @todo mitch test rollback and failed expiration