[20/50] [abbrv] incubator-geode git commit: GEODE-108: Fix up HAInterestPart*DUnitTests

[bs...@apache.org]

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/a336e81c/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/tier/sockets/HAInterestPart2DUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/tier/sockets/HAInterestPart2DUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/tier/sockets/HAInterestPart2DUnitTest.java
index 8bad0b2..2ea7f7c 100755
--- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/tier/sockets/HAInterestPart2DUnitTest.java
+++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/cache/tier/sockets/HAInterestPart2DUnitTest.java
@@ -14,20 +14,19 @@ import com.gemstone.gemfire.cache.client.ServerConnectivityException;
 import dunit.DistributedTestCase;
 import dunit.VM;
 
-public class HAInterestPart2DUnitTest extends HAInterestBaseTest
-{
-  
-  /** constructor */
+@SuppressWarnings({"rawtypes", "serial"})
+public class HAInterestPart2DUnitTest extends HAInterestBaseTest {
+
   public HAInterestPart2DUnitTest(String name) {
     super(name);
   }
 
   /**
-   * Tests if Primary fails during interest un registration should initiate failover
-   * should pick new primary
+   * Tests if Primary fails during interest un registration should initiate
+   * failover should pick new primary
    */
-  public void testPrimaryFailureInUNregisterInterest()throws Exception{
-    createClientPoolCache(this.getName(),getServerHostName(server1.getHost()));
+  public void testPrimaryFailureInUNregisterInterest() throws Exception {
+    createClientPoolCache(this.getName(), getServerHostName(server1.getHost()));
     createEntriesK1andK2();
     server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
     server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
@@ -38,44 +37,44 @@ public class HAInterestPart2DUnitTest extends HAInterestBaseTest
     VM oldPrimary = getPrimaryVM();
     stopPrimaryAndUnregisterRegisterK1();
 
-    verifyDeadAndLiveServers(1,2);
+    verifyDeadAndLiveServers(1, 2);
 
     VM newPrimary = getPrimaryVM(oldPrimary);
     newPrimary.invoke(HAInterestBaseTest.class, "verifyDispatcherIsAlive");
-    //primary
+    // primary
     newPrimary.invoke(HAInterestBaseTest.class, "verifyInterestUNRegistration");
-    //secondary
+    // secondary
     getBackupVM().invoke(HAInterestBaseTest.class, "verifyInterestUNRegistration");
   }
 
   /**
-   * Tests if Secondary fails during interest un registration should add to dead Ep list
+   * Tests if Secondary fails during interest un registration should add to dead
+   * Ep list
    */
-  public void testSecondaryFailureInUNRegisterInterest()throws Exception{
-
-    createClientPoolCache(this.getName(),getServerHostName(server1.getHost()));
+  public void testSecondaryFailureInUNRegisterInterest() throws Exception {
+    createClientPoolCache(this.getName(), getServerHostName(server1.getHost()));
     createEntriesK1andK2();
     server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
     server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
     server3.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
     registerK1AndK2();
     VM stoppedBackup = stopSecondaryAndUNregisterK1();
-    verifyDeadAndLiveServers(1,2);
-    //still primary
+    verifyDeadAndLiveServers(1, 2);
+    // still primary
     getPrimaryVM().invoke(HAInterestBaseTest.class, "verifyDispatcherIsAlive");
-    //primary
+    // primary
     getPrimaryVM().invoke(HAInterestBaseTest.class, "verifyInterestUNRegistration");
-    //secondary
+    // secondary
     getBackupVM(stoppedBackup).invoke(HAInterestBaseTest.class, "verifyInterestUNRegistration");
   }
 
   /**
-   * Tests a scenario in which Dead Server Monitor detects Server Live Just before interest registration
-   * then interst should be registerd on the newly detcted live server as well
-   * @throws Exception
+   * Tests a scenario in which Dead Server Monitor detects Server Live Just
+   * before interest registration then interest should be registered on the newly
+   * detected live server as well
    */
-  public void testDSMDetectsServerLiveJustBeforeInterestRegistration()throws Exception{
-    createClientPoolCache(this.getName(),getServerHostName(server1.getHost()));
+  public void testDSMDetectsServerLiveJustBeforeInterestRegistration() throws Exception {
+    createClientPoolCache(this.getName(), getServerHostName(server1.getHost()));
     createEntriesK1andK2();
     server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
     server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
@@ -93,17 +92,15 @@ public class HAInterestPart2DUnitTest extends HAInterestBaseTest
     server1.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
     server2.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
     server3.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
-
   }
+
   /**
-   * Tests a scenario in which Dead Server Monitor detects Server Live Just After interest registration
-   * then interst should be registerd on the newly detcted live server as well
-   *
-   * @throws Exception
+   * Tests a scenario in which Dead Server Monitor detects Server Live Just
+   * After interest registration then interest should be registered on the newly
+   * detected live server as well
    */
-  public void testDSMDetectsServerLiveJustAfterInterestRegistration()throws Exception{
-
-    createClientPoolCache(this.getName(),getServerHostName(server1.getHost()));
+  public void testDSMDetectsServerLiveJustAfterInterestRegistration() throws Exception {
+    createClientPoolCache(this.getName(), getServerHostName(server1.getHost()));
 
     createEntriesK1andK2();
     server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
@@ -112,7 +109,7 @@ public class HAInterestPart2DUnitTest extends HAInterestBaseTest
 
     VM backup = getBackupVM();
     backup.invoke(HAInterestBaseTest.class, "stopServer");
-    verifyDeadAndLiveServers(1,2);
+    verifyDeadAndLiveServers(1, 2);
 
     setBridgeObserverForAfterRegistration(backup);
     try {
@@ -125,239 +122,228 @@ public class HAInterestPart2DUnitTest extends HAInterestBaseTest
     server1.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
     server2.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
     server3.invoke(HAInterestBaseTest.class, "verifyInterestRegistration");
+  }
 
+  /**
+   * Tests a Scenario: Only one server, register interest on the server stop
+   * server , and update the registered entries on the server start the server ,
+   * DSM will recover interest list on this live server and verify that as a
+   * part of recovery it refreshes registered entries from the server, because it
+   * is primary
+   */
+  public void testRefreshEntriesFromPrimaryWhenDSMDetectsServerLive() throws Exception {
+    addExpectedException(ServerConnectivityException.class.getName());
+    
+    PORT1 = ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
+    server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    createClientPoolCacheConnectionToSingleServer(this.getName(), getServerHostName(server1.getHost()));
+    registerK1AndK2();
+    verifyRefreshedEntriesFromServer();
+
+    server1.invoke(HAInterestBaseTest.class, "stopServer");
+    verifyDeadAndLiveServers(1, 0);
+    server1.invoke(HAInterestBaseTest.class, "putK1andK2");
+    server1.invoke(HAInterestBaseTest.class, "startServer");
+    verifyDeadAndLiveServers(0, 1);
+    final Region r1 = cache.getRegion(Region.SEPARATOR + REGION_NAME);
+    assertNotNull(r1);
+
+    WaitCriterion wc = new WaitCriterion() {
+      private String excuse;
+
+      @Override
+      public boolean done() {
+        Region.Entry e1;
+        Region.Entry e2;
+
+        try {
+          e1 = r1.getEntry(k1);
+          if (e1 == null) {
+            excuse = "Entry for k1 is null";
+            return false;
+          }
+        } catch (EntryDestroyedException e) {
+          excuse = "Entry destroyed";
+          return false;
+        }
+        if (!server_k1.equals(e1.getValue())) {
+          excuse = "e1 value is not server_k1";
+          return false;
+        }
+        try {
+          e2 = r1.getEntry(k2);
+          if (e2 == null) {
+            excuse = "Entry for k2 is null";
+            return false;
+          }
+        } catch (EntryDestroyedException e) {
+          excuse = "Entry destroyed";
+          return false;
+        }
+        if (!server_k2.equals(e2.getValue())) {
+          excuse = "e2 value is not server_k2";
+          return false;
+        }
+        return true;
+      }
+
+      @Override
+      public String description() {
+        return excuse;
+      }
+    };
+    DistributedTestCase.waitForCriterion(wc, TIMEOUT_MILLIS, INTERVAL_MILLIS, true);
   }
-   /**
-    * Tests a Scenario:
-    * Only one server, register interst on the server
-    * stop server , and update the registered entries on the server
-    * start the server , DSM will recover interest list on this live server
-    * and verify that as a part of recovery it refreshes
-    * registerd entries from the server, because it is primary
-    *
-    * @throws Exception
-    */
- public void testRefreshEntriesFromPrimaryWhenDSMDetectsServerLive()throws Exception{
-
- PORT1 =  ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
- server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
- createClientPoolCacheConnectionToSingleServer(this.getName(),getServerHostName(server1.getHost()));
- registerK1AndK2();
- verifyRefreshedEntriesFromServer();
-    // expected ServerConnectivityException
-    final ExpectedException expectedEx = addExpectedException(
-        ServerConnectivityException.class.getName());
- server1.invoke(HAInterestBaseTest.class, "stopServer");
- verifyDeadAndLiveServers(1, 0);
- server1.invoke(HAInterestBaseTest.class, "putK1andK2");
- server1.invoke(HAInterestBaseTest.class, "startServer");
- verifyDeadAndLiveServers(0, 1);
- final Region r1 = cache.getRegion(Region.SEPARATOR+ REGION_NAME);
- assertNotNull(r1);   
- 
- WaitCriterion wc = new WaitCriterion() {
-   String excuse;
-   public boolean done() {
-     Region.Entry e1;
-     Region.Entry e2;
-
-     try {
-       e1 = r1.getEntry(k1);
-       if (e1 == null) {
-         excuse = "Entry for k1 is null";
-         return false;
-       }
-     }
-     catch (EntryDestroyedException e) {
-       excuse = "Entry destroyed";
-       return false;
-     }
-     if (!server_k1.equals(e1.getValue())) {
-       excuse = "e1 value is not server_k1";
-       return false;
-     }
-     try {
-       e2 = r1.getEntry(k2);
-       if (e2 == null) {
-         excuse = "Entry for k2 is null";
-         return false;
-       }
-     }
-     catch (EntryDestroyedException e) {
-       excuse = "Entry destroyed";
-       return false;
-     }
-     if (!server_k2.equals(e2.getValue())) {
-       excuse = "e2 value is not server_k2";
-       return false;
-     }
-     return true;
-   }
-   public String description() {
-     return excuse;
-   }
- };
- DistributedTestCase.waitForCriterion(wc, 120 * 1000, 1000, true);
-
- expectedEx.remove();
-
- //assertEquals(server_k2, r1.getEntry(k2).getValue());
- //assertEquals(server_k1 ,r1.getEntry(k1).getValue());
-}
 
- /**
-    * Tests a Scenario:
-    * stop a secondary server and update the registered entries on the stopped server
-    * start the server , DSM will recover interest list on this live server
-    * and verify that as a part of recovery it does not refreshes
-    * registerd entries from the server, because it is secondary
-    *
-    * @throws Exception
-    */
- public void testGIIFromSecondaryWhenDSMDetectsServerLive()throws Exception{
-
-   server1.invoke(HAInterestBaseTest.class, "closeCache");
-   server2.invoke(HAInterestBaseTest.class, "closeCache");
-   server3.invoke(HAInterestBaseTest.class, "closeCache");
-
-   PORT1 =  ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
-   PORT2 =  ((Integer) server2.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
-   PORT3 =  ((Integer) server3.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
-
-   server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-   server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-   server3.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-
-   createClientPoolCache(this.getName(),getServerHostName(server1.getHost()));
-
-   VM backup1 = getBackupVM();
-   VM backup2 = getBackupVM(backup1);
-   backup1.invoke(HAInterestBaseTest.class, "stopServer");
-   backup2.invoke(HAInterestBaseTest.class, "stopServer");
-   verifyDeadAndLiveServers(2, 1);
-   registerK1AndK2();
-   verifyRefreshedEntriesFromServer();
-   backup1.invoke(HAInterestBaseTest.class, "putK1andK2");
-   backup1.invoke(HAInterestBaseTest.class, "startServer");
-   verifyDeadAndLiveServers(1, 2);
-   verifyRefreshedEntriesFromServer();
- }
- /**
-  * Bug Test for Bug # 35945
-  * A java level Deadlock between acquireConnection and RegionEntry
-  * during processRecoveredEndpoint by Dead Server Monitor Thread.
-  *
-  * @throws Exception
-  */
- public void testBug35945()throws Exception{
-
-   PORT1 =  ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
-   server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-   createClientPoolCacheConnectionToSingleServer(this.getName(),getServerHostName(server1.getHost()));
-   registerK1AndK2();
-   verifyRefreshedEntriesFromServer();
-
-   server1.invoke(HAInterestBaseTest.class, "stopServer");
-   verifyDeadAndLiveServers(1, 0);
-   // put on stopped server
-   server1.invoke(HAInterestBaseTest.class, "putK1andK2");
-   // spawn a thread to put on server , which will acquire a lock on entry 
-   setBridgeObserverForBeforeInterestRecovery();
-   server1.invoke(HAInterestBaseTest.class, "startServer");
-   verifyDeadAndLiveServers(0, 1);
-   waitForBeforeInterestRecoveryCallBack();
-   // verify updated value of k1 as a refreshEntriesFromServer
-   final Region r1 = cache.getRegion(Region.SEPARATOR+ REGION_NAME);
-   assertNotNull(r1);
-   
-   WaitCriterion wc = new WaitCriterion() {
-     String excuse;
-     public boolean done() {
-       Region.Entry e1 = r1.getEntry(k1);
-       Region.Entry e2 = r1.getEntry(k2);
-       if (e1 == null || !server_k1_updated.equals(e1.getValue())) {
-         excuse="k1="+(e1==null?"null":e1.getValue());
-         return false;
-       }
-       if (e2 == null || !server_k2.equals(e2.getValue())) {
-         excuse="k2="+(e2==null?"null":e2.getValue());
-         return false;
-       }
-       return true;
-     }
-     public String description() {
-       return excuse;
-     }
-   };
-   DistributedTestCase.waitForCriterion(wc, 30 * 1000, 1000, true);
-  // assertEquals(server_k2, r1.getEntry(k2).getValue());
-  // assertEquals(server_k1_updated ,r1.getEntry(k1).getValue());
- }
   /**
-   * Tests if failure occured in Interest recovery thread, then it should select new endpoint to
-   * register interest
+   * Tests a Scenario: stop a secondary server and update the registered entries
+   * on the stopped server start the server , DSM will recover interest list on
+   * this live server and verify that as a part of recovery it does not
+   * refreshes registered entries from the server, because it is secondary
+   */
+  public void testGIIFromSecondaryWhenDSMDetectsServerLive() throws Exception {
+    server1.invoke(HAInterestBaseTest.class, "closeCache");
+    server2.invoke(HAInterestBaseTest.class, "closeCache");
+    server3.invoke(HAInterestBaseTest.class, "closeCache");
+
+    PORT1 = ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
+    PORT2 = ((Integer) server2.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
+    PORT3 = ((Integer) server3.invoke(HAInterestBaseTest.class, "createServerCacheWithLocalRegion")).intValue();
+
+    server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    server3.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+
+    createClientPoolCache(this.getName(), getServerHostName(server1.getHost()));
+
+    VM backup1 = getBackupVM();
+    VM backup2 = getBackupVM(backup1);
+    backup1.invoke(HAInterestBaseTest.class, "stopServer");
+    backup2.invoke(HAInterestBaseTest.class, "stopServer");
+    verifyDeadAndLiveServers(2, 1);
+    registerK1AndK2();
+    verifyRefreshedEntriesFromServer();
+    backup1.invoke(HAInterestBaseTest.class, "putK1andK2");
+    backup1.invoke(HAInterestBaseTest.class, "startServer");
+    verifyDeadAndLiveServers(1, 2);
+    verifyRefreshedEntriesFromServer();
+  }
+
+  /**
+   * Bug Test for Bug # 35945 A java level Deadlock between acquireConnection
+   * and RegionEntry during processRecoveredEndpoint by Dead Server Monitor
+   * Thread.
    *
    * @throws Exception
    */
-  public void testInterestRecoveryFailure()throws Exception{
-
-   PORT1 =  ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
-   server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-   PORT2 =  ((Integer) server2.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
-   server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
-   createClientPoolCacheWithSmallRetryInterval(this.getName(),getServerHostName(server1.getHost()));
-   registerK1AndK2();
-   verifyRefreshedEntriesFromServer();
-   VM backup = getBackupVM();
-   VM primary = getPrimaryVM();
-   cache.getLogger().info("<ExpectedException action=add>" + "Server unreachable" +
-       "</ExpectedException>");
-   try {
-     backup.invoke(HAInterestBaseTest.class, "stopServer");
-     primary.invoke(HAInterestBaseTest.class, "stopServer");
-     verifyDeadAndLiveServers(2, 0);
-   } finally {
-     cache.getLogger().info("<ExpectedException action=remove>" + "Server unreachable" +
-         "</ExpectedException>");
-   }
-   primary.invoke(HAInterestBaseTest.class, "putK1andK2");
-   setBridgeObserverForBeforeInterestRecoveryFailure();
-   primary.invoke(HAInterestBaseTest.class, "startServer");
-   waitForBeforeInterestRecoveryCallBack();
-   if(exceptionOccured) {
-     fail("The DSM could not ensure that server 1 is started & serevr 2 is stopped");
-   }
-   final Region r1 = cache.getRegion(Region.SEPARATOR+ REGION_NAME);
-   assertNotNull(r1);
-   //pause(10000);
-   WaitCriterion wc = new WaitCriterion() {
-     String excuse;
-     public boolean done() {
-       Region.Entry e1 = r1.getEntry(k1);
-       Region.Entry e2 = r1.getEntry(k2);
-       if (e1 == null) {
-         excuse = "Entry for k1 still null";
-         return false;
-       }
-       if (e2 == null) {
-         excuse = "Entry for k2 still null";
-         return false;
-       }
-       if (!(server_k1.equals(e1.getValue()))) {
-         excuse = "Value for k1 wrong";
-         return false;
-       }
-       if (!(server_k2.equals(e2.getValue()))) {
-         excuse = "Value for k2 wrong";
-         return false;
-       }
-       return true;
-     }
-     public String description() {
-       return excuse;
-     }
-   };
-   DistributedTestCase.waitForCriterion(wc, 2 * 60 * 1000, 1000, true);
- }
+  public void testBug35945() throws Exception {
+    PORT1 = ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
+    server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    createClientPoolCacheConnectionToSingleServer(this.getName(), getServerHostName(server1.getHost()));
+    registerK1AndK2();
+    verifyRefreshedEntriesFromServer();
+
+    server1.invoke(HAInterestBaseTest.class, "stopServer");
+    verifyDeadAndLiveServers(1, 0);
+    // put on stopped server
+    server1.invoke(HAInterestBaseTest.class, "putK1andK2");
+    // spawn a thread to put on server , which will acquire a lock on entry
+    setBridgeObserverForBeforeInterestRecovery();
+    server1.invoke(HAInterestBaseTest.class, "startServer");
+    verifyDeadAndLiveServers(0, 1);
+    waitForBeforeInterestRecoveryCallBack();
+    // verify updated value of k1 as a refreshEntriesFromServer
+    final Region r1 = cache.getRegion(Region.SEPARATOR + REGION_NAME);
+    assertNotNull(r1);
+
+    WaitCriterion wc = new WaitCriterion() {
+      private String excuse;
+
+      @Override
+      public boolean done() {
+        Region.Entry e1 = r1.getEntry(k1);
+        Region.Entry e2 = r1.getEntry(k2);
+        if (e1 == null || !server_k1_updated.equals(e1.getValue())) {
+          excuse = "k1=" + (e1 == null ? "null" : e1.getValue());
+          return false;
+        }
+        if (e2 == null || !server_k2.equals(e2.getValue())) {
+          excuse = "k2=" + (e2 == null ? "null" : e2.getValue());
+          return false;
+        }
+        return true;
+      }
+
+      @Override
+      public String description() {
+        return excuse;
+      }
+    };
+    DistributedTestCase.waitForCriterion(wc, TIMEOUT_MILLIS, INTERVAL_MILLIS, true);
+  }
 
+  /**
+   * Tests if failure occurred in Interest recovery thread, then it should select
+   * new endpoint to register interest
+   */
+  public void testInterestRecoveryFailure() throws Exception {
+    addExpectedException("Server unreachable");
+    
+    PORT1 = ((Integer) server1.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
+    server1.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    PORT2 = ((Integer) server2.invoke(HAInterestBaseTest.class, "createServerCache")).intValue();
+    server2.invoke(HAInterestBaseTest.class, "createEntriesK1andK2");
+    createClientPoolCacheWithSmallRetryInterval(this.getName(), getServerHostName(server1.getHost()));
+    registerK1AndK2();
+    verifyRefreshedEntriesFromServer();
+    VM backup = getBackupVM();
+    VM primary = getPrimaryVM();
+
+    backup.invoke(HAInterestBaseTest.class, "stopServer");
+    primary.invoke(HAInterestBaseTest.class, "stopServer");
+    verifyDeadAndLiveServers(2, 0);
+
+    primary.invoke(HAInterestBaseTest.class, "putK1andK2");
+    setBridgeObserverForBeforeInterestRecoveryFailure();
+    primary.invoke(HAInterestBaseTest.class, "startServer");
+    waitForBeforeInterestRecoveryCallBack();
+    if (exceptionOccured) {
+      fail("The DSM could not ensure that server 1 is started & serevr 2 is stopped");
+    }
+    final Region r1 = cache.getRegion(Region.SEPARATOR + REGION_NAME);
+    assertNotNull(r1);
+    
+    WaitCriterion wc = new WaitCriterion() {
+      private String excuse;
+
+      @Override
+      public boolean done() {
+        Region.Entry e1 = r1.getEntry(k1);
+        Region.Entry e2 = r1.getEntry(k2);
+        if (e1 == null) {
+          excuse = "Entry for k1 still null";
+          return false;
+        }
+        if (e2 == null) {
+          excuse = "Entry for k2 still null";
+          return false;
+        }
+        if (!(server_k1.equals(e1.getValue()))) {
+          excuse = "Value for k1 wrong";
+          return false;
+        }
+        if (!(server_k2.equals(e2.getValue()))) {
+          excuse = "Value for k2 wrong";
+          return false;
+        }
+        return true;
+      }
+
+      @Override
+      public String description() {
+        return excuse;
+      }
+    };
+    DistributedTestCase.waitForCriterion(wc, TIMEOUT_MILLIS, INTERVAL_MILLIS, true);
+  }
 }