You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by bs...@apache.org on 2016/05/09 22:10:03 UTC
[1/3] incubator-geode git commit: Moving a distributed lock service
unit test to open-source.
Repository: incubator-geode
Updated Branches:
refs/heads/develop 717ba5c8b -> 2c148caaf
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2c148caa/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/standalone/DUnitLauncher.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/standalone/DUnitLauncher.java b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/standalone/DUnitLauncher.java
old mode 100644
new mode 100755
index f45d2dd..e06958a
--- a/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/standalone/DUnitLauncher.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/test/dunit/standalone/DUnitLauncher.java
@@ -92,7 +92,7 @@ public class DUnitLauncher {
public static final boolean LOCATOR_LOG_TO_DISK = Boolean.getBoolean("locatorLogToDisk");
static final String MASTER_PARAM = "DUNIT_MASTER";
- static final String RMI_PORT_PARAM = "gemfire.DUnitLauncher.RMI_PORT";
+ public static final String RMI_PORT_PARAM = "gemfire.DUnitLauncher.RMI_PORT";
static final String VM_NUM_PARAM = "gemfire.DUnitLauncher.VM_NUM";
private static final String LAUNCHED_PROPERTY = "gemfire.DUnitLauncher.LAUNCHED";
@@ -158,6 +158,7 @@ public class DUnitLauncher {
//create an RMI registry and add an object to share our tests config
int namingPort = AvailablePortHelper.getRandomAvailableTCPPort();
Registry registry = LocateRegistry.createRegistry(namingPort);
+ System.setProperty(RMI_PORT_PARAM, ""+namingPort);
final ProcessManager processManager = new ProcessManager(namingPort, registry);
master = new Master(registry, processManager);
[3/3] incubator-geode git commit: Moving a distributed lock service
unit test to open-source.
Posted by bs...@apache.org.
Moving a distributed lock service unit test to open-source.
There are still a few distributed unit tests that haven't been ported to
Geode. This one covers a lot of the functionality of the distributed lock
service.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/2c148caa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/2c148caa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/2c148caa
Branch: refs/heads/develop
Commit: 2c148caaf6a484bce065843d4d6ba8e916bd9ad2
Parents: 717ba5c
Author: Bruce Schuchardt <bs...@pivotal.io>
Authored: Mon May 9 15:08:31 2016 -0700
Committer: Bruce Schuchardt <bs...@pivotal.io>
Committed: Mon May 9 15:09:47 2016 -0700
----------------------------------------------------------------------
.../distributed/DistributedLockBlackboard.java | 20 +
.../DistributedLockBlackboardImpl.java | 97 +
.../DistributedLockServiceDUnitTest.java | 3246 ++++++++++++++++++
.../test/dunit/standalone/DUnitLauncher.java | 3 +-
4 files changed, 3365 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2c148caa/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboard.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboard.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboard.java
new file mode 100755
index 0000000..de8809c
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboard.java
@@ -0,0 +1,20 @@
+package com.gemstone.gemfire.distributed;
+
+import java.io.Serializable;
+import java.rmi.Remote;
+import java.rmi.RemoteException;
+
+/**
+ * Created by bschuchardt on 5/9/2016.
+ */
+public interface DistributedLockBlackboard extends Remote, Serializable {
+ void initCount() throws RemoteException;
+
+ void incCount() throws RemoteException;
+
+ long getCount() throws RemoteException;
+
+ void setIsLocked(boolean isLocked) throws RemoteException;
+
+ boolean getIsLocked() throws RemoteException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2c148caa/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboardImpl.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboardImpl.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboardImpl.java
new file mode 100755
index 0000000..6525cd1
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockBlackboardImpl.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed;
+
+import com.gemstone.gemfire.internal.Assert;
+import com.gemstone.gemfire.test.dunit.standalone.DUnitLauncher;
+
+import java.rmi.Naming;
+import java.rmi.NotBoundException;
+import java.rmi.RemoteException;
+import java.rmi.server.UnicastRemoteObject;
+
+
+public class DistributedLockBlackboardImpl extends UnicastRemoteObject implements DistributedLockBlackboard {
+ public static int Count;
+ public static int IsLocked;
+
+ public static DistributedLockBlackboard blackboard;
+
+ /**
+ * Zero-arg constructor for remote method invocations.
+ */
+ public DistributedLockBlackboardImpl() throws RemoteException {
+ super();
+ }
+
+ /**
+ * Creates a singleton event listeners blackboard.
+ */
+ public static DistributedLockBlackboard getInstance() throws Exception {
+ if ( blackboard == null )
+ initialize();
+ return blackboard;
+ }
+ private static synchronized void initialize() throws Exception {
+ if ( blackboard == null ) {
+ System.out.println(DUnitLauncher.RMI_PORT_PARAM + "=" + System.getProperty(DUnitLauncher.RMI_PORT_PARAM));
+ int namingPort = Integer.getInteger(DUnitLauncher.RMI_PORT_PARAM).intValue();
+ String name = "//localhost:" + namingPort + "/" + "DistributedLockBlackboard";
+ try {
+ blackboard = (DistributedLockBlackboard)Naming.lookup(name);
+ } catch (NotBoundException e) {
+ // create the master blackboard in this VM
+ blackboard = new DistributedLockBlackboardImpl();
+ Naming.bind(name, blackboard);
+ }
+ }
+ }
+
+ @Override
+ public synchronized void initCount() {
+ Count = 0;
+ }
+
+ @Override
+ public synchronized void incCount() {
+ Count = Count + 1;
+ }
+
+
+ @Override
+ public synchronized long getCount() {
+ return Count;
+ }
+
+ @Override
+ public synchronized void setIsLocked(boolean isLocked) {
+ if (isLocked) {
+ if (IsLocked < 1) IsLocked = 1;
+ } else {
+ if (IsLocked > 0) IsLocked = 0;
+ }
+ }
+
+ @Override
+ public synchronized boolean getIsLocked() {
+ long isLocked = IsLocked;
+ Assert.assertTrue(isLocked == 0 || isLocked == 1,
+ "DistributedLockBlackboard internal error - IsLocked is " + isLocked);
+ return isLocked == 1;
+ }
+
+}
[2/3] incubator-geode git commit: Moving a distributed lock service
unit test to open-source.
Posted by bs...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2c148caa/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
new file mode 100755
index 0000000..42c3f01
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
@@ -0,0 +1,3246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import junit.framework.Assert;
+import junit.framework.AssertionFailedError;
+
+import com.gemstone.gemfire.SystemFailure;
+import com.gemstone.gemfire.distributed.internal.DistributionManager;
+import com.gemstone.gemfire.distributed.internal.DistributionMessage;
+import com.gemstone.gemfire.distributed.internal.DistributionMessageObserver;
+import com.gemstone.gemfire.distributed.internal.locks.DLockGrantor;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRemoteToken;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor.DLockRequestMessage;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor.DLockResponseMessage;
+import com.gemstone.gemfire.distributed.internal.locks.DLockService;
+import com.gemstone.gemfire.distributed.internal.locks.DLockToken;
+import com.gemstone.gemfire.distributed.internal.locks.RemoteThread;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.util.StopWatch;
+import com.gemstone.gemfire.test.dunit.AsyncInvocation;
+import com.gemstone.gemfire.test.dunit.DistributedTestCase;
+import com.gemstone.gemfire.test.dunit.Host;
+import com.gemstone.gemfire.test.dunit.Invoke;
+import com.gemstone.gemfire.test.dunit.LogWriterUtils;
+import com.gemstone.gemfire.test.dunit.RMIException;
+import com.gemstone.gemfire.test.dunit.SerializableCallable;
+import com.gemstone.gemfire.test.dunit.SerializableRunnable;
+import com.gemstone.gemfire.test.dunit.ThreadUtils;
+import com.gemstone.gemfire.test.dunit.VM;
+import com.gemstone.gemfire.test.dunit.Wait;
+import com.gemstone.gemfire.test.dunit.WaitCriterion;
+
+/**
+ * This class tests distributed ownership via the DistributedLockService api.
+ */
+public class DistributedLockServiceDUnitTest extends DistributedTestCase {
+
+ protected static DistributedSystem dlstSystem;
+ private static DistributedLockBlackboard blackboard;
+ protected static Object monitor = new Object();
+
+ private int hits = 0;
+ private int completes = 0;
+ private boolean done;
+ private boolean got;
+
+
+ public DistributedLockServiceDUnitTest(String name) {
+ super(name);
+ if (blackboard == null) {
+ try {
+ blackboard = DistributedLockBlackboardImpl.getInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("initialization error", e);
+ }
+ }
+ }
+
+ /////////// Test lifecycle //////////
+
+ public static void caseSetUp() throws Exception {
+ disconnectAllFromDS();
+ }
+
+ public static void caseTearDown() throws Exception {
+ disconnectAllFromDS();
+ }
+
+ /**
+ * Returns a previously created (or new, if this is the first
+ * time this method is called in this VM) distributed system
+ * which is somewhat configurable via hydra test parameters.
+ */
+ @Override
+ public final void postSetUp() throws Exception {
+ // Create a DistributedSystem in every VM
+ connectDistributedSystem();
+
+ for (int h = 0; h < Host.getHostCount(); h++) {
+ Host host = Host.getHost(h);
+
+ for (int v = 0; v < host.getVMCount(); v++) {
+ host.getVM(v).invoke(
+ DistributedLockServiceDUnitTest.class, "connectDistributedSystem", null);
+ }
+ }
+ }
+
+ @Override
+ public final void preTearDown() throws Exception {
+ Invoke.invokeInEveryVM(DistributedLockServiceDUnitTest.class,
+ "destroyAllDLockServices");
+// invokeInEveryVM(DistributedLockServiceDUnitTest.class,
+// "remoteDumpAllDLockServices");
+
+ //InternalDistributedLockService.destroyAll();
+
+// // Disconnects the DistributedSystem in every VM - since
+// // each test randomly chooses whether shared memory is used
+// disconnectAllFromDS();
+
+ this.lockGrantor = null;
+ }
+
+ public static void destroyAllDLockServices() {
+ DLockService.destroyAll();
+ dlstSystem = null;
+ }
+
+ public static void remoteDumpAllDLockServices() {
+ DLockService.dumpAllServices();
+ }
+
+ ///////// Remote setup/teardown support
+
+ /**
+ * Connects a DistributedSystem, saves it in static variable "system"
+ */
+ protected static void connectDistributedSystem() {
+ dlstSystem = (new DistributedLockServiceDUnitTest("dummy")).getSystem();
+ }
+
+ ///////// Public test methods
+
+ public void testBasic() {
+ String serviceName = getUniqueName();
+ String objectName = "object";
+
+ // Create service
+ DistributedLockService service = DistributedLockService.create(serviceName, dlstSystem);
+
+ // Not locked initially
+ assertFalse(service.isHeldByCurrentThread(objectName));
+
+ // Get lock
+ assertTrue(service.lock(objectName, 3000, -1));
+ assertTrue(service.isHeldByCurrentThread(objectName));
+ assertTrue(service.lock(objectName, 3000, -1));
+ assertTrue(service.isHeldByCurrentThread(objectName));
+
+ // Release lock
+ service.unlock(objectName);
+ assertTrue(service.isHeldByCurrentThread(objectName));
+ service.unlock(objectName);
+ assertFalse(service.isHeldByCurrentThread(objectName));
+
+ // Destroy service
+ DistributedLockService.destroy(serviceName);
+ }
+
+ public void testCreateDestroy() throws Exception {
+ final String serviceName = getUniqueName();
+ final String abc = "abc";
+
+ // create and destroy dls
+ assertNull(DistributedLockService.getServiceNamed(serviceName));
+ DistributedLockService service = DistributedLockService.create(serviceName, getSystem());
+ assertSame(service, DistributedLockService.getServiceNamed(serviceName));
+ DistributedLockService.destroy(serviceName);
+
+ // assert attempt to use dls throws LockServiceDestroyedException
+ try {
+ service.lock(abc, -1, -1);
+ fail("didn't get LockServiceDestroyedException");
+ } catch (LockServiceDestroyedException ex) {
+ }
+
+ // assert that destroyed dls is no longer available
+ service = DistributedLockService.getServiceNamed(serviceName);
+ assertNull("" + service, service);
+
+ // recreate the dls
+ service = DistributedLockService.create(serviceName, getSystem());
+ assertTrue(!((DLockService)service).isDestroyed());
+ ((DLockService)service).checkDestroyed();
+
+ // get the same dls from another thread and hold a lock
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ DistributedLockService dls =
+ DistributedLockService.getServiceNamed(serviceName);
+ assertTrue(!((DLockService)dls).isDestroyed());
+ ((DLockService)dls).checkDestroyed();
+ dls.lock(abc, -1, -1); // get lock on abc and hold it
+ }
+ });
+ thread.start();
+ ThreadUtils.join(thread, 30 * 1000);
+
+ // start a new thread to wait for lock on abc
+ AsyncInvocation remoteWaitingThread =
+ Host.getHost(0).getVM(0).invokeAsync(new SerializableRunnable() {
+ public void run() {
+ DistributedLockService dls =
+ DistributedLockService.create(serviceName, getSystem());
+ try {
+ dls.lock(abc, -1, -1); // waiting to get lock abc
+ fail("remoteWaitingThread got lock after dls destroyed");
+ }
+ catch (LockServiceDestroyedException expected) {
+ return;
+ }
+ fail("remoteWaitingThread lock failed to throw LockServiceDestroyedException");
+ }
+ });
+
+ // loop will handle race condition with 1 sec sleep and retry
+ int retry = 10;
+ for (int i = 0; i < retry; i++) {
+ try {
+ // destroy DLS and free up remoteWaitingThread
+ Host.getHost(0).getVM(0).invoke(new SerializableRunnable() {
+ public void run() {
+ DistributedLockService.destroy(serviceName);
+ }
+ });
+ }
+ catch (RMIException e) {
+ // race condition: remoteWaitingThread probably hasn't created DLS yet
+ if (i < retry && e.getCause() instanceof IllegalArgumentException) {
+ sleep(1000);
+ continue;
+ }
+ else {
+ throw e;
+ }
+ }
+ break; // completed so break out of loop
+ }
+
+ DistributedLockService.destroy(serviceName);
+
+ // make sure remoteWaitingThread stopped waiting and threw LockServiceDestroyedException
+ ThreadUtils.join(remoteWaitingThread, 10 * 1000);
+ if (remoteWaitingThread.exceptionOccurred()) {
+ Throwable e = remoteWaitingThread.getException();
+ com.gemstone.gemfire.test.dunit.Assert.fail(e.getMessage(), e);
+ }
+
+ // make sure LockServiceDestroyedException is thrown
+ try {
+ service.lock(abc, -1, -1);
+ fail("didn't get LockServiceDestroyedException");
+ } catch (LockServiceDestroyedException ex) {
+ }
+
+ // make sure getServiceNamed returns null
+ service = DistributedLockService.getServiceNamed(serviceName);
+ assertNull("" + service, service);
+ }
+
+ protected static DistributedLockService dls_testFairness;
+ protected static int count_testFairness[] = new int[16];
+ protected static volatile boolean stop_testFairness;
+ protected static volatile boolean[] done_testFairness = new boolean[16];
+ static { Arrays.fill(done_testFairness, true); }
+ public void testFairness() throws Exception {
+ final String serviceName = "testFairness_" + getUniqueName();
+ final Object lock = "lock";
+
+ // get the lock and hold it until all threads are ready to go
+ DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ assertTrue(service.lock(lock, -1, -1));
+
+ final int[] vmThreads = new int[] { 1, 4, 8, 16 };
+ forNumVMsInvoke(vmThreads.length,
+ "remoteCreateService",
+ new Object[] { serviceName });
+ sleep(100);
+
+ // line up threads for the fairness race...
+ for (int i = 0; i < vmThreads.length; i++) {
+ final int vm = i;
+ LogWriterUtils.getLogWriter().info("[testFairness] lining up " + vmThreads[vm] +
+ " threads in vm " + vm);
+
+ for (int j = 0; j < vmThreads[vm]; j++) {
+ final int thread = j;
+ /*getLogWriter().info("[testFairness] setting up thread " + thread +
+ " in vm " + vm);*/
+
+ Host.getHost(0).getVM(vm).invokeAsync(new SerializableRunnable() {
+ public void run() {
+ // lock, inc count, and unlock until stop_testFairness is set true
+ try {
+ done_testFairness[thread] = false;
+ dls_testFairness = DistributedLockService.getServiceNamed(serviceName);
+ while (!stop_testFairness) {
+ assertTrue(dls_testFairness.lock(lock, -1, -1));
+ count_testFairness[thread]++;
+ dls_testFairness.unlock(lock);
+ }
+ done_testFairness[thread] = true;
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable t) {
+ LogWriterUtils.getLogWriter().warning(t);
+ fail(t.getMessage());
+ }
+ }
+ });
+ }
+ }
+ sleep(500); // 500 ms
+
+ // start the race!
+ service.unlock(lock);
+ sleep(1000 * 5); // 5 seconds
+ assertTrue(service.lock(lock, -1, -1));
+
+ // stop the race...
+ for (int i = 0; i < vmThreads.length; i++) {
+ final int vm = i;
+ Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {
+ public void run() {
+ stop_testFairness = true;
+ }
+ });
+ }
+ service.unlock(lock);
+ for (int i = 0; i < vmThreads.length; i++) {
+ final int vm = i;
+ Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {
+ public void run() {
+ try {
+ boolean testIsDone = false;
+ while (!stop_testFairness || !testIsDone) {
+ testIsDone = true;
+ for (int i2 = 0; i2 < done_testFairness.length; i2++) {
+ if (!done_testFairness[i2]) testIsDone = false;
+ }
+ }
+ DistributedLockService.destroy(serviceName);
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable t) {
+ fail(t.getMessage());
+ }
+ }
+ });
+ }
+
+ // calc total locks granted...
+ int totalLocks = 0;
+ int minLocks = Integer.MAX_VALUE;
+ int maxLocks = 0;
+
+ // add up total locks across all vms and threads...
+ int numThreads = 0;
+ for (int i = 0; i < vmThreads.length; i++) {
+ final int vm = i;
+ for (int j = 0; j < vmThreads[vm]; j++) {
+ final int thread = j;
+ Integer count = (Integer)Host.getHost(0).getVM(vm).invoke(() -> DistributedLockServiceDUnitTest.get_count_testFairness( new Integer(thread) ));
+ int numLocks = count.intValue();
+ if (numLocks < minLocks) minLocks = numLocks;
+ if (numLocks > maxLocks) maxLocks = numLocks;
+ totalLocks = totalLocks + numLocks;
+ numThreads++;
+ }
+ }
+
+ LogWriterUtils.getLogWriter().info("[testFairness] totalLocks=" + totalLocks +
+ " minLocks=" + minLocks +
+ " maxLocks=" + maxLocks);
+
+ int expectedLocks = (totalLocks / numThreads) + 1;
+
+ int deviation = (int)(expectedLocks * 0.3);
+ int lowThreshold = expectedLocks - deviation;
+ int highThreshold = expectedLocks + deviation;
+
+ LogWriterUtils.getLogWriter().info("[testFairness] deviation=" + deviation +
+ " expectedLocks=" + expectedLocks +
+ " lowThreshold=" + lowThreshold +
+ " highThreshold=" + highThreshold);
+
+ assertTrue("minLocks is less than lowThreshold",
+ minLocks >= lowThreshold);
+ assertTrue("maxLocks is greater than highThreshold",
+ maxLocks <= highThreshold);
+ }
+
+ /**
+ * Accessed by reflection. DO NOT REMOVE
+ * @param i
+ * @return
+ */
+ public static Integer get_count_testFairness(Integer i) {
+ return new Integer(count_testFairness[i.intValue()]);
+ }
+
+ public void testOneGetsAndOthersTimeOut() throws Exception {
+ doOneGetsAndOthersTimeOut(1, 1);
+// doOneGetsAndOthersTimeOut(2, 2);
+// doOneGetsAndOthersTimeOut(3, 2);
+ doOneGetsAndOthersTimeOut(4, 3);
+ }
+
+ private InternalDistributedMember lockGrantor;
+ private synchronized void assertGrantorIsConsistent(InternalDistributedMember id) {
+ if (this.lockGrantor == null) {
+ this.lockGrantor = id;
+ } else {
+ assertEquals("assertGrantorIsConsistent failed", lockGrantor, id);
+ }
+ }
+
+ /**
+ * Accessed via reflection. DO NOT REMOVE
+ * @param serviceName
+ * @return
+ */
+ public static InternalDistributedMember identifyLockGrantor(String serviceName) {
+ DLockService service = (DLockService)
+ DistributedLockService.getServiceNamed(serviceName);
+ assertNotNull(service);
+ InternalDistributedMember grantor = service.getLockGrantorId().getLockGrantorMember();
+ assertNotNull(grantor);
+ logInfo("In identifyLockGrantor - grantor is " + grantor);
+ return grantor;
+ }
+
+ /**
+ * Accessed via reflection. DO NOT REMOVE.
+ * @param serviceName
+ * @return
+ */
+ public static Boolean isLockGrantor(String serviceName) {
+ DLockService service = (DLockService)
+ DistributedLockService.getServiceNamed(serviceName);
+ assertNotNull(service);
+ Boolean result = Boolean.valueOf(service.isLockGrantor());
+ logInfo("In isLockGrantor: " + result);
+ return result;
+ }
+
+ /**
+ * Accessed via reflection. DO NOT REMOVE.
+ * @param serviceName
+ */
+ protected static void becomeLockGrantor(String serviceName) {
+ DLockService service = (DLockService)
+ DistributedLockService.getServiceNamed(serviceName);
+ assertNotNull(service);
+ logInfo("About to call becomeLockGrantor...");
+ service.becomeLockGrantor();
+ }
+
+ public void testGrantorSelection() {
+ // TODO change distributedCreateService usage to be concurrent threads
+
+ // bring up 4 members and make sure all identify one as grantor
+ int numVMs = 4;
+ final String serviceName = "testGrantorSelection_" + getUniqueName();
+ distributedCreateService(numVMs, serviceName);
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException ignore) {fail("interrupted");}
+
+ final Object[] args = new Object[] {serviceName};
+ final Host host = Host.getHost(0);
+ for (int vm=0; vm<numVMs; vm++) {
+ final int finalvm = vm;
+ logInfo("VM " + finalvm + " in " + serviceName + " about to invoke");
+ InternalDistributedMember id = (InternalDistributedMember)host.getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class,
+ "identifyLockGrantor",
+ args);
+ logInfo("VM " + finalvm + " in " + serviceName + " got " + id);
+ assertGrantorIsConsistent(id);
+ }
+ }
+
+ public void testBasicGrantorRecovery() {
+ //DLockGrantor.setUncleanDestroyEnabled(true);
+// try {
+ // 1) start up 4 VM members...
+ int numVMs = 4;
+ final String serviceName = "testBasicGrantorRecovery_" + getUniqueName();
+ distributedCreateService(numVMs, serviceName);
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException ignore) {fail("interrupted");}
+
+ final Object[] args = new Object[] {serviceName};
+ final Host host = Host.getHost(0);
+
+ int originalGrantor = 3;
+ host.getVM(originalGrantor).invoke(
+ DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+
+ // 2) find the grantor and disconnect him...
+ int originalVM = -1;
+ InternalDistributedMember oldGrantor = null;
+ for (int vm = 0; vm < numVMs; vm++) {
+ final int finalvm = vm;
+ Boolean isGrantor = (Boolean)host.getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+ if (isGrantor.booleanValue()) {
+ originalVM = vm;
+ oldGrantor = (InternalDistributedMember)host.getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+ break;
+ }
+ }
+
+ assertTrue(originalVM == originalGrantor);
+
+ host.getVM(originalVM).invoke(new SerializableRunnable() {
+ public void run() {
+ disconnectFromDS();
+ }
+ });
+
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException ignore) {fail("interrupted");}
+
+ // 3) verify that another member recovers for grantor
+ int attempts = 3;
+ for (int attempt = 0; attempt < attempts; attempt++) {
+ try {
+ for (int vm = 0; vm < numVMs; vm++) {
+ if (vm == originalVM) continue; // skip because he's disconnected
+ final int finalvm = vm;
+ logInfo("[testBasicGrantorRecovery] VM " + finalvm +
+ " in " + serviceName + " about to invoke");
+ InternalDistributedMember id = (InternalDistributedMember)host.getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class,
+ "identifyLockGrantor",
+ args);
+ logInfo("[testBasicGrantorRecovery] VM " + finalvm +
+ " in " + serviceName + " got " + id);
+ assertGrantorIsConsistent(id);
+ logInfo("[testBasicGrantorRecovery] new grantor " + id +
+ " is not old grantor " + oldGrantor);
+ assertEquals("New grantor must not equal the old grantor", true,
+ !id.equals(oldGrantor)); // new grantor != old grantor
+ } // loop thru vms
+ logInfo("[testBasicGrantorRecovery] succeeded attempt " + attempt);
+ break; // success
+ }
+ catch (AssertionFailedError e) {
+ logInfo("[testBasicGrantorRecovery] failed attempt " + attempt);
+ if (attempt == attempts-1) throw e;
+ }
+ } // loop thru attempts
+// }
+// finally {
+ //DLockGrantor.setUncleanDestroyEnabled(false);
+// }
+ }
+
+ public void testLockFailover() {
+ final int originalGrantorVM = 0;
+ final int oneVM = 1;
+ final int twoVM = 2;
+ final String serviceName = "testLockFailover-" + getUniqueName();
+
+ // create lock services...
+ LogWriterUtils.getLogWriter().fine("[testLockFailover] create services");
+
+ Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor( serviceName ));
+
+ Boolean isGrantor = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+ assertEquals("First member calling getLockGrantor failed to become grantor",
+ Boolean.TRUE, isGrantor);
+
+ // get locks...
+ LogWriterUtils.getLogWriter().fine("[testLockFailover] get lock");
+
+ Boolean locked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+originalGrantorVM ));
+ assertEquals("Failed to get lock in testLockFailover",
+ Boolean.TRUE, locked);
+
+ locked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+twoVM ));
+ assertEquals("Failed to get lock in testLockFailover",
+ Boolean.TRUE, locked);
+
+ locked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+oneVM ));
+ assertEquals("Failed to get lock in testLockFailover",
+ Boolean.TRUE, locked);
+
+ // disconnect originalGrantorVM...
+ LogWriterUtils.getLogWriter().fine("[testLockFailover] disconnect originalGrantorVM");
+
+ Host.getHost(0).getVM(originalGrantorVM).invoke(new SerializableRunnable() {
+ public void run() {
+ disconnectFromDS();
+ }
+ });
+
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException ignore) {fail("interrupted");}
+
+ // verify locks by unlocking...
+ LogWriterUtils.getLogWriter().fine("[testLockFailover] release locks");
+
+ Boolean unlocked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+twoVM ));
+ assertEquals("Failed to release lock in testLockFailover",
+ Boolean.TRUE, unlocked);
+
+ unlocked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+oneVM ));
+ assertEquals("Failed to release lock in testLockFailover",
+ Boolean.TRUE, unlocked);
+
+ // switch locks...
+ locked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+twoVM ));
+ assertEquals("Failed to get lock in testLockFailover",
+ Boolean.TRUE, locked);
+
+ locked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+oneVM ));
+ assertEquals("Failed to get lock in testLockFailover",
+ Boolean.TRUE, locked);
+
+ unlocked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+twoVM ));
+ assertEquals("Failed to release lock in testLockFailover",
+ Boolean.TRUE, unlocked);
+
+ unlocked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+oneVM ));
+ assertEquals("Failed to release lock in testLockFailover",
+ Boolean.TRUE, unlocked);
+
+ // verify grantor is unique...
+ LogWriterUtils.getLogWriter().fine("[testLockFailover] verify grantor identity");
+
+ InternalDistributedMember oneID = (InternalDistributedMember)Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor(serviceName));
+ InternalDistributedMember twoID = (InternalDistributedMember)Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor(serviceName));
+ assertTrue("Failed to identifyLockGrantor in testLockFailover",
+ oneID != null && twoID != null);
+ assertEquals("Failed grantor uniqueness in testLockFailover",
+ oneID, twoID);
+ }
+
+ public void testLockThenBecomeLockGrantor() {
+ final int originalGrantorVM = 0;
+ final int becomeGrantorVM = 1;
+ final int thirdPartyVM = 2;
+ final String serviceName = "testLockThenBecomeLockGrantor-" + getUniqueName();
+
+ // create lock services...
+ LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] create services");
+
+ Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException ignore) {fail("interrupted");}
+
+ Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+
+ Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor( serviceName ));
+
+ Boolean isGrantor = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+ assertEquals("First member calling getLockGrantor failed to become grantor",
+ Boolean.TRUE, isGrantor);
+
+ // control...
+ LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] check control");
+ Boolean check = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+ assertEquals("Check of control failed... unlock succeeded but nothing locked",
+ Boolean.FALSE, check);
+
+ // get locks...
+ LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] get lock");
+
+ Boolean locked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+originalGrantorVM ));
+ assertEquals("Failed to get lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, locked);
+
+ locked = (Boolean) Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+thirdPartyVM ));
+ assertEquals("Failed to get lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, locked);
+
+ locked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+becomeGrantorVM ));
+ assertEquals("Failed to get lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, locked);
+
+ // become lock grantor...
+ LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] become lock grantor");
+
+ Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.becomeLockGrantor( serviceName ));
+
+ try {
+ Thread.sleep(20);
+ } catch (InterruptedException ignore) {fail("interrupted");}
+
+ isGrantor = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+ assertEquals("Failed to become lock grantor",
+ Boolean.TRUE, isGrantor);
+
+ // verify locks by unlocking...
+ LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] release locks");
+
+ Boolean unlocked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+originalGrantorVM ));
+ assertEquals("Failed to release lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, unlocked);
+
+ unlocked = (Boolean) Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+thirdPartyVM ));
+ assertEquals("Failed to release lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, unlocked);
+
+ unlocked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+ assertEquals("Failed to release lock in testLockThenBecomeLockGrantor",
+ Boolean.TRUE, unlocked);
+
+ // test for bug in which transferred token gets re-entered causing lock recursion
+ unlocked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+ assertEquals("Transfer of tokens caused lock recursion in held lock",
+ Boolean.FALSE, unlocked);
+ }
+
+ public void testBecomeLockGrantor() {
+ // create lock services...
+ int numVMs = 4;
+ final String serviceName = "testBecomeLockGrantor-" + getUniqueName();
+ distributedCreateService(numVMs, serviceName);
+
+ // each one gets a lock...
+ for (int vm = 0; vm < numVMs; vm++) {
+ final int finalvm = vm;
+ Boolean locked = Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "obj-"+finalvm ));
+ assertEquals("Failed to get lock in testBecomeLockGrantor",
+ Boolean.TRUE, locked);
+ }
+
+ // find the grantor...
+ final Object[] args = new Object[] {serviceName};
+ int originalVM = -1;
+ InternalDistributedMember oldGrantor = null;
+ for (int vm = 0; vm < numVMs; vm++) {
+ final int finalvm = vm;
+ Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+ if (isGrantor.booleanValue()) {
+ originalVM = vm;
+ oldGrantor = (InternalDistributedMember)Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+ break;
+ }
+ }
+
+ LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] original grantor is " + oldGrantor);
+
+ // have one call becomeLockGrantor
+ for (int vm = 0; vm < numVMs; vm++) {
+ if (vm != originalVM) {
+ final int finalvm = vm;
+ Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "becomeLockGrantor", args);
+ Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+ assertEquals("isLockGrantor is false after calling becomeLockGrantor",
+ Boolean.TRUE, isGrantor);
+ break;
+ }
+ }
+
+ LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] one vm has called becomeLockGrantor...");
+
+ InternalDistributedMember newGrantor = null;
+ for (int vm = 0; vm < numVMs; vm++) {
+ final int finalvm = vm;
+ Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+ if (isGrantor.booleanValue()) {
+ newGrantor = (InternalDistributedMember)Host.getHost(0).getVM(finalvm).invoke(
+ DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+ break;
+ }
+ }
+ LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] new Grantor is " + newGrantor);
+ assertEquals(false, newGrantor.equals(oldGrantor));
+
+ // verify locks still held by unlocking
+ // each one unlocks...
+ for (int vm = 0; vm < numVMs; vm++) {
+ final int finalvm = vm;
+ Boolean unlocked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "obj-"+finalvm ));
+ assertEquals("Failed to unlock in testBecomeLockGrantor",
+ Boolean.TRUE, unlocked);
+ }
+
+ LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] finished");
+
+ // verify that pending requests are granted by unlocking them also
+ }
+
+ public void testTryLock() {
+ final Long waitMillis = new Long(100);
+
+ // create lock services...
+ LogWriterUtils.getLogWriter().fine("[testTryLock] create lock services");
+ final String serviceName = "testTryLock-" + getUniqueName();
+ distributedCreateService(4, serviceName);
+
+ // all 4 vms scramble to get tryLock but only one should succeed...
+ LogWriterUtils.getLogWriter().fine("[testTryLock] attempt to get tryLock");
+ int lockCount = 0;
+ for (int vm = 0; vm < 4; vm++) {
+ final int finalvm = vm;
+ Boolean locked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.tryLock( serviceName, "KEY", waitMillis ));
+ if (locked.booleanValue()) lockCount++;
+ }
+
+ assertEquals("More than one vm acquired the tryLock",
+ 1, lockCount);
+
+ LogWriterUtils.getLogWriter().fine("[testTryLock] unlock tryLock");
+ int unlockCount = 0;
+ for (int vm = 0; vm < 4; vm++) {
+ final int finalvm = vm;
+ Boolean unlocked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY" ));
+ if (unlocked.booleanValue()) unlockCount++;
+ }
+
+ assertEquals("More than one vm unlocked the tryLock",
+ 1, unlockCount);
+ }
+
+ public void testOneGetsThenOtherGets() throws Exception { // (numVMs, numThreadsPerVM)
+ doOneGetsThenOtherGets(1, 1);
+// doOneGetsThenOtherGets(2, 2);
+// doOneGetsThenOtherGets(3, 3);
+ doOneGetsThenOtherGets(4, 3);
+ }
+
+ public void testLockDifferentNames() throws Exception {
+ String serviceName = getUniqueName();
+
+ // Same VM
+ remoteCreateService(serviceName);
+ DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+ assertTrue(service.lock("obj1", -1, -1));
+ assertTrue(service.lock("obj2", -1, -1));
+ service.unlock("obj1");
+ service.unlock("obj2");
+
+ // Different VMs
+ VM vm = Host.getHost(0).getVM(0);
+ vm.invoke(() -> this.remoteCreateService(serviceName));
+ assertTrue(service.lock("masterVMobj", -1, -1));
+
+ assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+ serviceName, "otherVMobj", new Integer(-1), new Integer(0)
+ )));
+
+ service.unlock("masterVMobj");
+ }
+
+ public void testLocalGetLockAndIncrement() throws Exception {
+ String serviceName = getUniqueName();
+ remoteCreateService(serviceName);
+ DistributedLockService.getServiceNamed(serviceName);
+ assertEquals(Boolean.TRUE,
+ getLockAndIncrement(serviceName, "localVMobj", -1, 0));
+ }
+ public void testRemoteGetLockAndIncrement() {
+ String serviceName = getUniqueName();
+ VM vm = Host.getHost(0).getVM(0);
+ vm.invoke(() -> this.remoteCreateService(serviceName));
+ assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+ serviceName, "remoteVMobj", new Integer(-1), new Integer(0)
+ )));
+ }
+
+ public void testLockSameNameDifferentService()
+ {
+ String serviceName1 = getUniqueName() + "_1";
+ String serviceName2 = getUniqueName() + "_2";
+ String objName = "obj";
+
+ // Same VM
+ remoteCreateService(serviceName1);
+ remoteCreateService(serviceName2);
+ DistributedLockService service1 = DistributedLockService.getServiceNamed(serviceName1);
+ DistributedLockService service2 = DistributedLockService.getServiceNamed(serviceName2);
+ assertTrue(service1.lock(objName, -1, -1));
+ assertTrue(service2.lock(objName, -1, -1));
+ service1.unlock(objName);
+ service2.unlock(objName);
+
+ // Different VMs
+ VM vm = Host.getHost(0).getVM(0);
+ vm.invoke(() -> this.remoteCreateService(serviceName1));
+ vm.invoke(() -> this.remoteCreateService(serviceName2));
+ assertTrue(service1.lock(objName, -1, -1));
+ assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+ serviceName2, objName, new Integer(-1), new Integer(0)
+ )));
+ service1.unlock(objName);
+ }
+
+ public void testLeaseDoesntExpire()
+ throws InterruptedException
+ {
+ String serviceName = getUniqueName();
+ final Object objName = new Integer(3);
+
+ // Same VM
+ remoteCreateService(serviceName);
+ final DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+ // lock objName with a sufficiently long lease
+ assertTrue(service.lock(objName, -1, 60000));
+ // try to lock in another thread, with a timeout shorter than above lease
+ final boolean[] resultHolder = new boolean[] { false };
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ resultHolder[0] = !service.lock(objName, 1000, -1);
+ }
+ });
+ thread.start();
+ ThreadUtils.join(thread, 30 * 1000);
+ assertTrue(resultHolder[0]);
+ // the unlock should succeed without throwing LeaseExpiredException
+ service.unlock(objName);
+
+ // Different VM
+ VM vm = Host.getHost(0).getVM(0);
+ vm.invoke(() -> this.remoteCreateService(serviceName));
+ // lock objName in this VM with a sufficiently long lease
+ assertTrue(service.lock(objName, -1, 60000));
+ // try to lock in another VM, with a timeout shorter than above lease
+ assertEquals(Boolean.FALSE, vm.invoke(() -> this.getLockAndIncrement(
+ serviceName, objName, new Long(1000), new Long(0)
+ )));
+ // the unlock should succeed without throwing LeaseExpiredException
+ service.unlock(objName);
+ }
+
+ public void testLockUnlock() {
+ String serviceName = getUniqueName();
+ Object objName = new Integer(42);
+
+ remoteCreateService(serviceName);
+ DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ service.lock(objName, -1, -1);
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ service.unlock(objName);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+ }
+
+ public void testLockExpireUnlock() {
+ long leaseMs = 200;
+ long waitBeforeLockingMs = 210;
+
+ String serviceName = getUniqueName();
+ Object objName = new Integer(42);
+
+ remoteCreateService(serviceName);
+ DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+
+ assertTrue(!service.isHeldByCurrentThread(objName));
+
+ assertTrue(service.lock(objName, -1, leaseMs));
+ assertTrue(service.isHeldByCurrentThread(objName));
+
+ sleep(waitBeforeLockingMs); // should expire...
+ assertTrue(!service.isHeldByCurrentThread(objName));
+
+ try {
+ service.unlock(objName);
+ fail("unlock should have thrown LeaseExpiredException");
+ } catch (LeaseExpiredException ex) {
+ }
+ }
+
+ public void testLockRecursion() {
+ String serviceName = getUniqueName();
+ Object objName = new Integer(42);
+
+ remoteCreateService(serviceName);
+ DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ // initial lock...
+ Assert.assertTrue(service.lock(objName, -1, -1));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // recursion +1...
+ Assert.assertTrue(service.lock(objName, -1, -1));
+
+ // recursion -1...
+ service.unlock(objName);
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // and unlock...
+ service.unlock(objName);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+ }
+
+ public void testLockRecursionWithExpiration() {
+ long leaseMs = 500;
+ long waitBeforeLockingMs = 750;
+
+ String serviceName = getUniqueName();
+ Object objName = new Integer(42);
+
+ remoteCreateService(serviceName);
+ DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ // initial lock...
+ Assert.assertTrue(service.lock(objName, -1, leaseMs));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // recursion +1...
+ Assert.assertTrue(service.lock(objName, -1, leaseMs));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // expire...
+ sleep(waitBeforeLockingMs);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ // should fail...
+ try {
+ service.unlock(objName);
+ fail("unlock should have thrown LeaseExpiredException");
+ } catch (LeaseExpiredException ex) {
+ }
+
+ // relock it...
+ Assert.assertTrue(service.lock(objName, -1, leaseMs));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // and unlock to verify no recursion...
+ service.unlock(objName);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName)); // throws failure!!
+
+ // go thru again in different order...
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ // initial lock...
+ Assert.assertTrue(service.lock(objName, -1, leaseMs));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // expire...
+ sleep(waitBeforeLockingMs);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+ // relock it...
+ Assert.assertTrue(service.lock(objName, -1, leaseMs));
+ Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+ // and unlock to verify no recursion...
+ service.unlock(objName);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+ }
+
+ public void testLeaseExpiresBeforeOtherLocks()
+ throws InterruptedException
+ {
+ leaseExpiresTest(false);
+ }
+
+ public void testLeaseExpiresWhileOtherLocks()
+ throws InterruptedException
+ {
+ leaseExpiresTest(true);
+ }
+
+ private void leaseExpiresTest(boolean tryToLockBeforeExpiration)
+ throws InterruptedException
+ {
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] prepping");
+ long leaseMs = 100;
+ long waitBeforeLockingMs = tryToLockBeforeExpiration ? 50 : 110;
+
+ final String serviceName = getUniqueName();
+ final Object objName = new Integer(3);
+
+ // Same VM
+ remoteCreateService(serviceName);
+ final DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire first lock");
+ // lock objName with a short lease
+ assertTrue(service.lock(objName, -1, leaseMs));
+ sleep(waitBeforeLockingMs);
+
+ if (waitBeforeLockingMs > leaseMs) {
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+ }
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire lock that expired");
+ // try to lock in another thread - lease should have expired
+ final boolean[] resultHolder = new boolean[] { false };
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ resultHolder[0] = service.lock(objName, -1, -1);
+ service.unlock(objName);
+ Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+ }
+ });
+ thread.start();
+ ThreadUtils.join(thread, 30 * 1000);
+ assertTrue(resultHolder[0]);
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] unlock should throw LeaseExpiredException");
+ // this thread's unlock should throw LeaseExpiredException
+ try {
+ service.unlock(objName);
+ fail("unlock should have thrown LeaseExpiredException");
+ } catch (LeaseExpiredException ex) {
+ }
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] create service in other vm");
+ // Different VM
+ VM vm = Host.getHost(0).getVM(0);
+ vm.invoke(() -> this.remoteCreateService(serviceName));
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire lock again and expire");
+ // lock objName in this VM with a short lease
+ assertTrue(service.lock(objName, -1, leaseMs));
+ sleep(waitBeforeLockingMs);
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] succeed lock in other vm");
+ // try to lock in another VM - should succeed
+ assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+ serviceName, objName, new Long(-1), new Long(0)
+ )));
+
+ LogWriterUtils.getLogWriter().fine("[testLeaseExpires] unlock should throw LeaseExpiredException again");
+ // this VMs unlock should throw LeaseExpiredException
+ try {
+ service.unlock(objName);
+ fail("unlock should have thrown LeaseExpiredException");
+ } catch (LeaseExpiredException ex) {
+ }
+ }
+
+ public void testSuspendLockingAfterExpiration() throws Exception {
+ LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest]");
+
+ final long leaseMillis = 100;
+ final long suspendWaitMillis = 10000;
+
+ final String serviceName = getUniqueName();
+ final Object key = new Integer(3);
+
+ // controller locks key and then expires - controller is grantor
+
+ DistributedLockService dls
+ = DistributedLockService.create(serviceName, getSystem());
+
+ assertTrue(dls.lock(key, -1, leaseMillis));
+
+ // wait for expiration
+ sleep(leaseMillis*2);
+
+ LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest] unlock should throw LeaseExpiredException");
+ // this thread's unlock should throw LeaseExpiredException
+ try {
+ dls.unlock(key);
+ fail("unlock should have thrown LeaseExpiredException");
+ } catch (LeaseExpiredException ex) {
+ }
+
+ // other vm calls suspend
+
+ LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest] call to suspend locking");
+ Host.getHost(0).getVM(0).invoke(new SerializableRunnable() {
+ public void run() {
+ final DistributedLockService dlock
+ = DistributedLockService.create(serviceName, getSystem());
+ dlock.suspendLocking(suspendWaitMillis);
+ dlock.resumeLocking();
+ assertTrue(dlock.lock(key, -1, leaseMillis));
+ dlock.unlock(key);
+ }
+ });
+ }
+
+ volatile boolean started = false;
+ volatile boolean gotLock = false;
+ volatile Throwable exception = null;
+ volatile Throwable throwable = null;
+
+ public void testLockInterruptiblyIsInterruptible() {
+ started = false;
+ gotLock = false;
+ exception = null;
+ throwable = null;
+
+ // Lock entire service in first thread
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] get and hold the lock");
+ final String serviceName = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ service.becomeLockGrantor();
+ assertTrue(service.lock("obj", 1000, -1));
+
+ // Start second thread that tries to lock in second thread
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] call lockInterruptibly");
+ Thread thread2 = new Thread(new Runnable() {
+ public void run() {
+ try {
+ started = true;
+ gotLock = service.lockInterruptibly("obj", -1, -1);
+ } catch (InterruptedException ex) {
+ exception = ex;
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable t) {
+ throwable = t;
+ }
+ }
+ });
+ thread2.start();
+
+ // Interrupt second thread
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] interrupt calling thread");
+ while (!started) Thread.yield();
+ thread2.interrupt();
+ ThreadUtils.join(thread2, 20 * 1000);
+
+ // Expect it got InterruptedException and didn't lock the service
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] verify failed to get lock");
+ assertFalse(gotLock);
+ if (throwable != null) {
+ LogWriterUtils.getLogWriter().warning(
+ "testLockInterruptiblyIsInterruptible threw unexpected Throwable",
+ throwable);
+ }
+ assertNotNull(exception);
+
+ // Unlock "obj" in first thread
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] unlock the lock");
+ service.unlock("obj");
+
+ // Make sure it didn't get locked by second thread
+ LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] try to get lock with timeout should not fail");
+ assertTrue(service.lock("obj", 5000, -1));
+ DistributedLockService.destroy(serviceName);
+ }
+
+ volatile boolean wasFlagSet = false;
+
+ public void testLockIsNotInterruptible() {
+ // Lock entire service in first thread
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] lock in first thread");
+ started = false;
+ gotLock = false;
+ exception = null;
+ wasFlagSet = false;
+
+ final String serviceName = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ assertTrue(service.lock("obj", 1000, -1));
+
+ // Start second thread that tries to lock in second thread
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] attempt lock in second thread");
+ Thread thread2 = new Thread(new Runnable() {
+ public void run() {
+ try {
+ started = true;
+ gotLock = service.lock("obj", -1, -1);
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] thread2 finished lock() - got " + gotLock);
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable ex) {
+ LogWriterUtils.getLogWriter().warning("[testLockIsNotInterruptible] Caught...", ex);
+ exception = ex;
+ }
+ wasFlagSet = Thread.currentThread().isInterrupted();
+ }
+ });
+ thread2.start();
+
+ // Interrupt second thread
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] interrupt second thread");
+ while (!started) Thread.yield();
+ sleep(500);
+ thread2.interrupt();
+ // Expect it didn't get an exception and didn't lock the service
+ sleep(500);
+ assertFalse(gotLock);
+ assertNull(exception);
+
+ // Unlock entire service in first thread
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] unlock in first thread");
+ service.unlock("obj");
+ sleep(500);
+
+ // Expect that thread2 should now complete execution.
+ ThreadUtils.join(thread2, 20 * 1000);
+
+ // Now thread2 should have gotten the lock, not the exception, but the
+ // thread's flag should be set
+ LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] verify second thread got lock");
+ assertNull(exception);
+ assertTrue(gotLock);
+ assertTrue(wasFlagSet);
+ }
+
+ /**
+ * Test DistributedLockService.acquireExclusiveLocking(), releaseExclusiveLocking()
+ */
+ public void testSuspendLockingBasic()
+ throws InterruptedException {
+ final DistributedLockService service =
+ DistributedLockService.create(getUniqueName(), dlstSystem);
+
+ try {
+ service.resumeLocking();
+ fail("Didn't throw LockNotHeldException");
+ } catch (LockNotHeldException ex) {
+ // expected
+ }
+
+ assertTrue(service.suspendLocking(-1));
+ service.resumeLocking();
+
+ // It's not reentrant
+ assertTrue(service.suspendLocking(1000));
+ try {
+ service.suspendLocking(1);
+ fail("didn't get IllegalStateException");
+ } catch (IllegalStateException ex) {
+ // expected
+ }
+ service.resumeLocking();
+
+ // Get "false" if another thread is holding it
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ logInfo("new thread about to suspendLocking()");
+ assertTrue(service.suspendLocking(1000));
+ }
+ });
+ thread.start();
+ ThreadUtils.join(thread, 30 * 1000);
+ logInfo("main thread about to suspendLocking");
+ assertTrue(!service.suspendLocking(1000));
+ }
+
+ /**
+ * Test that exlusive locking prohibits locking activity
+ */
+ public void testSuspendLockingProhibitsLocking()
+ {
+ final String name = getUniqueName();
+ distributedCreateService(2, name);
+ DistributedLockService service =
+ DistributedLockService.getServiceNamed(name);
+
+ // Should be able to lock from other VM
+ VM vm1 = Host.getHost(0).getVM(1);
+ assertTrue(vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+
+ assertTrue(service.suspendLocking(1000));
+
+ // vm1 is the grantor... use debugHandleSuspendTimeouts
+ vm1.invoke(new SerializableRunnable("setDebugHandleSuspendTimeouts") {
+ public void run() {
+ DLockService dls =
+ (DLockService) DistributedLockService.getServiceNamed(name);
+ assertTrue(dls.isLockGrantor());
+ DLockGrantor grantor = dls.getGrantorWithNoSync();
+ grantor.setDebugHandleSuspendTimeouts(5000);
+ }
+ });
+
+ // Shouldn't be able to lock a name from another VM
+ assertTrue(!vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+
+ service.resumeLocking();
+
+ vm1.invoke(new SerializableRunnable("unsetDebugHandleSuspendTimeouts") {
+ public void run() {
+ DLockService dls =
+ (DLockService) DistributedLockService.getServiceNamed(name);
+ assertTrue(dls.isLockGrantor());
+ DLockGrantor grantor = dls.getGrantorWithNoSync();
+ grantor.setDebugHandleSuspendTimeouts(0);
+ }
+ });
+
+ // Should be able to lock again
+ assertTrue(vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+
+ }
+
+ /**
+ * Test that suspend locking behaves under various usage patterns. This
+ * ensures that suspend and regular locks behave as ReadWriteLocks and
+ * processing occurs in order.
+ */
+ public void notestSuspendLockingBehaves() throws Exception {
+ try {
+ doTestSuspendLockingBehaves();
+ }
+ finally {
+ Invoke.invokeInEveryVM(new SerializableRunnable() {
+ public void run() {
+ try {
+ if (suspendClientSuspendLockingBehaves != null) {
+ suspendClientSuspendLockingBehaves.stop();
+ suspendClientSuspendLockingBehaves = null;
+ }
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable t) {
+ LogWriterUtils.getLogWriter().error("Error in testSuspendLockingBehaves finally", t);
+ }
+ try {
+ if (lockClientSuspendLockingBehaves != null) {
+ lockClientSuspendLockingBehaves.stop();
+ lockClientSuspendLockingBehaves = null;
+ }
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable t) {
+ LogWriterUtils.getLogWriter().error("Error in testSuspendLockingBehaves finally", t);
+ }
+ }
+ });
+ }
+ }
+ private void doTestSuspendLockingBehaves() throws Exception {
+ final String dlsName = getUniqueName();
+ final VM vmGrantor = Host.getHost(0).getVM(0);
+ final VM vmOne = Host.getHost(0).getVM(1);
+ final VM vmTwo = Host.getHost(0).getVM(2);
+ final VM vmThree = Host.getHost(0).getVM(3);
+ final String key1 = "key1";
+
+ // TODO: make sure suspend thread can get other locks
+
+ // TODO: test local (in grantor) locks and suspends also
+
+ // define some SerializableRunnables
+ final SerializableRunnable createDLS =
+ new SerializableRunnable("Create "+dlsName) {
+ public void run() {
+ DistributedLockService.create(dlsName, getSystem());
+ lockClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
+ suspendClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
+ assertFalse(isLockGrantor(dlsName).booleanValue());
+ }
+ };
+ final SerializableRunnable suspendLocking =
+ new SerializableRunnable("Suspend locking "+dlsName) {
+ public void run() {
+ suspendClientSuspendLockingBehaves.suspend();
+ }
+ };
+ final SerializableRunnable resumeLocking =
+ new SerializableRunnable("Resume locking "+dlsName) {
+ public void run() {
+ suspendClientSuspendLockingBehaves.resume();
+ }
+ };
+ final SerializableRunnable lockKey =
+ new SerializableRunnable("Get lock "+dlsName) {
+ public void run() {
+ lockClientSuspendLockingBehaves.lock();
+ }
+ };
+ final SerializableRunnable unlockKey =
+ new SerializableRunnable("Unlock "+dlsName) {
+ public void run() {
+ lockClientSuspendLockingBehaves.unlock();
+ }
+ };
+
+ // create grantor
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] Create grantor "+dlsName);
+ vmGrantor.invoke(new SerializableRunnable("Create grantor "+dlsName) {
+ public void run() {
+ DistributedLockService.create(dlsName, getSystem());
+ DistributedLockService.getServiceNamed(dlsName).lock(key1, -1, -1);
+ DistributedLockService.getServiceNamed(dlsName).unlock(key1);
+ assertTrue(isLockGrantor(dlsName).booleanValue());
+ }
+ });
+
+ // create dls in other vms
+// getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmOne");
+ vmOne.invoke(createDLS);
+// getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmTwo");
+ vmTwo.invoke(createDLS);
+// getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmThree");
+ vmThree.invoke(createDLS);
+
+ // get a lock
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vms for lock");
+// getLogWriter().info("[testSuspendLockingBehaves] vmOne lock");
+ vmOne.invoke(lockKey);
+// getLogWriter().info("[testSuspendLockingBehaves] start vmTwoLocking");
+ AsyncInvocation vmTwoLocking = vmTwo.invokeAsync(lockKey);
+ Wait.pause(2000); // make sure vmTwo is first in line
+// getLogWriter().info("[testSuspendLockingBehaves] start vmThreeLocking");
+ AsyncInvocation vmThreeLocking = vmThree.invokeAsync(lockKey);
+ Wait.pause(2000);
+
+ // make sure vmTwo and vmThree are still waiting for lock on key1
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoLocking still alive");
+ Wait.pause(100);
+ assertTrue(vmTwoLocking.isAlive());
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeLocking still alive");
+ Wait.pause(100);
+ assertTrue(vmThreeLocking.isAlive());
+
+ // let vmTwo get key
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmTwo can get key");
+// getLogWriter().info("[testSuspendLockingBehaves] vmOne unlock");
+ vmOne.invoke(unlockKey);
+ ThreadUtils.join(vmTwoLocking, 10 * 1000);
+
+ // start suspending in vmOne and vmTwo
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start suspending requests");
+// getLogWriter().info("[testSuspendLockingBehaves] start vmOneSuspending");
+ AsyncInvocation vmOneSuspending = vmOne.invokeAsync(suspendLocking);
+ Wait.pause(2000); // make sure vmOne is first in line
+// getLogWriter().info("[testSuspendLockingBehaves] start vmTwoSuspending");
+ AsyncInvocation vmTwoSuspending = vmTwo.invokeAsync(suspendLocking);
+ Wait.pause(2000);
+
+ // let vmThree finish locking key
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmThree can get key");
+// getLogWriter().info("[testSuspendLockingBehaves] vmTwo unlock");
+ vmTwo.invoke(unlockKey);
+ ThreadUtils.join(vmThreeLocking, 10 * 1000);
+
+ // have vmOne get back in line for locking key
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start another lock request");
+// getLogWriter().info("[testSuspendLockingBehaves] start vmOneLockingAgain");
+ AsyncInvocation vmOneLockingAgain = vmOne.invokeAsync(lockKey);
+ Wait.pause(2000);
+
+ // let vmOne suspend locking
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne suspend locking");
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmOneSuspending still alive");
+ Wait.pause(100);
+ assertTrue(vmOneSuspending.isAlive());
+// getLogWriter().info("[testSuspendLockingBehaves] vmThree unlock");
+ vmThree.invoke(unlockKey);
+ ThreadUtils.join(vmOneSuspending, 10 * 1000);
+
+ // start suspending in vmThree
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vmThree for suspending");
+// getLogWriter().info("[testSuspendLockingBehaves] start vmThreeSuspending");
+ AsyncInvocation vmThreeSuspending = vmThree.invokeAsync(suspendLocking);
+ Wait.pause(2000);
+
+ // let vmTwo suspend locking
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmTwo suspend locking");
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoSuspending still alive");
+ Wait.pause(100);
+ assertTrue(vmTwoSuspending.isAlive());
+// getLogWriter().info("[testSuspendLockingBehaves] vmOne resumes locking");
+ vmOne.invoke(resumeLocking);
+ ThreadUtils.join(vmTwoSuspending, 10 * 1000);
+
+ // let vmOne get that lock
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne get that lock");
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmOneLockingAgain still alive");
+ Wait.pause(100);
+ assertTrue(vmOneLockingAgain.isAlive());
+// getLogWriter().info("[testSuspendLockingBehaves] vmTwo resumes locking");
+ vmTwo.invoke(resumeLocking);
+ ThreadUtils.join(vmOneLockingAgain, 10 * 1000);
+
+ // let vmThree suspend locking
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmThree suspend locking");
+// getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeSuspending still alive");
+ Wait.pause(100);
+ assertTrue(vmThreeSuspending.isAlive());
+// getLogWriter().info("[testSuspendLockingBehaves] vmOne unlocks again");
+ vmOne.invoke(unlockKey);
+ ThreadUtils.join(vmThreeSuspending, 10 * 1000);
+
+ // done
+// getLogWriter().info("[testSuspendLockingBehaves] vmThree resumes locking");
+ vmThree.invoke(resumeLocking);
+ }
+ protected static BasicLockClient suspendClientSuspendLockingBehaves;
+ protected static BasicLockClient lockClientSuspendLockingBehaves;
+
+ /**
+ * Test that exlusive locking prohibits locking activity
+ */
+ public void testSuspendLockingBlocksUntilNoLocks()
+ throws InterruptedException
+ {
+
+ final String name = getUniqueName();
+ distributedCreateService(2, name);
+ final DistributedLockService service =
+ DistributedLockService.getServiceNamed(name);
+
+ // Get lock from other VM. Since same thread needs to lock and unlock,
+ // invoke asynchronously, get lock, wait to be notified, then unlock.
+ VM vm1 = Host.getHost(0).getVM(1);
+ vm1.invokeAsync(new SerializableRunnable("Lock & unlock in vm1") {
+ public void run() {
+ DistributedLockService service2 =
+ DistributedLockService.getServiceNamed(name);
+ assertTrue(service2.lock("lock", -1, -1));
+ synchronized (monitor) {
+ try {
+ monitor.wait();
+ } catch (InterruptedException ex) {
+ System.out.println("Unexpected InterruptedException");
+ fail("interrupted");
+ }
+ }
+ service2.unlock("lock");
+ }
+ });
+ // Let vm1's thread get the lock and go into wait()
+ Thread.sleep(100);
+
+ Thread thread = new Thread(new Runnable() {
+ public void run() {
+ setGot(service.suspendLocking(-1));
+ setDone(true);
+ service.resumeLocking();
+ }
+ });
+ setGot(false);
+ setDone(false);
+ thread.start();
+
+ // Let thread start, make sure it's blocked in suspendLocking
+ Thread.sleep(100);
+ assertFalse("Before release, got: " + getGot() + ", done: " + getDone(), getGot() || getDone());
+
+ vm1.invoke(new SerializableRunnable("notify vm1 to unlock") {
+ public void run() {
+ synchronized (monitor) {
+ monitor.notify();
+ }
+ }
+ });
+
+ // Let thread finish, make sure it successfully suspended and is done
+ WaitCriterion ev = new WaitCriterion() {
+ public boolean done() {
+ return getDone();
+ }
+ public String description() {
+ return null;
+ }
+ };
+ Wait.waitForCriterion(ev, 30 * 1000, 200, true);
+ if (!getGot() || !getDone()) {
+ ThreadUtils.dumpAllStacks();
+ }
+ assertTrue("After release, got: " + getGot() + ", done: " + getDone(), getGot() && getDone());
+
+ }
+
+ public void testSuspendLockingInterruptiblyIsInterruptible() {
+
+ started = false;
+ gotLock = false;
+ exception = null;
+
+ // Lock entire service in first thread
+ final String name = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(name, dlstSystem);
+ assertTrue(service.suspendLocking(1000));
+
+ // Start second thread that tries to lock in second thread
+ Thread thread2 = new Thread(new Runnable() {
+ public void run() {
+ try {
+ started = true;
+ gotLock = service.suspendLockingInterruptibly(-1);
+ } catch (InterruptedException ex) {
+ exception = ex;
+ }
+ }
+ });
+ thread2.start();
+
+ // Interrupt second thread
+ while (!started) Thread.yield();
+ thread2.interrupt();
+ ThreadUtils.join(thread2, 20 * 1000);
+
+ // Expect it got InterruptedException and didn't lock the service
+ sleep(500);
+ assertFalse(gotLock);
+ assertNotNull(exception);
+
+ // Unlock entire service in first thread
+ service.resumeLocking();
+ sleep(500);
+
+ // Make sure it didn't get locked by second thread
+ assertTrue(service.suspendLocking(1000));
+ DistributedLockService.destroy(name);
+ }
+
+ public void testSuspendLockingIsNotInterruptible() {
+
+ started = false;
+ gotLock = false;
+ exception = null;
+ wasFlagSet = false;
+
+ // Lock entire service in first thread
+ final String name = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(name, dlstSystem);
+ assertTrue(service.suspendLocking(1000));
+
+ // Start second thread that tries to lock in second thread
+ Thread thread2 = new Thread(new Runnable() {
+ public void run() {
+ try {
+ started = true;
+ gotLock = service.suspendLocking(-1);
+ }
+ catch (VirtualMachineError e) {
+ SystemFailure.initiateFailure(e);
+ throw e;
+ }
+ catch (Throwable ex) {
+ exception = ex;
+ }
+ wasFlagSet = Thread.currentThread().isInterrupted();
+ }
+ });
+ thread2.start();
+
+ // Interrupt second thread
+ while (!started) Thread.yield();
+ thread2.interrupt();
+ // Expect it didn't get an exception and didn't lock the service
+ sleep(500);
+ assertFalse(gotLock);
+ assertNull(exception);
+
+ // Unlock entire service in first thread
+ service.resumeLocking();
+ ThreadUtils.join(thread2, 20 * 1000);
+
+ // Now thread2 should have gotten the lock, not the exception, but the
+ // thread's flag should be set
+ LogWriterUtils.getLogWriter().info("[testSuspendLockingIsNotInterruptible]" +
+ " gotLock=" + gotLock +
+ " wasFlagSet=" + wasFlagSet +
+ " exception=" + exception, exception);
+ assertTrue(gotLock);
+ assertNull(exception);
+ assertTrue(wasFlagSet);
+ }
+
+ /**
+ * Tests what happens when you attempt to lock a name on a lock
+ * service that has been destroyed.
+ *
+ * @author David Whitlock
+ */
+ public void testLockDestroyedService() {
+ String serviceName = this.getUniqueName();
+ DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ DistributedLockService.destroy(serviceName);
+ try {
+ boolean locked = service.lock("TEST", -1, -1);
+ fail("Lock of destroyed service returned: " + locked);
+
+ } catch (LockServiceDestroyedException ex) {
+ // pass...
+ }
+ }
+
+ public void testDepartedLastOwnerWithLease() {
+ final String serviceName = this.getUniqueName();
+
+ // Create service in this VM
+ DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ assertTrue(service.lock("key", -1, -1));
+ service.unlock("key");
+
+ // Create service in other VM
+ VM otherVm = Host.getHost(0).getVM(0);
+ otherVm.invoke(new SerializableRunnable() {
+ public void run() {
+ DistributedLockService service2 =
+ DistributedLockService.create(serviceName, dlstSystem);
+ service2.lock("key", -1, 360000);
+ service2.unlock("key");
+ // Wait for asynchronous messaging to complete
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ex) {
+ fail("interrupted");
+ }
+ disconnectFromDS();
+ }
+ });
+
+ // Now lock back in this VM
+ assertTrue(service.lock("key", -1, -1));
+
+ }
+
+ public void testDepartedLastOwnerNoLease() {
+ final String serviceName = this.getUniqueName();
+
+ // Create service in this VM
+ DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ assertTrue(service.lock("key", -1, -1));
+ service.unlock("key");
+
+ // Create service in other VM
+ VM otherVm = Host.getHost(0).getVM(0);
+ otherVm.invoke(new SerializableRunnable() {
+ public void run() {
+ DistributedLockService service2 =
+ DistributedLockService.create(serviceName, dlstSystem);
+ service2.lock("key", -1, -1);
+ service2.unlock("key");
+ // Wait for asynchronous messaging to complete
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ex) {
+ fail("interrupted");
+ }
+ disconnectFromDS();
+ }
+ });
+
+ // Now lock back in this VM
+ assertTrue(service.lock("key", -1, -1));
+
+ }
+
+ /**
+ * Tests for 32461 R3 StuckLocks can occur on locks with an expiration lease
+ * <p>
+ * VM-A locks/unlocks "lock", VM-B leases "lock" and disconnects, VM-C
+ * attempts to lock "lock" and old dlock throws StuckLockException. VM-C
+ * should now succeed in acquiring the lock.
+ */
+ public void testBug32461() throws Exception {
+ LogWriterUtils.getLogWriter().fine("[testBug32461] prepping");
+
+ final String serviceName = getUniqueName();
+ final Object objName = "32461";
+ final int VM_A = 0;
+ final int VM_B = 1;
+ final int VM_C = 2;
+
+ // VM-A locks/unlocks "lock"...
+ LogWriterUtils.getLogWriter().fine("[testBug32461] VM-A locks/unlocks '32461'");
+
+ Host.getHost(0).getVM(VM_A).invoke(new SerializableRunnable() {
+ public void run() {
+ remoteCreateService(serviceName);
+ final DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+ assertTrue(service.lock(objName, -1, Long.MAX_VALUE));
+ service.unlock(objName);
+ }
+ });
+
+ // VM-B leases "lock" and disconnects,
+ LogWriterUtils.getLogWriter().fine("[testBug32461] VM_B leases '32461' and disconnects");
+
+ Host.getHost(0).getVM(VM_B).invoke(new SerializableRunnable() {
+ public void run() {
+ remoteCreateService(serviceName);
+ final DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+ assertTrue(service.lock(objName, -1, Long.MAX_VALUE));
+ DistributedLockService.destroy(serviceName);
+ disconnectFromDS();
+ }
+ });
+
+ LogWriterUtils.getLogWriter().fine("[testBug32461] VM_C attempts to lock '32461'");
+
+ Host.getHost(0).getVM(VM_C).invoke(new SerializableRunnable() {
+ public void run() {
+ remoteCreateService(serviceName);
+ final DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+ assertTrue(service.lock(objName, -1, -1));
+ service.unlock(objName);
+ }
+ });
+ }
+
+ public void testNoStuckLock() {
+ final String serviceName = this.getUniqueName();
+ final Object keyWithLease = "key-with-lease";
+ final Object keyNoLease = "key-no-lease";
+
+ // Create service in this VM
+ DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+
+ assertTrue(service.lock(keyWithLease, -1, -1));
+ service.unlock(keyWithLease);
+
+ assertTrue(service.lock(keyNoLease, -1, -1));
+ service.unlock(keyNoLease);
+
+ // Create service in other VM
+ VM otherVm = Host.getHost(0).getVM(0);
+ otherVm.invoke(new SerializableRunnable() {
+ public void run() {
+ DistributedLockService service2 =
+ DistributedLockService.create(serviceName, dlstSystem);
+ service2.lock(keyWithLease, -1, 360000);
+ service2.lock(keyNoLease, -1, -1);
+ disconnectFromDS();
+ }
+ });
+
+ // Now lock back in this VM... no stuck locks anymore
+ assertTrue(service.lock(keyWithLease, -1, -1));
+ service.unlock(keyWithLease);
+ assertTrue(service.lock(keyNoLease, -1, -1));
+ service.unlock(keyNoLease);
+ }
+
+ volatile boolean startedThread1_testReleaseOrphanedGrant;
+ volatile boolean releaseThread1_testReleaseOrphanedGrant;
+ volatile boolean startedThread2_testReleaseOrphanedGrant;
+ volatile boolean gotLockThread2_testReleaseOrphanedGrant;
+ /**
+ * Client requests lock and then interrupts lock request before processing
+ * the grant reply. This causes the Client to send a release msg to the
+ * grantor.
+ */
+ public void testReleaseOrphanedGrant_Local() {
+ DLockRequestProcessor.setDebugReleaseOrphanedGrant(true);
+ DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+ try {
+ startedThread2_testReleaseOrphanedGrant = false;
+ gotLockThread2_testReleaseOrphanedGrant = false;
+ releaseThread1_testReleaseOrphanedGrant = false;
+
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] create lock service");
+ final String serviceName = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+
+ // thread to get lock and wait and then unlock
+ final Thread thread1 = new Thread(new Runnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] get the lock");
+ assertTrue(service.lock("obj", -1, -1));
+ DLockRequestProcessor.setWaitToProcessDLockResponse(true);
+ startedThread1_testReleaseOrphanedGrant = true;
+ synchronized(Thread.currentThread()) {
+ while (!releaseThread1_testReleaseOrphanedGrant) {
+ try {
+ Thread.currentThread().wait();
+ } catch (InterruptedException ignore) {fail("interrupted");}
+ }
+ }
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] unlock the lock");
+ service.unlock("obj");
+ }
+ });
+ thread1.start();
+ while (!startedThread1_testReleaseOrphanedGrant) {
+ Thread.yield();
+ }
+
+ // thread to interrupt lockInterruptibly call to cause zombie grant
+ final Thread thread2 = new Thread(new Runnable() {
+ public void run() {
+ try {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] call lockInterruptibly");
+ startedThread2_testReleaseOrphanedGrant = true;
+ assertFalse(service.lockInterruptibly("obj", -1, -1));
+ } catch (InterruptedException expected) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ });
+ thread2.start();
+ while (!startedThread2_testReleaseOrphanedGrant) {
+ Thread.yield();
+ }
+
+ // release first thread to unlock
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] release 1st thread");
+ sleep(500);
+ synchronized(thread1) {
+ releaseThread1_testReleaseOrphanedGrant = true;
+ thread1.notifyAll();
+ }
+ sleep(500);
+
+ // while first thread is stuck on waitToProcessDLockResponse,
+ // interrupt 2nd thread
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] interrupt 2nd thread");
+ thread2.interrupt();
+ ThreadUtils.join(thread2, 20 * 1000);
+
+ // release waitToProcessDLockResponse
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] process lock response");
+ sleep(500);
+ DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+
+ // relock obj to make sure zombie release worked
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] verify lock not held");
+ assertTrue(service.lock("obj", 1000, -1));
+ }
+ finally {
+ DLockRequestProcessor.setDebugReleaseOrphanedGrant(false);
+ DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+ }
+ }
+
+ static volatile Thread threadVM1_testReleaseOrphanedGrant_Remote;
+ static volatile Thread threadVM2_testReleaseOrphanedGrant_Remote;
+ static volatile boolean startedThreadVM1_testReleaseOrphanedGrant_Remote;
+ static volatile boolean releaseThreadVM1_testReleaseOrphanedGrant_Remote;
+ static volatile boolean unlockedThreadVM1_testReleaseOrphanedGrant_Remote;
+ static volatile boolean startedThreadVM2_testReleaseOrphanedGrant_Remote;
+ static volatile boolean gotLockThreadVM2_testReleaseOrphanedGrant_Remote;
+ public void testReleaseOrphanedGrant_Remote() {
+ doTestReleaseOrphanedGrant_Remote(false);
+ }
+ public void testReleaseOrphanedGrant_RemoteWithDestroy() {
+ doTestReleaseOrphanedGrant_Remote(true);
+ }
+ public void doTestReleaseOrphanedGrant_Remote(final boolean destroyLockService) {
+ final VM vm1 = Host.getHost(0).getVM(0);
+ final VM vm2 = Host.getHost(0).getVM(1);
+
+ try {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] create lock service");
+ final String serviceName = getUniqueName();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+
+ // lock and unlock to make sure this vm is grantor
+ assertTrue(service.lock("obj", -1, -1));
+ service.unlock("obj");
+
+ // thread to get lock and wait and then unlock
+ vm1.invokeAsync(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] get the lock");
+ threadVM1_testReleaseOrphanedGrant_Remote = Thread.currentThread();
+ connectDistributedSystem();
+ DistributedLockService service_vm1 =
+ DistributedLockService.create(serviceName, getSystem());
+ assertTrue(service_vm1.lock("obj", -1, -1));
+ synchronized(threadVM1_testReleaseOrphanedGrant_Remote) {
+ while (!releaseThreadVM1_testReleaseOrphanedGrant_Remote) {
+ try {
+ startedThreadVM1_testReleaseOrphanedGrant_Remote = true;
+ Thread.currentThread().wait();
+ } catch (InterruptedException ignore) {fail("interrupted");}
+ }
+ }
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] unlock the lock");
+ service_vm1.unlock("obj");
+ unlockedThreadVM1_testReleaseOrphanedGrant_Remote = true;
+ }
+ });
+ vm1.invoke(new SerializableRunnable() {
+ public void run() {
+ while (!startedThreadVM1_testReleaseOrphanedGrant_Remote) {
+ Thread.yield();
+ }
+ }
+ });
+ sleep(500);
+
+ // thread to interrupt lockInterruptibly call to cause zombie grant
+ vm2.invokeAsync(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] call lockInterruptibly");
+ threadVM2_testReleaseOrphanedGrant_Remote = Thread.currentThread();
+ DistributedLockService service_vm2 =
+ DistributedLockService.create(serviceName, getSystem());
+ startedThreadVM2_testReleaseOrphanedGrant_Remote = true;
+ try {
+ DLockRequestProcessor.setDebugReleaseOrphanedGrant(true);
+ DLockRequestProcessor.setWaitToProcessDLockResponse(true);
+ assertFalse(service_vm2.lockInterruptibly("obj", -1, -1));
+ } catch (InterruptedException expected) {Thread.currentThread().interrupt();}
+ }
+ });
+ vm2.invoke(new SerializableRunnable() {
+ public void run() {
+ while (!startedThreadVM2_testReleaseOrphanedGrant_Remote) {
+ Thread.yield();
+ }
+ }
+ });
+ sleep(500);
+
+ // release first thread to unlock
+ vm1.invoke(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] release 1st thread");
+ synchronized(threadVM1_testReleaseOrphanedGrant_Remote) {
+ releaseThreadVM1_testReleaseOrphanedGrant_Remote = true;
+ threadVM1_testReleaseOrphanedGrant_Remote.notifyAll();
+ }
+ }
+ });
+ sleep(500); // lock is being released, grantor will grant lock to vm2
+
+ // while first thread is stuck on waitToProcessDLockResponse,
+ // interrupt 2nd thread
+ vm2.invoke(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] interrupt 2nd thread");
+ threadVM2_testReleaseOrphanedGrant_Remote.interrupt();
+ ThreadUtils.join(threadVM2_testReleaseOrphanedGrant_Remote,
+ 5 * 60 * 1000);
+ if (destroyLockService) {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] destroy lock service");
+ DistributedLockService.destroy(serviceName);
+ assertNull(DistributedLockService.getServiceNamed(serviceName));
+ }
+ }
+ });
+ sleep(500); // grant is blocked while reply processor is being destroyed
+
+ // release waitToProcessDLockResponse
+ vm2.invoke(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] process lock response");
+ DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+ }
+ });
+ sleep(500); // process grant and send zombie release to grantor
+
+ // relock obj to make sure zombie release worked
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] verify lock not held");
+ assertTrue(service.lock("obj", 1000, -1));
+ }
+ finally {
+ vm2.invoke(new SerializableRunnable() {
+ public void run() {
+ LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] clean up DebugReleaseOrphanedGrant");
+ DLockRequestProcessor.setDebugReleaseOrphanedGrant(false);
+ DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+ }
+ });
+ }
+ }
+
+ public void testDestroyLockServiceAfterGrantResponse() throws Throwable {
+ Host host = Host.getHost(0);
+ VM vm0 = host.getVM(0);
+
+ final String serviceName = getUniqueName();
+
+ vm0.invoke(new SerializableRunnable("Create the grantor") {
+
+ public void run() {
+ connectDistributedSystem();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+
+ // lock and unlock to make sure this vm is grantor
+ assertTrue(service.lock("obj", -1, -1));
+ service.unlock("obj");
+ }
+ });
+
+ DistributionMessageObserver.setInstance(new DistributionMessageObserver() {
+
+ @Override
+ public void beforeProcessMessage(DistributionManager dm,
+ DistributionMessage message) {
+ if(message instanceof DLockResponseMessage) {
+ DistributedLockService.destroy(serviceName);
+ }
+ }
+ });
+
+ connectDistributedSystem();
+ final DistributedLockService service =
+ DistributedLockService.create(serviceName, dlstSystem);
+ try {
+ service.lock("obj", -1, -1);
+ fail("The lock service should have been destroyed");
+ } catch(LockServiceDestroyedException expected) {
+ //Do nothing
+ }
+
+ vm0.invoke(new SerializableRunnable("check to make sure the lock is not orphaned") {
+
+ public void run() {
+ final DistributedLockService service =
+ DistributedLockService.getServiceNamed(serviceName);
+
+ // lock and unlock to make sure this vm is grantor
+ assertTrue(service.lock("obj", -1, -1));
+ service.unlock("obj");
<TRUNCATED>