You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by al...@apache.org on 2015/07/29 17:35:31 UTC

[3/4] incubator-brooklyn git commit: Add tests for apps double stop

Add tests for apps double stop


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/20a52b2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/20a52b2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/20a52b2a

Branch: refs/heads/master
Commit: 20a52b2ab1911b9c85d79614569808edc0574f70
Parents: 79a1847
Author: Svetoslav Neykov <sv...@cloudsoftcorp.com>
Authored: Mon Jul 27 16:28:25 2015 +0300
Committer: Svetoslav Neykov <sv...@cloudsoftcorp.com>
Committed: Wed Jul 29 17:04:23 2015 +0300

----------------------------------------------------------------------
 .../entity/basic/SoftwareProcessEntityTest.java | 257 +++++++++++++++++--
 .../rest/resources/ServerResourceTest.java      |  76 ++++--
 .../rest/resources/ServerShutdownTest.java      | 187 ++++++++++++++
 .../rest/testing/BrooklynRestApiTest.java       |   4 +
 4 files changed, 474 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/20a52b2a/software/base/src/test/java/brooklyn/entity/basic/SoftwareProcessEntityTest.java
----------------------------------------------------------------------
diff --git a/software/base/src/test/java/brooklyn/entity/basic/SoftwareProcessEntityTest.java b/software/base/src/test/java/brooklyn/entity/basic/SoftwareProcessEntityTest.java
index 70bc416..b770293 100644
--- a/software/base/src/test/java/brooklyn/entity/basic/SoftwareProcessEntityTest.java
+++ b/software/base/src/test/java/brooklyn/entity/basic/SoftwareProcessEntityTest.java
@@ -18,6 +18,30 @@
  */
 package brooklyn.entity.basic;
 
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.jclouds.util.Throwables2;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
 import brooklyn.config.ConfigKey;
 import brooklyn.entity.BrooklynAppUnitTestSupport;
 import brooklyn.entity.Entity;
@@ -25,6 +49,8 @@ import brooklyn.entity.basic.SoftwareProcess.RestartSoftwareParameters;
 import brooklyn.entity.basic.SoftwareProcess.RestartSoftwareParameters.RestartMachineMode;
 import brooklyn.entity.basic.SoftwareProcess.StopSoftwareParameters;
 import brooklyn.entity.basic.SoftwareProcess.StopSoftwareParameters.StopMode;
+import brooklyn.entity.drivers.BasicEntityDriverManager;
+import brooklyn.entity.drivers.ReflectiveEntityDriverFactory;
 import brooklyn.entity.effector.Effectors;
 import brooklyn.entity.proxying.EntitySpec;
 import brooklyn.entity.proxying.ImplementedBy;
@@ -33,13 +59,20 @@ import brooklyn.entity.trait.Startable;
 import brooklyn.event.basic.PortAttributeSensorAndConfigKey;
 import brooklyn.location.Location;
 import brooklyn.location.LocationSpec;
+import brooklyn.location.MachineLocation;
 import brooklyn.location.basic.FixedListMachineProvisioningLocation;
 import brooklyn.location.basic.Locations;
+import brooklyn.location.basic.SimulatedLocation;
 import brooklyn.location.basic.SshMachineLocation;
+import brooklyn.management.EntityManager;
 import brooklyn.management.Task;
 import brooklyn.management.TaskAdaptable;
+import brooklyn.test.Asserts;
+import brooklyn.test.EntityTestUtils;
+import brooklyn.test.entity.TestApplication;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.config.ConfigBag;
+import brooklyn.util.exceptions.Exceptions;
 import brooklyn.util.exceptions.PropagatedRuntimeException;
 import brooklyn.util.net.UserAndHostAndPort;
 import brooklyn.util.os.Os;
@@ -48,28 +81,6 @@ import brooklyn.util.task.Tasks;
 import brooklyn.util.text.Strings;
 import brooklyn.util.time.Duration;
 
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-
-import org.jclouds.util.Throwables2;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
 
 public class SoftwareProcessEntityTest extends BrooklynAppUnitTestSupport {
 
@@ -413,6 +424,133 @@ public class SoftwareProcessEntityTest extends BrooklynAppUnitTestSupport {
     }
 
     @Test
+    public void testDoubleStopEntity() {
+        ReflectiveEntityDriverFactory f = ((BasicEntityDriverManager)mgmt.getEntityDriverManager()).getReflectiveDriverFactory();
+        f.addClassFullNameMapping(EmptySoftwareProcessDriver.class.getName(), MinimalEmptySoftwareProcessTestDriver.class.getName());
+
+        // Second stop on SoftwareProcess will return early, while the first stop is still in progress
+        // This causes the app to shutdown prematurely, leaking machines.
+        EntityManager emgr = mgmt.getEntityManager();
+        EntitySpec<TestApplication> appSpec = EntitySpec.create(TestApplication.class);
+        TestApplication app = emgr.createEntity(appSpec);
+        emgr.manage(app);
+        EntitySpec<?> latchEntitySpec = EntitySpec.create(EmptySoftwareProcess.class);
+        Entity entity = app.createAndManageChild(latchEntitySpec);
+
+        final ReleaseLatchLocation loc = mgmt.getLocationManager().createLocation(LocationSpec.create(ReleaseLatchLocation.class));
+        try {
+            app.start(ImmutableSet.of(loc));
+            EntityTestUtils.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
+    
+            final Task<Void> firstStop = entity.invoke(Startable.STOP, ImmutableMap.<String, Object>of());
+            // Wait until first task tries to release the location, at this point the entity's reference 
+            // to the location is already cleared.
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(loc.isBlocked());
+                }
+            });
+    
+            // Subsequent stops will end quickly - no location to release,
+            // while the first one is still releasing the machine.
+            final Task<Void> secondStop = entity.invoke(Startable.STOP, ImmutableMap.<String, Object>of());;
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(secondStop.isDone());
+                }
+            });
+    
+            // Entity state is STOPPED even though first location is still releasing
+            EntityTestUtils.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.STOPPED);
+            Asserts.succeedsContinually(new Runnable() {
+                @Override
+                public void run() {
+                    assertFalse(firstStop.isDone());
+                }
+            });
+
+            loc.unblock();
+
+            // After the location is released, first task ends as well.
+            EntityTestUtils.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.STOPPED);
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(firstStop.isDone());
+                }
+            });
+
+        } finally {
+            loc.unblock();
+        }
+
+    }
+
+    @Test
+    public void testDoubleStopApp() {
+        ReflectiveEntityDriverFactory f = ((BasicEntityDriverManager)mgmt.getEntityDriverManager()).getReflectiveDriverFactory();
+        f.addClassFullNameMapping(EmptySoftwareProcessDriver.class.getName(), MinimalEmptySoftwareProcessTestDriver.class.getName());
+
+        // Second stop on SoftwareProcess will return early, while the first stop is still in progress
+        // This causes the app to shutdown prematurely, leaking machines.
+        EntityManager emgr = mgmt.getEntityManager();
+        EntitySpec<TestApplication> appSpec = EntitySpec.create(TestApplication.class);
+        final TestApplication app = emgr.createEntity(appSpec);
+        emgr.manage(app);
+        EntitySpec<?> latchEntitySpec = EntitySpec.create(EmptySoftwareProcess.class);
+        final Entity entity = app.createAndManageChild(latchEntitySpec);
+
+        final ReleaseLatchLocation loc = mgmt.getLocationManager().createLocation(LocationSpec.create(ReleaseLatchLocation.class));
+        try {
+            app.start(ImmutableSet.of(loc));
+            EntityTestUtils.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
+    
+            final Task<Void> firstStop = app.invoke(Startable.STOP, ImmutableMap.<String, Object>of());
+            // Wait until first task tries to release the location, at this point the entity's reference 
+            // to the location is already cleared.
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(loc.isBlocked());
+                }
+            });
+    
+            // Subsequent stops will end quickly - no location to release,
+            // while the first one is still releasing the machine.
+            final Task<Void> secondStop = app.invoke(Startable.STOP, ImmutableMap.<String, Object>of());;
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(secondStop.isDone());
+                }
+            });
+    
+            // Since second stop succeeded the app will get unmanaged.
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(!Entities.isManaged(entity));
+                    assertTrue(!Entities.isManaged(app));
+                }
+            });
+    
+            // Unmanage will cancel the first task
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(firstStop.isDone());
+                }
+            });
+        } finally {
+            // We still haven't unblocked the location release, but entity is already unmanaged.
+            // Double STOP on an application could leak locations!!!
+            loc.unblock();
+        }
+    }
+
+    @Test
     public void testOpenPortsWithPortRangeConfig() throws Exception {
         MyService entity = app.createAndManageChild(EntitySpec.create(MyService.class)
             .configure("http.port", "9999+"));
@@ -568,4 +706,79 @@ public class SoftwareProcessEntityTest extends BrooklynAppUnitTestSupport {
             return (String)getEntity().getConfigRaw(ConfigKeys.newStringConfigKey("salt"), true).or((String)null);
         }
     }
+
+    public static class ReleaseLatchLocation extends SimulatedLocation {
+        private static final long serialVersionUID = 1L;
+        
+        private CountDownLatch lock = new CountDownLatch(1);
+        private volatile boolean isBlocked;
+
+        public void unblock() {
+            lock.countDown();
+        }
+        @Override
+        public void release(MachineLocation machine) {
+            super.release(machine);
+            try {
+                isBlocked = true;
+                lock.await();
+                isBlocked = false;
+            } catch (InterruptedException e) {
+                throw Exceptions.propagate(e);
+            }
+        }
+
+        public boolean isBlocked() {
+            return isBlocked;
+        }
+
+    }
+
+    public static class MinimalEmptySoftwareProcessTestDriver implements EmptySoftwareProcessDriver {
+
+        private EmptySoftwareProcessImpl entity;
+        private Location location;
+
+        public MinimalEmptySoftwareProcessTestDriver(EmptySoftwareProcessImpl entity, Location location) {
+            this.entity = entity;
+            this.location = location;
+        }
+
+        @Override
+        public Location getLocation() {
+            return location;
+        }
+
+        @Override
+        public EntityLocal getEntity() {
+            return entity;
+        }
+
+        @Override
+        public boolean isRunning() {
+            return true;
+        }
+
+        @Override
+        public void rebind() {
+        }
+
+        @Override
+        public void start() {
+        }
+
+        @Override
+        public void restart() {
+        }
+
+        @Override
+        public void stop() {
+        }
+
+        @Override
+        public void kill() {
+        }
+
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/20a52b2a/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerResourceTest.java
----------------------------------------------------------------------
diff --git a/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerResourceTest.java b/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerResourceTest.java
index cdfd501..ca5d860 100644
--- a/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerResourceTest.java
+++ b/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerResourceTest.java
@@ -23,6 +23,8 @@ import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertTrue;
 
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import javax.ws.rs.core.MultivaluedMap;
@@ -38,24 +40,23 @@ import com.sun.jersey.core.util.MultivaluedMapImpl;
 
 import brooklyn.BrooklynVersion;
 import brooklyn.config.BrooklynProperties;
+import brooklyn.entity.basic.EmptySoftwareProcess;
+import brooklyn.entity.basic.EmptySoftwareProcessDriver;
+import brooklyn.entity.basic.EmptySoftwareProcessImpl;
+import brooklyn.entity.proxying.ImplementedBy;
 import brooklyn.management.ManagementContext;
 import brooklyn.management.internal.ManagementContextInternal;
 import brooklyn.rest.domain.HighAvailabilitySummary;
 import brooklyn.rest.domain.VersionSummary;
 import brooklyn.rest.testing.BrooklynRestResourceTest;
 import brooklyn.test.Asserts;
+import brooklyn.util.exceptions.Exceptions;
 
 @Test(singleThreaded = true)
 public class ServerResourceTest extends BrooklynRestResourceTest {
 
     private static final Logger log = LoggerFactory.getLogger(ServerResourceTest.class);
     
-    @Override
-    @BeforeClass(alwaysRun = true)
-    public void setUp() throws Exception {
-        super.setUp();
-    }
-
     @Test
     public void testGetVersion() throws Exception {
         VersionSummary version = client().resource("/v1/server/version").get(VersionSummary.class);
@@ -98,28 +99,6 @@ public class ServerResourceTest extends BrooklynRestResourceTest {
         client().resource("/v1/server/properties/reload").post();
         assertEquals(reloadCount.get(), 1);
     }
-    
-    @Test
-    public void testShutdown() throws Exception {
-        assertTrue(getManagementContext().isRunning());
-        assertFalse(shutdownListener.isRequested());
-
-        MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
-        formData.add("requestTimeout", "0");
-        formData.add("delayForHttpReturn", "0");
-        client().resource("/v1/server/shutdown").entity(formData).post();
-        
-        Asserts.succeedsEventually(new Runnable() {
-            @Override
-            public void run() {
-                assertTrue(shutdownListener.isRequested());
-            }
-        });
-        Asserts.succeedsEventually(new Runnable() {
-            @Override public void run() {
-                assertFalse(getManagementContext().isRunning());
-            }});
-    }
 
     @Test
     void testGetConfig() throws Exception {
@@ -153,4 +132,45 @@ public class ServerResourceTest extends BrooklynRestResourceTest {
             }
         }
     }
+
+    // Alternatively could reuse a blocking location, see brooklyn.entity.basic.SoftwareProcessEntityTest.ReleaseLatchLocation
+    @ImplementedBy(StopLatchEntityImpl.class)
+    public interface StopLatchEntity extends EmptySoftwareProcess {
+        public void unblock();
+        public boolean isBlocked();
+    }
+
+    public static class StopLatchEntityImpl extends EmptySoftwareProcessImpl implements StopLatchEntity {
+        private CountDownLatch lock = new CountDownLatch(1);
+        private volatile boolean isBlocked;
+
+        @Override
+        public void unblock() {
+            lock.countDown();
+        }
+
+        @Override
+        protected void postStop() {
+            super.preStop();
+            try {
+                isBlocked = true;
+                lock.await();
+                isBlocked = false;
+            } catch (InterruptedException e) {
+                throw Exceptions.propagate(e);
+            }
+        }
+
+        @Override
+        public Class<?> getDriverInterface() {
+            return EmptySoftwareProcessDriver.class;
+        }
+
+        @Override
+        public boolean isBlocked() {
+            return isBlocked;
+        }
+
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/20a52b2a/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerShutdownTest.java
----------------------------------------------------------------------
diff --git a/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerShutdownTest.java b/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerShutdownTest.java
new file mode 100644
index 0000000..061599e
--- /dev/null
+++ b/usage/rest-server/src/test/java/brooklyn/rest/resources/ServerShutdownTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.rest.resources;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.sun.jersey.core.util.MultivaluedMapImpl;
+
+import brooklyn.entity.basic.Attributes;
+import brooklyn.entity.basic.Entities;
+import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.drivers.BasicEntityDriverManager;
+import brooklyn.entity.drivers.ReflectiveEntityDriverFactory;
+import brooklyn.entity.proxying.EntitySpec;
+import brooklyn.entity.trait.Startable;
+import brooklyn.management.EntityManager;
+import brooklyn.management.Task;
+import brooklyn.rest.resources.ServerResourceTest.StopLatchEntity;
+import brooklyn.rest.testing.BrooklynRestResourceTest;
+import brooklyn.test.Asserts;
+import brooklyn.test.EntityTestUtils;
+import brooklyn.test.entity.TestApplication;
+import brooklyn.util.exceptions.Exceptions;
+
+public class ServerShutdownTest extends BrooklynRestResourceTest {
+    private static final Logger log = LoggerFactory.getLogger(ServerResourceTest.class);
+
+    // Need to initialise the ManagementContext before each test as it is destroyed.
+    @Override
+    @BeforeClass(alwaysRun = true)
+    public void setUp() throws Exception {
+    }
+
+    @Override
+    @AfterClass(alwaysRun = true)
+    public void tearDown() throws Exception {
+    }
+
+    @Override
+    @BeforeMethod(alwaysRun = true)
+    public void setUpMethod() {
+        setUpJersey();
+        super.setUpMethod();
+    }
+
+    @AfterMethod(alwaysRun = true)
+    public void tearDownMethod() {
+        tearDownJersey();
+        destroyManagementContext();
+    }
+
+    @Test
+    public void testShutdown() throws Exception {
+        assertTrue(getManagementContext().isRunning());
+        assertFalse(shutdownListener.isRequested());
+
+        MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
+        formData.add("requestTimeout", "0");
+        formData.add("delayForHttpReturn", "0");
+        client().resource("/v1/server/shutdown").entity(formData).post();
+
+        Asserts.succeedsEventually(new Runnable() {
+            @Override
+            public void run() {
+                assertTrue(shutdownListener.isRequested());
+            }
+        });
+        Asserts.succeedsEventually(new Runnable() {
+            @Override public void run() {
+                assertFalse(getManagementContext().isRunning());
+            }});
+    }
+
+    @Test
+    public void testStopAppThenShutdownAndStopAppsWaitsForFirstStop() throws InterruptedException {
+        ReflectiveEntityDriverFactory f = ((BasicEntityDriverManager)getManagementContext().getEntityDriverManager()).getReflectiveDriverFactory();
+        f.addClassFullNameMapping("brooklyn.entity.basic.EmptySoftwareProcessDriver", "brooklyn.rest.resources.ServerResourceTest$EmptySoftwareProcessTestDriver");
+
+        // Second stop on SoftwareProcess could return early, while the first stop is still in progress
+        // This causes the app to shutdown prematurely, leaking machines.
+        EntityManager emgr = getManagementContext().getEntityManager();
+        EntitySpec<TestApplication> appSpec = EntitySpec.create(TestApplication.class);
+        TestApplication app = emgr.createEntity(appSpec);
+        emgr.manage(app);
+        EntitySpec<StopLatchEntity> latchEntitySpec = EntitySpec.create(StopLatchEntity.class);
+        final StopLatchEntity entity = app.createAndManageChild(latchEntitySpec);
+        app.start(ImmutableSet.of(app.newLocalhostProvisioningLocation()));
+        EntityTestUtils.assertAttributeEquals(entity, Attributes.SERVICE_STATE_ACTUAL, Lifecycle.RUNNING);
+
+        try {
+            final Task<Void> firstStop = app.invoke(Startable.STOP, ImmutableMap.<String, Object>of());
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(entity.isBlocked());
+                }
+            });
+
+            final AtomicReference<Exception> shutdownError = new AtomicReference<>();
+            // Can't use ExecutionContext as it will be stopped on shutdown
+            Thread shutdownThread = new Thread() {
+                @Override
+                public void run() {
+                    try {
+                        MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
+                        formData.add("stopAppsFirst", "true");
+                        formData.add("shutdownTimeout", "0");
+                        formData.add("requestTimeout", "0");
+                        formData.add("delayForHttpReturn", "0");
+                        client().resource("/v1/server/shutdown").entity(formData).post();
+                    } catch (Exception e) {
+                        log.error("Shutdown request error", e);
+                        shutdownError.set(e);
+                        throw Exceptions.propagate(e);
+                    }
+                }
+            };
+            shutdownThread.start();
+
+            //shutdown must wait until the first stop completes (or time out)
+            Asserts.succeedsContinually(new Runnable() {
+                @Override
+                public void run() {
+                    assertFalse(firstStop.isDone());
+                    assertEquals(getManagementContext().getApplications().size(), 1);
+                    assertFalse(shutdownListener.isRequested());
+                }
+            });
+
+            // NOTE test is not fully deterministic. Depending on thread scheduling this will
+            // execute before or after ServerResource.shutdown does the app stop loop. This
+            // means that the shutdown code might not see the app at all. In any case though
+            // the test must succeed.
+            entity.unblock();
+
+            Asserts.succeedsEventually(new Runnable() {
+                @Override
+                public void run() {
+                    assertTrue(firstStop.isDone());
+                    assertTrue(shutdownListener.isRequested());
+                    assertFalse(getManagementContext().isRunning());
+                }
+            });
+
+            shutdownThread.join();
+            assertNull(shutdownError.get(), "Shutdown request error, logged above");
+        } finally {
+            // Be sure we always unblock entity stop even in the case of an exception.
+            // In the success path the entity is already unblocked above.
+            entity.unblock();
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/20a52b2a/usage/rest-server/src/test/java/brooklyn/rest/testing/BrooklynRestApiTest.java
----------------------------------------------------------------------
diff --git a/usage/rest-server/src/test/java/brooklyn/rest/testing/BrooklynRestApiTest.java b/usage/rest-server/src/test/java/brooklyn/rest/testing/BrooklynRestApiTest.java
index 7526254..446eef0 100644
--- a/usage/rest-server/src/test/java/brooklyn/rest/testing/BrooklynRestApiTest.java
+++ b/usage/rest-server/src/test/java/brooklyn/rest/testing/BrooklynRestApiTest.java
@@ -93,6 +93,10 @@ public abstract class BrooklynRestApiTest {
     
     @AfterClass
     public void tearDown() throws Exception {
+        destroyManagementContext();
+    }
+
+    protected void destroyManagementContext() {
         if (manager!=null) {
             Entities.destroyAll(manager);
             manager = null;