You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2014/07/29 21:32:05 UTC

[10/31] git commit: effector invocations had been somewhat inconsistent in what failed -- the method call, the surrounding context -- this ensures that a programmatic call to invoke an effector will throw if it fails, that the effector task is added to a

effector invocations had been somewhat inconsistent in what failed -- the method call, the surrounding context -- this ensures that a programmatic call to invoke an effector will throw if it fails, that the effector task is added to any parent context, but as an inessential task so it does not fail the parent context.  new tests for this are in BasicEffectorTest.  the effect is particularly magnified when we attempt to add programmatically driven effector calls to the dynamic task context (so they show up in the gui).  these are now marked inessential, which i think is right because the user is interacting programmatically, and as the caller they would typically get on the result and handle errors themselves.

this largely preserves existing behaviour, apart from with the exception that effector method calls inside other effectors could previously fail without throwing, and now they throw.

this mainly affected dynamic cluster which in some places relied on failures being silently ignored, e.g. in the resize effector call.  this effector now reliably fails if the cluster does not resize.  previously it would depend whether it was in a task or not.  to preserve compatibility in that class as much as possible, some interface methods (non-effector) have had their signature changed.  it was not clear why these were on the interface, so i have deprecated them there also.


Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/1d8551ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/1d8551ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/1d8551ed

Branch: refs/heads/master
Commit: 1d8551ed67fd788bca29f49646de69ef45ca412d
Parents: 0394467
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Tue Jul 22 13:18:41 2014 -0700
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Tue Jul 29 10:42:03 2014 -0400

----------------------------------------------------------------------
 .../java/brooklyn/entity/basic/Entities.java    | 10 ++-
 .../brooklyn/entity/group/DynamicCluster.java   | 14 ++-
 .../entity/group/DynamicClusterImpl.java        | 44 +++++++---
 .../internal/AbstractManagementContext.java     |  7 +-
 .../internal/LocalManagementContext.java        |  6 +-
 .../java/brooklyn/util/task/ParallelTask.java   |  2 +-
 .../entity/effector/EffectorBasicTest.java      | 90 +++++++++++++++++++-
 .../entity/group/DynamicClusterTest.java        | 21 ++++-
 ...DynamicClusterWithAvailabilityZonesTest.java |  4 +-
 .../util/exceptions/ReferenceWithError.java     | 86 +++++++++++++++++++
 10 files changed, 258 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/entity/basic/Entities.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/basic/Entities.java b/core/src/main/java/brooklyn/entity/basic/Entities.java
index 2811890..3e8b225 100644
--- a/core/src/main/java/brooklyn/entity/basic/Entities.java
+++ b/core/src/main/java/brooklyn/entity/basic/Entities.java
@@ -80,6 +80,7 @@ import brooklyn.util.repeat.Repeater;
 import brooklyn.util.stream.Streams;
 import brooklyn.util.task.DynamicTasks;
 import brooklyn.util.task.ParallelTask;
+import brooklyn.util.task.TaskTags;
 import brooklyn.util.task.Tasks;
 import brooklyn.util.task.system.ProcessTaskWrapper;
 import brooklyn.util.task.system.SystemTasks;
@@ -161,7 +162,7 @@ public class Entities {
                         "description", "Invoking effector \""+effector.getName()+"\" on "+tasks.size()+(tasks.size() == 1 ? " entity" : " entities"),
                         "tag", BrooklynTaskTags.tagForCallerEntity(callingEntity)),
                 tasks);
-        
+        TaskTags.markInessential(invoke);
         return DynamicTasks.queueIfPossible(invoke).orSubmitAsync(callingEntity).asTask();
     }
     public static <T> Task<List<T>> invokeEffectorListWithMap(EntityLocal callingEntity, Iterable<? extends Entity> entitiesToCall,
@@ -183,11 +184,18 @@ public class Entities {
     public static <T> Task<T> invokeEffector(EntityLocal callingEntity, Entity entityToCall,
             final Effector<T> effector, final Map<String,?> parameters) {
         Task<T> t = Effectors.invocation(entityToCall, effector, parameters).asTask();
+        TaskTags.markInessential(t);
         
         // we pass to callingEntity for consistency above, but in exec-context it should be
         // re-dispatched to targetEntity
         ((EntityInternal)callingEntity).getManagementSupport().getExecutionContext().submit(
                 MutableMap.of("tag", BrooklynTaskTags.tagForCallerEntity(callingEntity)), t);
+        
+        if (DynamicTasks.getTaskQueuingContext()!=null) {
+            // include it as a child (in the gui), marked inessential, because the caller is invoking programmatically
+            DynamicTasks.queue(t);
+        }
+        
         return t;
     }
     @SuppressWarnings("unchecked")

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/entity/group/DynamicCluster.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/group/DynamicCluster.java b/core/src/main/java/brooklyn/entity/group/DynamicCluster.java
index 917752f..6fec83e 100644
--- a/core/src/main/java/brooklyn/entity/group/DynamicCluster.java
+++ b/core/src/main/java/brooklyn/entity/group/DynamicCluster.java
@@ -45,6 +45,7 @@ import brooklyn.event.basic.BasicAttributeSensor;
 import brooklyn.event.basic.BasicNotificationSensor;
 import brooklyn.event.basic.Sensors;
 import brooklyn.location.Location;
+import brooklyn.util.exceptions.ReferenceWithError;
 import brooklyn.util.flags.SetFromFlag;
 import brooklyn.util.time.Duration;
 
@@ -80,6 +81,7 @@ import com.google.common.reflect.TypeToken;
  */
 // TODO document use of advanced availability zone configuration and features
 @ImplementedBy(DynamicClusterImpl.class)
+@SuppressWarnings("serial")
 public interface DynamicCluster extends AbstractGroup, Cluster, MemberReplaceable {
 
     @Beta
@@ -121,6 +123,7 @@ public interface DynamicCluster extends AbstractGroup, Cluster, MemberReplaceabl
             "dynamiccluster.memberspec", "entity spec for creating new cluster members", null);
 
     /** @deprecated since 0.7.0; use {@link #MEMBER_SPEC} instead. */
+    @SuppressWarnings("rawtypes")
     @Deprecated
     @SetFromFlag("factory")
     ConfigKey<EntityFactory> FACTORY = ConfigKeys.newConfigKey(
@@ -131,6 +134,7 @@ public interface DynamicCluster extends AbstractGroup, Cluster, MemberReplaceabl
             new TypeToken<Function<Collection<Entity>, Entity>>() {},
             "dynamiccluster.removalstrategy", "strategy for deciding what to remove when down-sizing", null);
 
+    @SuppressWarnings("rawtypes")
     @SetFromFlag("customChildFlags")
     ConfigKey<Map> CUSTOM_CHILD_FLAGS = ConfigKeys.newConfigKey(
             Map.class, "dynamiccluster.customChildFlags", "Additional flags to be passed to children when they are being created", ImmutableMap.of());
@@ -176,13 +180,19 @@ public interface DynamicCluster extends AbstractGroup, Cluster, MemberReplaceabl
 
     /**
      * Adds a node to the cluster in a single {@link Location}
+     *
+     * @deprecated since 0.7.0 tricky having this on the interface as implementation details
+     * may change; for instance we are (22 Jul) changing the return type to be a ReferenceWithError 
      */
-    Optional<Entity> addInSingleLocation(Location loc, Map<?,?> extraFlags);
+    ReferenceWithError<Optional<Entity>> addInSingleLocation(Location loc, Map<?,?> extraFlags);
 
     /**
      * Adds a node to the cluster in each {@link Location}
+     * 
+     * @deprecated since 0.7.0 tricky having this on the interface as implementation details
+     * may change; for instance we are (22 Jul) changing the return type to be a ReferenceWithError 
      */
-    Collection<Entity> addInEachLocation(Iterable<Location> locs, Map<?,?> extraFlags);
+    ReferenceWithError<Collection<Entity>> addInEachLocation(Iterable<Location> locs, Map<?,?> extraFlags);
 
     void setRemovalStrategy(Function<Collection<Entity>, Entity> val);
 

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/entity/group/DynamicClusterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/group/DynamicClusterImpl.java b/core/src/main/java/brooklyn/entity/group/DynamicClusterImpl.java
index 410b01e..271d9e5 100644
--- a/core/src/main/java/brooklyn/entity/group/DynamicClusterImpl.java
+++ b/core/src/main/java/brooklyn/entity/group/DynamicClusterImpl.java
@@ -56,11 +56,13 @@ import brooklyn.policy.Policy;
 import brooklyn.util.collections.MutableList;
 import brooklyn.util.collections.MutableMap;
 import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.exceptions.ReferenceWithError;
 import brooklyn.util.flags.TypeCoercions;
 import brooklyn.util.guava.Maybe;
 import brooklyn.util.javalang.JavaClassNames;
 import brooklyn.util.javalang.Reflections;
 import brooklyn.util.task.DynamicTasks;
+import brooklyn.util.task.TaskTags;
 import brooklyn.util.task.Tasks;
 import brooklyn.util.text.StringPredicates;
 import brooklyn.util.text.Strings;
@@ -257,7 +259,12 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
             int initialSize = getConfig(INITIAL_SIZE).intValue();
             int initialQuorumSize = getInitialQuorumSize();
 
-            resize(initialSize);
+            try {
+                resize(initialSize);
+            } catch (Exception e) {
+                Exceptions.propagateIfFatal(e);
+                // ignore problems here; we extract them below
+            }
 
             Iterable<Task<?>> failed = Tasks.failed(Tasks.children(Tasks.current()));
             Iterator<Task<?>> fi = failed.iterator();
@@ -478,10 +485,12 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
      */
     protected Entity replaceMember(Entity member, Location memberLoc, Map<?, ?> extraFlags) {
         synchronized (mutex) {
-            Optional<Entity> added = addInSingleLocation(memberLoc, extraFlags);
+            ReferenceWithError<Optional<Entity>> added = addInSingleLocation(memberLoc, extraFlags);
 
-            if (!added.isPresent()) {
+            if (!added.getIgnoringError().isPresent()) {
                 String msg = String.format("In %s, failed to grow, to replace %s; not removing", this, member);
+                if (added.hasError())
+                    throw new IllegalStateException(msg, added.getError());
                 throw new IllegalStateException(msg);
             }
 
@@ -492,7 +501,7 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
                 throw new StopFailedRuntimeException("replaceMember failed to stop and remove old member "+member.getId(), e);
             }
 
-            return added.get();
+            return added.getOrThrowError().get();
         }
     }
 
@@ -575,7 +584,7 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
         }
 
         // create and start the entities
-        return addInEachLocation(chosenLocations, ImmutableMap.of());
+        return addInEachLocation(chosenLocations, ImmutableMap.of()).getOrThrowError();
     }
 
     /** <strong>Note</strong> for sub-clases; this method can be called while synchronized on {@link #mutex}. */
@@ -607,13 +616,15 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
     }
 
     @Override
-    public Optional<Entity> addInSingleLocation(Location location, Map<?,?> flags) {
-        Collection<Entity> added = addInEachLocation(ImmutableList.of(location), flags);
-        return Iterables.isEmpty(added) ? Optional.<Entity>absent() : Optional.of(Iterables.getOnlyElement(added));
+    public ReferenceWithError<Optional<Entity>> addInSingleLocation(Location location, Map<?,?> flags) {
+        ReferenceWithError<Collection<Entity>> added = addInEachLocation(ImmutableList.of(location), flags);
+        return ReferenceWithError.newInstanceWithInformativeError(
+            Iterables.isEmpty(added.getIgnoringError()) ? Optional.<Entity>absent() : Optional.of(Iterables.getOnlyElement(added.getIgnoringError())),
+                added.getError());
     }
 
     @Override
-    public Collection<Entity> addInEachLocation(Iterable<Location> locations, Map<?,?> flags) {
+    public ReferenceWithError<Collection<Entity>> addInEachLocation(Iterable<Location> locations, Map<?,?> flags) {
         List<Entity> addedEntities = Lists.newArrayList();
         Map<Entity, Location> addedEntityLocations = Maps.newLinkedHashMap();
         Map<Entity, Task<?>> tasks = Maps.newLinkedHashMap();
@@ -627,7 +638,9 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
             tasks.put(entity, task);
         }
 
-        DynamicTasks.queueIfPossible(Tasks.parallel("starting "+tasks.size()+" node"+Strings.s(tasks.size())+" (parallel)", tasks.values())).orSubmitAsync(this);
+        Task<List<?>> parallel = Tasks.parallel("starting "+tasks.size()+" node"+Strings.s(tasks.size())+" (parallel)", tasks.values());
+        TaskTags.markInessential(parallel);
+        DynamicTasks.queueIfPossible(parallel).orSubmitAsync(this);
         Map<Entity, Throwable> errors = waitForTasksOnEntityStart(tasks);
 
         // if tracking, then report success/fail to the ZoneFailureDetector
@@ -643,6 +656,11 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
                 }
             }
         }
+        
+        Collection<Entity> result = MutableList.<Entity> builder()
+            .addAll(addedEntities)
+            .removeAll(errors.keySet())
+            .build();
 
         // quarantine/cleanup as necessary
         if (!errors.isEmpty()) {
@@ -651,12 +669,10 @@ public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicClus
             } else {
                 cleanupFailedNodes(errors.keySet());
             }
+            return ReferenceWithError.newInstanceWithInformativeError(result, Exceptions.create(errors.values()));
         }
 
-        return MutableList.<Entity> builder()
-                .addAll(addedEntities)
-                .removeAll(errors.keySet())
-                .build();
+        return ReferenceWithError.newInstanceWithNoError(result);
     }
 
     protected void quarantineFailedNodes(Collection<Entity> failedEntities) {

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/management/internal/AbstractManagementContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/AbstractManagementContext.java b/core/src/main/java/brooklyn/management/internal/AbstractManagementContext.java
index 46e751d..0a3077a 100644
--- a/core/src/main/java/brooklyn/management/internal/AbstractManagementContext.java
+++ b/core/src/main/java/brooklyn/management/internal/AbstractManagementContext.java
@@ -290,12 +290,17 @@ public abstract class AbstractManagementContext implements ManagementContextInte
      * Returns the actual task (if it is local) or a proxy task (if it is remote);
      * if management for the entity has not yet started this may start it.
      * 
-     * @deprecated since 0.6.0 use effectors (or support {@code runAtEntity(Entity, Task)} if something else is needed);
+     * @deprecated since 0.6.0 use effectors (or support {@code runAtEntity(Entity, Effector, Map)} if something else is needed);
      * (Callable with Map flags is too open-ended, bothersome to support, and not used much) 
      */
     @Deprecated
     public abstract <T> Task<T> runAtEntity(@SuppressWarnings("rawtypes") Map flags, Entity entity, Callable<T> c);
 
+    /** Runs the given effector in the right place for the given entity.
+     * The task is immediately submitted in the background, but also recorded in the queueing context (if present)
+     * so it appears as a child, but marked inessential so it does not fail the parent task, who will ordinarily
+     * call {@link Task#get()} on the object and may do their own failure handling. 
+     */
     protected abstract <T> Task<T> runAtEntity(final Entity entity, final Effector<T> eff, @SuppressWarnings("rawtypes") final Map parameters);
 
     @Override

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/management/internal/LocalManagementContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/LocalManagementContext.java b/core/src/main/java/brooklyn/management/internal/LocalManagementContext.java
index 2ab2be6..136dba1 100644
--- a/core/src/main/java/brooklyn/management/internal/LocalManagementContext.java
+++ b/core/src/main/java/brooklyn/management/internal/LocalManagementContext.java
@@ -59,6 +59,7 @@ import brooklyn.util.guava.Maybe;
 import brooklyn.util.task.BasicExecutionContext;
 import brooklyn.util.task.BasicExecutionManager;
 import brooklyn.util.task.DynamicTasks;
+import brooklyn.util.task.TaskTags;
 import brooklyn.util.task.Tasks;
 import brooklyn.util.text.Strings;
 
@@ -335,7 +336,10 @@ public class LocalManagementContext extends AbstractManagementContext {
     protected <T> Task<T> runAtEntity(Entity entity, TaskAdaptable<T> task) {
         getExecutionContext(entity).submit(task);
         if (DynamicTasks.getTaskQueuingContext()!=null) {
-            // put it in the queueing context so it appears
+            // put it in the queueing context so it appears in the GUI
+            // mark it inessential as this is being invoked from code,
+            // the caller will do 'get' to handle errors
+            TaskTags.markInessential(task);
             DynamicTasks.getTaskQueuingContext().queue(task.asTask());
         }
         return task.asTask();

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/main/java/brooklyn/util/task/ParallelTask.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/task/ParallelTask.java b/core/src/main/java/brooklyn/util/task/ParallelTask.java
index eef3ed7..9828bbf 100644
--- a/core/src/main/java/brooklyn/util/task/ParallelTask.java
+++ b/core/src/main/java/brooklyn/util/task/ParallelTask.java
@@ -64,7 +64,7 @@ public class ParallelTask<T> extends CompoundTask<T> {
             } catch (Exception e) {
                 Exceptions.propagateIfFatal(e);
                 if (TaskTags.isInessential(task)) {
-                    // ignore exception is it's inessential
+                    // ignore exception as it's inessential
                 } else {
                     exceptions.add(e);
                 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/test/java/brooklyn/entity/effector/EffectorBasicTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/effector/EffectorBasicTest.java b/core/src/test/java/brooklyn/entity/effector/EffectorBasicTest.java
index e435445..66618f8 100644
--- a/core/src/test/java/brooklyn/entity/effector/EffectorBasicTest.java
+++ b/core/src/test/java/brooklyn/entity/effector/EffectorBasicTest.java
@@ -19,6 +19,7 @@
 package brooklyn.entity.effector;
 
 import java.util.List;
+import java.util.concurrent.Callable;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -30,13 +31,17 @@ import brooklyn.entity.BrooklynAppUnitTestSupport;
 import brooklyn.entity.basic.Entities;
 import brooklyn.entity.basic.EntityLocal;
 import brooklyn.entity.proxying.EntitySpec;
+import brooklyn.entity.trait.FailingEntity;
 import brooklyn.entity.trait.Startable;
 import brooklyn.location.basic.SimulatedLocation;
+import brooklyn.management.HasTaskChildren;
 import brooklyn.management.Task;
 import brooklyn.management.internal.ManagementContextInternal;
 import brooklyn.test.TestUtils;
 import brooklyn.test.entity.TestEntity;
 import brooklyn.util.collections.MutableMap;
+import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.task.Tasks;
 
 import com.google.common.collect.ImmutableList;
 
@@ -75,7 +80,6 @@ public class EffectorBasicTest extends BrooklynAppUnitTestSupport {
         TestUtils.assertSetsEqual(locs, app.getLocations());
     }
 
-
     @Test
     public void testInvokeEffectorStartWithTwoEntities() {
         TestEntity entity = app.createAndManageChild(EntitySpec.create(TestEntity.class));
@@ -92,5 +96,89 @@ public class EffectorBasicTest extends BrooklynAppUnitTestSupport {
 //        log.info("TAGS: "+starting.getTags());
         Assert.assertTrue(starting.getTags().contains(ManagementContextInternal.EFFECTOR_TAG));
     }
+
+    // check various failure situations
+    
+    private FailingEntity createFailingEntity() {
+        FailingEntity entity = app.createAndManageChild(EntitySpec.create(FailingEntity.class)
+            .configure(FailingEntity.FAIL_ON_START, true));
+        return entity;
+    }
+
+    // uncaught failures are propagates
+    
+    @Test
+    public void testInvokeEffectorStartFailing_Method() {
+        FailingEntity entity = createFailingEntity();
+        assertStartMethodFails(entity);
+    }
+
+    @Test
+    public void testInvokeEffectorStartFailing_EntityInvoke() {
+        FailingEntity entity = createFailingEntity();
+        assertTaskFails( entity.invoke(Startable.START, MutableMap.of("locations", locs)) );
+    }
+     
+    @Test
+    public void testInvokeEffectorStartFailing_EntitiesInvoke() {
+        FailingEntity entity = createFailingEntity();
+        
+        assertTaskFails( Entities.invokeEffectorWithArgs(entity, entity, Startable.START, locs) );
+    }
+
+    // caught failures are NOT propagated!
+    
+    @Test
+    public void testInvokeEffectorStartFailing_MethodInDynamicTask() {
+        Task<Void> task = app.getExecutionContext().submit(Tasks.<Void>builder().dynamic(true).body(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                testInvokeEffectorStartFailing_Method();
+                return null;
+            }
+        }).build());
+        
+        assertTaskSucceeds(task);
+        assertTaskHasFailedChild(task);
+    }
+
+    @Test
+    public void testInvokeEffectorStartFailing_MethodInTask() {
+        Task<Void> task = app.getExecutionContext().submit(Tasks.<Void>builder().dynamic(false).body(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                testInvokeEffectorStartFailing_Method();
+                return null;
+            }
+        }).build());
+        
+        assertTaskSucceeds(task);
+    }
+
+    private void assertTaskSucceeds(Task<Void> task) {
+        task.getUnchecked();
+        Assert.assertFalse(task.isError());
+    }
+
+    private void assertTaskHasFailedChild(Task<Void> task) {
+        Assert.assertTrue(Tasks.failed( ((HasTaskChildren)task).getChildren() ).iterator().hasNext());
+    }
+        
+    private void assertStartMethodFails(FailingEntity entity) {
+        try {
+            entity.start(locs);
+            Assert.fail("Should have failed");
+        } catch (Exception e) {
+            // expected
+        }
+    }
+     
+    protected void assertTaskFails(Task<?> t) {
+        try {
+            t.get();
+            Assert.fail("Should have failed");
+        } catch (Exception e) {
+            Exceptions.propagateIfFatal(e);
+            // expected
+        }
+    }
     
 }

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/test/java/brooklyn/entity/group/DynamicClusterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/group/DynamicClusterTest.java b/core/src/test/java/brooklyn/entity/group/DynamicClusterTest.java
index ebdd5fe..298d6d2 100644
--- a/core/src/test/java/brooklyn/entity/group/DynamicClusterTest.java
+++ b/core/src/test/java/brooklyn/entity/group/DynamicClusterTest.java
@@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
+import org.testng.Assert;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
@@ -349,7 +350,7 @@ public class DynamicClusterTest extends BrooklynAppUnitTestSupport {
                     }}));
 
         cluster.start(ImmutableList.of(loc));
-        cluster.resize(3);
+        resizeExpectingError(cluster, 3);
         assertEquals(cluster.getCurrentSize(), (Integer)2);
         assertEquals(cluster.getMembers().size(), 2);
         for (Entity member : cluster.getMembers()) {
@@ -357,6 +358,20 @@ public class DynamicClusterTest extends BrooklynAppUnitTestSupport {
         }
     }
 
+    static Exception resizeExpectingError(DynamicCluster cluster, int size) {
+        try {
+            cluster.resize(size);
+            Assert.fail("Resize should have failed");
+            // unreachable:
+            return null;
+        } catch (Exception e) {
+            Exceptions.propagateIfFatal(e);
+            // expect: brooklyn.util.exceptions.PropagatedRuntimeException: Error invoking resize at DynamicClusterImpl{id=I9Ggxfc1}: 1 of 3 parallel child tasks failed: Simulating entity stop failure for test
+            Assert.assertTrue(e.toString().contains("resize"));
+            return e;
+        }
+    }
+
     @Test
     public void testInitialQuorumSizeSufficientForStartup() throws Exception {
         final int failNum = 1;
@@ -447,7 +462,7 @@ public class DynamicClusterTest extends BrooklynAppUnitTestSupport {
                     }}));
 
         cluster.start(ImmutableList.of(loc));
-        cluster.resize(3);
+        resizeExpectingError(cluster, 3);
         assertEquals(cluster.getCurrentSize(), (Integer)2);
         assertEquals(cluster.getMembers().size(), 2);
         assertEquals(Iterables.size(Iterables.filter(cluster.getChildren(), Predicates.instanceOf(FailingEntity.class))), 3);
@@ -484,7 +499,7 @@ public class DynamicClusterTest extends BrooklynAppUnitTestSupport {
         assertEquals(cluster.getChildren().size(), 0, "children="+cluster.getChildren());
         
         // Failed node will not be a member or child
-        cluster.resize(3);
+        resizeExpectingError(cluster, 3);
         assertEquals(cluster.getCurrentSize(), (Integer)2);
         assertEquals(cluster.getMembers().size(), 2);
         assertEquals(cluster.getChildren().size(), 2, "children="+cluster.getChildren());

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/core/src/test/java/brooklyn/entity/group/DynamicClusterWithAvailabilityZonesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/brooklyn/entity/group/DynamicClusterWithAvailabilityZonesTest.java b/core/src/test/java/brooklyn/entity/group/DynamicClusterWithAvailabilityZonesTest.java
index 569f99f..6574400 100644
--- a/core/src/test/java/brooklyn/entity/group/DynamicClusterWithAvailabilityZonesTest.java
+++ b/core/src/test/java/brooklyn/entity/group/DynamicClusterWithAvailabilityZonesTest.java
@@ -154,9 +154,9 @@ public class DynamicClusterWithAvailabilityZonesTest extends BrooklynAppUnitTest
         String otherLoc = (locUsed.equals("zone1") ? "zone2" : "zone1");
         
         // This entity will fail; configured to give up on that zone after just two failure
-        cluster.resize(2);
+        DynamicClusterTest.resizeExpectingError(cluster, 2);
         assertEquals(cluster.getCurrentSize(), (Integer)1);
-        cluster.resize(2);
+        DynamicClusterTest.resizeExpectingError(cluster, 2);
         assertEquals(cluster.getCurrentSize(), (Integer)1);
         
         cluster.resize(3);

http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/1d8551ed/utils/common/src/main/java/brooklyn/util/exceptions/ReferenceWithError.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/exceptions/ReferenceWithError.java b/utils/common/src/main/java/brooklyn/util/exceptions/ReferenceWithError.java
new file mode 100644
index 0000000..7de6fb5
--- /dev/null
+++ b/utils/common/src/main/java/brooklyn/util/exceptions/ReferenceWithError.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.util.exceptions;
+
+import javax.annotation.Nullable;
+
+import com.google.common.base.Supplier;
+
+/** A reference to an object which can carry an object alongside it. */
+public class ReferenceWithError<T> implements Supplier<T> {
+
+    private final T object;
+    private final Throwable error;
+    private final boolean throwErrorOnAccess;
+
+    /** returns a reference which includes an error, and where attempts to get the content cause the error to throw */
+    public static <T> ReferenceWithError<T> newInstanceWithFatalError(T object, Throwable error) {
+        return new ReferenceWithError<T>(object, error, true);
+    }
+    
+    /** returns a reference which includes an error, but attempts to get the content do not cause the error to throw */
+    public static <T> ReferenceWithError<T> newInstanceWithInformativeError(T object, Throwable error) {
+        return new ReferenceWithError<T>(object, error, false);
+    }
+    
+    /** returns a reference which includes an error, but attempts to get the content do not cause the error to throw */
+    public static <T> ReferenceWithError<T> newInstanceWithNoError(T object) {
+        return new ReferenceWithError<T>(object, null, false);
+    }
+    
+    protected ReferenceWithError(@Nullable T object, @Nullable Throwable error, boolean throwErrorOnAccess) {
+        this.object = object;
+        this.error = error;
+        this.throwErrorOnAccess = throwErrorOnAccess;
+    }
+
+    public boolean throwsErrorOnAccess() {
+        return throwErrorOnAccess;
+    }
+
+    public T get() {
+        if (throwsErrorOnAccess()) {
+            return getOrThrowError();
+        }
+        return getIgnoringError();
+    }
+
+    public T getIgnoringError() {
+        return object;
+    }
+
+    public T getOrThrowError() {
+        checkNoError();
+        return object;
+    }
+
+    public void checkNoError() {
+        if (hasError())
+            Exceptions.propagate(error);
+    }
+    
+    public Throwable getError() {
+        return error;
+    }
+    
+    public boolean hasError() {
+        return error!=null;
+    }
+    
+}