You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2014/09/19 00:22:59 UTC
git commit: Save instance update events when updating a job.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 99ad092df -> 53e5cd2d0
Save instance update events when updating a job.
Bugs closed: AURORA-613
Reviewed at https://reviews.apache.org/r/25791/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/53e5cd2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/53e5cd2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/53e5cd2d
Branch: refs/heads/master
Commit: 53e5cd2d03219db56758b1745149b30d4359c4c1
Parents: 99ad092
Author: Bill Farner <wf...@apache.org>
Authored: Thu Sep 18 15:20:37 2014 -0700
Committer: Bill Farner <wf...@apache.org>
Committed: Thu Sep 18 15:20:37 2014 -0700
----------------------------------------------------------------------
.../aurora/scheduler/http/RequestLogger.java | 6 +-
.../scheduler/storage/db/DbAttributeStore.java | 5 +-
.../updater/JobUpdateControllerImpl.java | 46 ++++-
.../aurora/scheduler/updater/SideEffect.java | 15 +-
.../aurora/scheduler/updater/JobUpdaterIT.java | 180 +++++++++++++++----
5 files changed, 209 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
index 0442191..916939d 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
@@ -27,7 +27,7 @@ import org.eclipse.jetty.server.Response;
import org.eclipse.jetty.util.DateCache;
import org.eclipse.jetty.util.component.AbstractLifeCycle;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
/**
* A copy of twitter RequestLogger from twitter commons, which is a port of jetty's NCSARequestLog
@@ -63,8 +63,8 @@ public class RequestLogger extends AbstractLifeCycle implements RequestLog {
@VisibleForTesting
RequestLogger(Clock clock, LogSink sink) {
- this.clock = checkNotNull(clock);
- this.sink = checkNotNull(sink);
+ this.clock = requireNonNull(clock);
+ this.sink = requireNonNull(sink);
logDateCache = new DateCache("dd/MMM/yyyy:HH:mm:ss Z", Locale.getDefault());
logDateCache.setTimeZoneID("GMT");
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java b/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
index 34f12a9..1b18ad5 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/db/DbAttributeStore.java
@@ -13,6 +13,7 @@
*/
package org.apache.aurora.scheduler.storage.db;
+import java.util.Objects;
import java.util.Set;
import com.google.common.base.Optional;
@@ -28,8 +29,6 @@ import org.apache.aurora.scheduler.storage.AttributeStore;
import org.apache.aurora.scheduler.storage.entities.IAttribute;
import org.apache.aurora.scheduler.storage.entities.IHostAttributes;
-import static com.google.common.base.Preconditions.checkNotNull;
-
/**
* Attribute store backed by a relational database.
*/
@@ -39,7 +38,7 @@ class DbAttributeStore implements AttributeStore.Mutable {
@Inject
DbAttributeStore(AttributeMapper mapper) {
- this.mapper = checkNotNull(mapper);
+ this.mapper = Objects.requireNonNull(mapper);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java b/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
index 5375ed2..3e8182d 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/JobUpdateControllerImpl.java
@@ -27,10 +27,13 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
+import com.twitter.common.collections.Pair;
import com.twitter.common.quantity.Amount;
import com.twitter.common.quantity.Time;
import com.twitter.common.util.Clock;
+import org.apache.aurora.gen.JobInstanceUpdateEvent;
+import org.apache.aurora.gen.JobUpdateAction;
import org.apache.aurora.gen.JobUpdateEvent;
import org.apache.aurora.gen.JobUpdateQuery;
import org.apache.aurora.gen.JobUpdateStatus;
@@ -46,6 +49,7 @@ import org.apache.aurora.scheduler.storage.JobUpdateStore;
import org.apache.aurora.scheduler.storage.Storage;
import org.apache.aurora.scheduler.storage.TaskStore;
import org.apache.aurora.scheduler.storage.entities.IInstanceKey;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
@@ -77,6 +81,7 @@ import static org.apache.aurora.scheduler.updater.JobUpdateStateMachine.assertTr
import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.EvaluationResult;
import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.OneWayStatus;
import static org.apache.aurora.scheduler.updater.OneWayJobUpdater.OneWayStatus.SUCCEEDED;
+import static org.apache.aurora.scheduler.updater.SideEffect.InstanceUpdateStatus;
/**
* Implementation of an updater that orchestrates the process of gradually updating the
@@ -434,6 +439,21 @@ class JobUpdateControllerImpl implements JobUpdateController {
EvaluationResult<Integer> result = update.getUpdater().evaluate(changedInstance, stateProvider);
LOG.info(JobKeys.canonicalString(job) + " evaluation result: " + result);
+
+ for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
+ for (InstanceUpdateStatus statusChange : entry.getValue().getStatusChanges()) {
+ JobUpdateAction action = STATE_MAP.get(Pair.of(statusChange, updaterStatus));
+ requireNonNull(action);
+
+ IJobInstanceUpdateEvent event = IJobInstanceUpdateEvent.build(
+ new JobInstanceUpdateEvent()
+ .setInstanceId(entry.getKey())
+ .setTimestampMs(clock.nowMillis())
+ .setAction(action));
+ updateStore.saveJobInstanceUpdateEvent(event, summary.getUpdateId());
+ }
+ }
+
OneWayStatus status = result.getStatus();
if (status == SUCCEEDED || status == OneWayStatus.FAILED) {
if (SideEffect.hasActions(result.getSideEffects().values())) {
@@ -450,7 +470,6 @@ class JobUpdateControllerImpl implements JobUpdateController {
LOG.info("Executing side-effects for update of " + job + ": " + result.getSideEffects());
for (Map.Entry<Integer, SideEffect> entry : result.getSideEffects().entrySet()) {
IInstanceKey instance = InstanceKeys.from(job, entry.getKey());
- // TODO(wfarner): Persist SideEffect.getStatusChanges as JobInstanceUpdateEvents.
Optional<InstanceAction> action = entry.getValue().getAction();
if (action.isPresent()) {
@@ -472,6 +491,31 @@ class JobUpdateControllerImpl implements JobUpdateController {
}
}
+ /**
+ * Associates an instance updater state change and the job's update status to an action.
+ */
+ private static final Map<Pair<InstanceUpdateStatus, JobUpdateStatus>, JobUpdateAction> STATE_MAP =
+ ImmutableMap.<Pair<InstanceUpdateStatus, JobUpdateStatus>, JobUpdateAction>builder()
+ .put(
+ Pair.of(InstanceUpdateStatus.WORKING, ROLLING_FORWARD),
+ JobUpdateAction.INSTANCE_UPDATING)
+ .put(
+ Pair.of(InstanceUpdateStatus.SUCCEEDED, ROLLING_FORWARD),
+ JobUpdateAction.INSTANCE_UPDATED)
+ .put(
+ Pair.of(InstanceUpdateStatus.FAILED, ROLLING_FORWARD),
+ JobUpdateAction.INSTANCE_UPDATE_FAILED)
+ .put(
+ Pair.of(InstanceUpdateStatus.WORKING, ROLLING_BACK),
+ JobUpdateAction.INSTANCE_ROLLING_BACK)
+ .put(
+ Pair.of(InstanceUpdateStatus.SUCCEEDED, ROLLING_BACK),
+ JobUpdateAction.INSTANCE_ROLLED_BACK)
+ .put(
+ Pair.of(InstanceUpdateStatus.FAILED, ROLLING_BACK),
+ JobUpdateAction.INSTANCE_ROLLBACK_FAILED)
+ .build();
+
private Runnable getDeferredEvaluator(final IInstanceKey instance, final String updateId) {
return new Runnable() {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
index d9e59b6..27e0654 100644
--- a/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
+++ b/src/main/java/org/apache/aurora/scheduler/updater/SideEffect.java
@@ -34,18 +34,25 @@ public class SideEffect {
* @param action Action to be taken on the instance, if necessary.
* @param statusChanges Any status changes to the instance monitor.
*/
- public SideEffect(
- Optional<InstanceAction> action,
- Set<InstanceUpdateStatus> statusChanges) {
-
+ public SideEffect(Optional<InstanceAction> action, Set<InstanceUpdateStatus> statusChanges) {
this.action = requireNonNull(action);
this.statusChanges = requireNonNull(statusChanges);
}
+ /**
+ * Gets the action that should be performed with this side-effect, if any.
+ *
+ * @return The action associated with this side-effect.
+ */
public Optional<InstanceAction> getAction() {
return action;
}
+ /**
+ * Gets the status changes that the instance underwent while being evaluated.
+ *
+ * @return Instance updater status changes.
+ */
public Set<InstanceUpdateStatus> getStatusChanges() {
return statusChanges;
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/53e5cd2d/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
index 061d7e7..d90c87c 100644
--- a/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
+++ b/src/test/java/org/apache/aurora/scheduler/updater/JobUpdaterIT.java
@@ -16,13 +16,18 @@ package org.apache.aurora.scheduler.updater;
import java.util.Map;
import java.util.concurrent.ScheduledExecutorService;
+import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.Ordering;
import com.google.common.eventbus.EventBus;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
@@ -40,6 +45,7 @@ import org.apache.aurora.gen.ExecutorConfig;
import org.apache.aurora.gen.Identity;
import org.apache.aurora.gen.InstanceTaskConfig;
import org.apache.aurora.gen.JobUpdate;
+import org.apache.aurora.gen.JobUpdateAction;
import org.apache.aurora.gen.JobUpdateEvent;
import org.apache.aurora.gen.JobUpdateInstructions;
import org.apache.aurora.gen.JobUpdateSettings;
@@ -72,8 +78,10 @@ import org.apache.aurora.scheduler.storage.Storage.StoreProvider;
import org.apache.aurora.scheduler.storage.Storage.Work;
import org.apache.aurora.scheduler.storage.db.DbModule;
import org.apache.aurora.scheduler.storage.entities.IInstanceTaskConfig;
+import org.apache.aurora.scheduler.storage.entities.IJobInstanceUpdateEvent;
import org.apache.aurora.scheduler.storage.entities.IJobKey;
import org.apache.aurora.scheduler.storage.entities.IJobUpdate;
+import org.apache.aurora.scheduler.storage.entities.IJobUpdateDetails;
import org.apache.aurora.scheduler.storage.entities.IJobUpdateEvent;
import org.apache.aurora.scheduler.storage.entities.IJobUpdateSummary;
import org.apache.aurora.scheduler.storage.entities.ILock;
@@ -88,6 +96,12 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLBACK_FAILED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLED_BACK;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_ROLLING_BACK;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATE_FAILED;
+import static org.apache.aurora.gen.JobUpdateAction.INSTANCE_UPDATING;
import static org.apache.aurora.gen.JobUpdateStatus.ABORTED;
import static org.apache.aurora.gen.JobUpdateStatus.ERROR;
import static org.apache.aurora.gen.JobUpdateStatus.ROLLED_BACK;
@@ -103,7 +117,6 @@ import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
import static org.apache.aurora.gen.ScheduleStatus.STARTING;
import static org.apache.aurora.scheduler.events.PubsubEvent.SchedulerActive;
import static org.apache.aurora.scheduler.storage.Storage.MutateWork.NoResult;
-import static org.apache.aurora.scheduler.updater.JobUpdateControllerImpl.queryByUpdateId;
import static org.apache.aurora.scheduler.updater.UpdateFactory.UpdateFactoryImpl.expandInstanceIds;
import static org.easymock.EasyMock.expectLastCall;
import static org.junit.Assert.assertEquals;
@@ -220,16 +233,46 @@ public class JobUpdaterIT extends EasyMockTest {
}
}
- private void assertUpdateStatus(JobUpdateStatus expected) {
- JobUpdateStatus status = storage.consistentRead(new Work.Quiet<JobUpdateStatus>() {
+ private static final Ordering<IJobInstanceUpdateEvent> EVENT_ORDER = Ordering.natural()
+ .onResultOf(new Function<IJobInstanceUpdateEvent, Long>() {
+ @Override
+ public Long apply(IJobInstanceUpdateEvent event) {
+ return event.getTimestampMs();
+ }
+ });
+ private static final Function<IJobInstanceUpdateEvent, Integer> EVENT_TO_INSTANCE =
+ new Function<IJobInstanceUpdateEvent, Integer>() {
+ @Override
+ public Integer apply(IJobInstanceUpdateEvent event) {
+ return event.getInstanceId();
+ }
+ };
+ private static final Function<IJobInstanceUpdateEvent, JobUpdateAction> EVENT_TO_ACTION =
+ new Function<IJobInstanceUpdateEvent, JobUpdateAction>() {
+ @Override
+ public JobUpdateAction apply(IJobInstanceUpdateEvent event) {
+ return event.getAction();
+ }
+ };
+
+ private void assertState(
+ JobUpdateStatus expected,
+ Multimap<Integer, JobUpdateAction> expectedActions) {
+
+ IJobUpdateDetails details = storage.consistentRead(new Work.Quiet<IJobUpdateDetails>() {
@Override
- public JobUpdateStatus apply(StoreProvider storeProvider) {
- IJobUpdateSummary summary = Iterables.getOnlyElement(
- storeProvider.getJobUpdateStore().fetchJobUpdateSummaries(queryByUpdateId(UPDATE_ID)));
- return summary.getState().getStatus();
+ public IJobUpdateDetails apply(StoreProvider storeProvider) {
+ return storeProvider.getJobUpdateStore().fetchJobUpdateDetails(UPDATE_ID).get();
}
});
- assertEquals(expected, status);
+ Iterable<IJobInstanceUpdateEvent> orderedEvents =
+ EVENT_ORDER.sortedCopy(details.getInstanceEvents());
+ Multimap<Integer, IJobInstanceUpdateEvent> eventsByInstance =
+ Multimaps.index(orderedEvents, EVENT_TO_INSTANCE);
+ Multimap<Integer, JobUpdateAction> actionsByInstance =
+ Multimaps.transformValues(eventsByInstance, EVENT_TO_ACTION);
+ assertEquals(expectedActions, actionsByInstance);
+ assertEquals(expected, details.getUpdate().getSummary().getState().getStatus());
}
private IExpectationSetters<String> expectTaskKilled() {
@@ -271,22 +314,31 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
// Instance 1 is added
updater.start(update, USER);
- assertUpdateStatus(ROLLING_FORWARD);
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
// Updates may be paused for arbitrarily-long amounts of time, and the updater should not
// take action while paused.
updater.pause(JOB);
updater.pause(JOB); // Pausing again is a no-op.
- assertUpdateStatus(ROLL_FORWARD_PAUSED);
+ assertState(ROLL_FORWARD_PAUSED, actions.build());
clock.advance(ONE_DAY);
changeState(JOB, 1, FAILED, ASSIGNED, STARTING, RUNNING);
changeState(JOB, 2, FAILED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
updater.resume(JOB);
- assertUpdateStatus(ROLLING_FORWARD);
+ // TODO(wfarner): Since the updater does not skip no-op work upfront, we end up with redundant
+ // states.
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .put(2, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
// A task outside the scope of the update should be ignored by the updater.
stateManager.insertPendingTasks(NEW_CONFIG, ImmutableSet.of(100));
@@ -295,7 +347,8 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 2, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ROLLED_FORWARD);
+ actions.put(2, INSTANCE_UPDATED);
+ assertState(ROLLED_FORWARD, actions.build());
assertJobState(
JOB,
ImmutableMap.of(0, NEW_CONFIG, 1, NEW_CONFIG, 2, NEW_CONFIG, 100, NEW_CONFIG));
@@ -323,7 +376,10 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ROLLED_FORWARD);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ assertState(
+ ROLLED_FORWARD,
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED).build());
assertJobState(
JOB,
ImmutableMap.of(0, NEW_CONFIG, 1, OLD_CONFIG));
@@ -345,42 +401,58 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 3, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
// Instance 0 is updated.
updater.start(update, USER);
- assertUpdateStatus(ROLLING_FORWARD);
+ actions.putAll(0, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
// Instance 1 is added.
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
+ actions.putAll(0, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED);
clock.advance(WATCH_TIMEOUT);
// Instance 2 is updated, but fails.
changeState(JOB, 2, KILLED, ASSIGNED, STARTING, RUNNING);
+ // TODO(wfarner): Since the updater does not skip no-ops upfront, we don't touch instance 3,
+ // but record state transitions anyway.
+ actions.putAll(2, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK)
+ .putAll(3, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
clock.advance(FLAPPING_THRESHOLD);
changeState(JOB, 2, FAILED);
// Instance 2 is rolled back.
- assertUpdateStatus(ROLLING_BACK);
+ assertState(ROLLING_BACK, actions.build());
changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
+ actions.putAll(1, INSTANCE_ROLLING_BACK)
+ .putAll(2, INSTANCE_ROLLED_BACK);
clock.advance(WATCH_TIMEOUT);
// A rollback may be paused.
updater.pause(JOB);
- assertUpdateStatus(ROLL_BACK_PAUSED);
+ assertState(ROLL_BACK_PAUSED, actions.build());
clock.advance(ONE_DAY);
updater.resume(JOB);
- assertUpdateStatus(ROLLING_BACK);
+ actions.putAll(1, INSTANCE_ROLLING_BACK)
+ .putAll(2, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK)
+ .putAll(3, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
+ assertState(ROLLING_BACK, actions.build());
// Instance 1 is removed.
changeState(JOB, 1, KILLED);
+ actions.putAll(1, INSTANCE_ROLLED_BACK);
clock.advance(WATCH_TIMEOUT);
// Instance 0 is rolled back.
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
+ actions.putAll(0, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ROLLED_BACK);
+ assertState(ROLLED_BACK, actions.build());
assertJobState(JOB, ImmutableMap.of(0, OLD_CONFIG, 2, OLD_CONFIG, 3, OLD_CONFIG));
}
@@ -402,8 +474,12 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING);
+
updater.abort(JOB);
- assertUpdateStatus(ABORTED);
+ assertState(ABORTED, actions.build());
clock.advance(WATCH_TIMEOUT);
assertJobState(JOB, ImmutableMap.of(0, NEW_CONFIG, 1, NEW_CONFIG, 2, OLD_CONFIG));
}
@@ -422,9 +498,12 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
// Instance 0 is updated.
updater.start(update, USER);
- assertUpdateStatus(ROLLING_FORWARD);
+ actions.putAll(0, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
@@ -434,12 +513,16 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 1, FAILED);
// Instance 1 is rolled back, but fails.
- assertUpdateStatus(ROLLING_BACK);
+ actions.putAll(0, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK)
+ .putAll(2, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK);
+ assertState(ROLLING_BACK, actions.build());
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
clock.advance(FLAPPING_THRESHOLD);
changeState(JOB, 1, FAILED);
- assertUpdateStatus(JobUpdateStatus.FAILED);
+ actions.putAll(1, INSTANCE_ROLLBACK_FAILED);
+ assertState(JobUpdateStatus.FAILED, actions.build());
clock.advance(WATCH_TIMEOUT);
assertJobState(JOB, ImmutableMap.of(0, NEW_CONFIG, 1, OLD_CONFIG));
}
@@ -464,7 +547,9 @@ public class JobUpdaterIT extends EasyMockTest {
lockManager.releaseLock(lock);
}
clock.advance(RUNNING_TIMEOUT);
- assertUpdateStatus(ERROR);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ actions.putAll(0, INSTANCE_UPDATING);
+ assertState(ERROR, actions.build());
}
private void expectInvalid(JobUpdate update)
@@ -525,9 +610,12 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
// Instance 0 is updated
updater.start(update, USER);
- assertUpdateStatus(ROLLING_FORWARD);
+ actions.putAll(0, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
storage.write(new NoResult.Quiet() {
@Override
@@ -549,9 +637,13 @@ public class JobUpdaterIT extends EasyMockTest {
// Instance 1 is updated, but fails.
changeState(JOB, 1, KILLED, ASSIGNED, STARTING, RUNNING, FAILED);
+ // Actions is reset here since we wiped the updates tables earlier in the test case.
+ actions = ImmutableMultimap.builder();
+ actions.putAll(0, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ERROR);
+ assertState(ERROR, actions.build());
}
private ILock saveJobUpdate(
@@ -607,7 +699,11 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 1, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ROLLED_FORWARD);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED);
+
+ assertState(ROLLED_FORWARD, actions.build());
}
@Test
@@ -626,7 +722,7 @@ public class JobUpdaterIT extends EasyMockTest {
});
eventBus.post(new SchedulerActive());
- assertUpdateStatus(ERROR);
+ assertState(ERROR, ImmutableMultimap.<Integer, JobUpdateAction>of());
}
@Test
@@ -640,7 +736,11 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 2, ASSIGNED, STARTING, RUNNING);
clock.advance(ONE_DAY);
updater.start(update, USER);
- assertUpdateStatus(ROLLED_FORWARD);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ actions.putAll(0, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATED)
+ .putAll(2, INSTANCE_UPDATING, INSTANCE_UPDATED);
+ assertState(ROLLED_FORWARD, actions.build());
}
@Test
@@ -652,7 +752,16 @@ public class JobUpdaterIT extends EasyMockTest {
insertInitialTasks(update);
clock.advance(ONE_DAY);
updater.start(update, USER);
- assertUpdateStatus(JobUpdateStatus.FAILED);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+ Iterable<JobUpdateAction> states = ImmutableList.of(
+ INSTANCE_UPDATING,
+ INSTANCE_UPDATE_FAILED,
+ INSTANCE_ROLLING_BACK,
+ INSTANCE_ROLLBACK_FAILED);
+ actions.putAll(0, states)
+ .putAll(1, states)
+ .putAll(2, states);
+ assertState(JobUpdateStatus.FAILED, actions.build());
}
@Test
@@ -666,16 +775,21 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 0, ASSIGNED, STARTING, RUNNING);
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
+ ImmutableMultimap.Builder<Integer, JobUpdateAction> actions = ImmutableMultimap.builder();
+
// Instance 0 is updated.
updater.start(update, USER);
- assertUpdateStatus(ROLLING_FORWARD);
+ actions.putAll(0, INSTANCE_UPDATING);
+ assertState(ROLLING_FORWARD, actions.build());
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
// Instance 1 is stuck in PENDING.
changeState(JOB, 1, KILLED);
clock.advance(RUNNING_TIMEOUT);
- assertUpdateStatus(ROLLING_BACK);
+ actions.putAll(0, INSTANCE_UPDATED)
+ .putAll(1, INSTANCE_UPDATING, INSTANCE_UPDATE_FAILED, INSTANCE_ROLLING_BACK);
+ assertState(ROLLING_BACK, actions.build());
// Instance 1 is reverted.
changeState(JOB, 1, ASSIGNED, STARTING, RUNNING);
@@ -685,7 +799,9 @@ public class JobUpdaterIT extends EasyMockTest {
changeState(JOB, 0, KILLED, ASSIGNED, STARTING, RUNNING);
clock.advance(WATCH_TIMEOUT);
- assertUpdateStatus(ROLLED_BACK);
+ actions.putAll(0, INSTANCE_ROLLING_BACK, INSTANCE_ROLLED_BACK)
+ .putAll(1, INSTANCE_ROLLED_BACK);
+ assertState(ROLLED_BACK, actions.build());
assertJobState(JOB, ImmutableMap.of(0, OLD_CONFIG, 1, OLD_CONFIG));
}