You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2014/04/03 23:00:19 UTC
git commit: Before we start storing UNKNOWN (SANDBOX_DELETED) state
need to prepare for a graceful rollback of that change.
Repository: incubator-aurora
Updated Branches:
refs/heads/master d65a6f6e4 -> 8262c8b62
Before we start storing UNKNOWN (SANDBOX_DELETED) state
need to prepare for a graceful rollback of that change.
Bugs closed: AURORA-261
Reviewed at https://reviews.apache.org/r/19888/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/8262c8b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/8262c8b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/8262c8b6
Branch: refs/heads/master
Commit: 8262c8b628757dfc06a8d52edc0170ae13fc2255
Parents: d65a6f6
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Thu Apr 3 13:59:29 2014 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Thu Apr 3 13:59:29 2014 -0700
----------------------------------------------------------------------
.../scheduler/storage/StorageBackfill.java | 24 ++++
.../scheduler/storage/StorageBackfillTest.java | 118 +++++++++++++++++++
2 files changed, 142 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8262c8b6/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java b/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
index 6622216..9abab00 100644
--- a/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
+++ b/src/main/java/org/apache/aurora/scheduler/storage/StorageBackfill.java
@@ -101,6 +101,29 @@ public final class StorageBackfill {
}
}
+ private static void rewriteUnknownState(
+ ScheduledTask task,
+ TaskStore.Mutable taskStore,
+ Clock clock) {
+
+ if (task.getStatus() == ScheduleStatus.UNKNOWN) {
+ if (!task.isSetTaskEvents() || task.getTaskEvents().size() < 2) {
+ // This should never happen normally. Convert it to FAILED to prevent downstream failures.
+ LOG.severe("Missing terminal event leading to UNKNOWN for:"
+ + task.getAssignedTask().getTaskId());
+
+ task.setStatus(ScheduleStatus.FAILED);
+ task.unsetTaskEvents();
+ task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.FAILED));
+ } else {
+ // Overwrite the task status with the previous (terminal) state.
+ TaskEvent terminalEvent = task.getTaskEvents().get(task.getTaskEvents().size() - 2);
+ task.setStatus(terminalEvent.getStatus());
+ task.getTaskEvents().remove(task.getTaskEvents().size() - 1);
+ }
+ }
+ }
+
/**
* Backfills the storage to make it match any assumptions that may have changed since
* the structs were first written.
@@ -120,6 +143,7 @@ public final class StorageBackfill {
// TODO(ksweeney): Guarantee tasks pass current validation code here and quarantine if they
// don't.
guaranteeShardUniqueness(builder, storeProvider.getUnsafeTaskStore(), clock);
+ rewriteUnknownState(builder, storeProvider.getUnsafeTaskStore(), clock);
return IScheduledTask.build(builder);
}
});
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/8262c8b6/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java b/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java
new file mode 100644
index 0000000..c6e318c
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/storage/StorageBackfillTest.java
@@ -0,0 +1,118 @@
+/**
+ * Copyright 2014 Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.storage;
+
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+
+import com.twitter.common.util.testing.FakeClock;
+
+import org.apache.aurora.gen.AssignedTask;
+import org.apache.aurora.gen.Constraint;
+import org.apache.aurora.gen.Identity;
+import org.apache.aurora.gen.LimitConstraint;
+import org.apache.aurora.gen.ScheduleStatus;
+import org.apache.aurora.gen.ScheduledTask;
+import org.apache.aurora.gen.TaskConfig;
+import org.apache.aurora.gen.TaskConstraint;
+import org.apache.aurora.gen.TaskEvent;
+import org.apache.aurora.scheduler.base.Query;
+import org.apache.aurora.scheduler.storage.entities.IScheduledTask;
+import org.apache.aurora.scheduler.storage.mem.MemStorage;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class StorageBackfillTest {
+ private Storage storage;
+ private FakeClock clock;
+
+ @Before
+ public void setUp() {
+ storage = MemStorage.newEmptyStorage();
+ clock = new FakeClock();
+ }
+
+ @Test
+ public void testRewriteUnknownState() {
+ final String taskId = "id";
+ final ScheduledTask task = makeTask(taskId).setStatus(ScheduleStatus.UNKNOWN);
+ task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.FINISHED));
+ task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.UNKNOWN));
+
+ final ScheduledTask expected = makeTask(taskId).setStatus(ScheduleStatus.FINISHED);
+ expected.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.FINISHED));
+
+ storage.write(new Storage.MutateWork.NoResult.Quiet() {
+ @Override
+ protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getUnsafeTaskStore().saveTasks(ImmutableSet.of(IScheduledTask.build(task)));
+ StorageBackfill.backfill(storeProvider, clock);
+ }
+ });
+
+ assertEquals(
+ IScheduledTask.build(expected),
+ Iterables.getOnlyElement(Storage.Util.consistentFetchTasks(
+ storage,
+ Query.taskScoped(taskId))));
+ }
+
+ @Test
+ public void testRewriteUnknownToFailed() {
+ final String taskId = "id";
+ final ScheduledTask task = makeTask(taskId).setStatus(ScheduleStatus.UNKNOWN);
+ task.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.UNKNOWN));
+
+ final ScheduledTask expected = makeTask(taskId).setStatus(ScheduleStatus.FAILED);
+ expected.addToTaskEvents(new TaskEvent(clock.nowMillis(), ScheduleStatus.FAILED));
+
+ storage.write(new Storage.MutateWork.NoResult.Quiet() {
+ @Override
+ protected void execute(Storage.MutableStoreProvider storeProvider) {
+ storeProvider.getUnsafeTaskStore().saveTasks(ImmutableSet.of(IScheduledTask.build(task)));
+ StorageBackfill.backfill(storeProvider, clock);
+ }
+ });
+
+ assertEquals(
+ IScheduledTask.build(expected),
+ Iterables.getOnlyElement(Storage.Util.consistentFetchTasks(
+ storage,
+ Query.taskScoped(taskId))));
+ }
+
+ private ScheduledTask makeTask(String id) {
+ TaskConfig config = new TaskConfig()
+ .setOwner(new Identity("user", "role"))
+ .setEnvironment("test")
+ .setJobName("jobName")
+ .setProduction(false)
+ .setRequestedPorts(ImmutableSet.<String>of())
+ .setTaskLinks(ImmutableMap.<String, String>of())
+ .setMaxTaskFailures(1)
+ .setConstraints(ImmutableSet.of(
+ new Constraint("host", TaskConstraint.limit(new LimitConstraint(1)))));
+
+ ScheduledTask task = new ScheduledTask().setAssignedTask(new AssignedTask().setTask(config));
+ task.getAssignedTask().setTaskId(id);
+
+ return task;
+ }
+}