You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by wf...@apache.org on 2013/12/13 04:05:13 UTC
git commit: Output all states in maintenance endpoint.
Updated Branches:
refs/heads/master ff5d993d0 -> 4eb26e744
Output all states in maintenance endpoint.
Improve the /maintenance endpoint to print out hosts affected by
SCHEDULED and DRAINED states.
Testing Done:
./gradlew clean build
Bugs closed: AURORA-9
Reviewed at https://reviews.apache.org/r/16220/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/4eb26e74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/4eb26e74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/4eb26e74
Branch: refs/heads/master
Commit: 4eb26e744e79cbeb3c844880e531af7f190139f5
Parents: ff5d993
Author: Zameer Manji <zm...@gmail.com>
Authored: Thu Dec 12 19:04:47 2013 -0800
Committer: Bill Farner <te...@gmail.com>
Committed: Thu Dec 12 19:04:47 2013 -0800
----------------------------------------------------------------------
.../aurora/scheduler/http/Maintenance.java | 76 ++++++++++++++++++--
.../scheduler/state/MaintenanceController.java | 42 ++---------
.../state/MaintenanceControllerImplTest.java | 27 +++++--
3 files changed, 95 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4eb26e74/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java b/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
index 30afce3..fb71539 100644
--- a/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
+++ b/src/main/java/com/twitter/aurora/scheduler/http/Maintenance.java
@@ -15,6 +15,8 @@
*/
package com.twitter.aurora.scheduler.http;
+import java.util.Map;
+
import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -22,25 +24,85 @@ import javax.ws.rs.Produces;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
+import com.google.common.base.Function;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+
+import com.twitter.aurora.gen.HostAttributes;
+import com.twitter.aurora.gen.MaintenanceMode;
+import com.twitter.aurora.scheduler.base.Query;
+import com.twitter.aurora.scheduler.base.Tasks;
+import com.twitter.aurora.scheduler.storage.Storage;
+import com.twitter.aurora.scheduler.storage.Storage.StoreProvider;
+import com.twitter.aurora.scheduler.storage.Storage.Work;
+import com.twitter.aurora.scheduler.storage.entities.IScheduledTask;
-import com.twitter.aurora.scheduler.state.MaintenanceController;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
+import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
+import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
/**
- * Servlet that exposes state of {@link MaintenanceController}.
+ * Servlet that exposes the maintenance state of hosts.
*/
@Path("/maintenance")
public class Maintenance {
- private final MaintenanceController maintenance;
+ private final Storage storage;
@Inject
- Maintenance(MaintenanceController maintenance) {
- this.maintenance = Preconditions.checkNotNull(maintenance);
+ Maintenance(Storage storage) {
+ this.storage = Preconditions.checkNotNull(storage);
}
@GET
@Produces(MediaType.APPLICATION_JSON)
- public Response getOffers() {
- return Response.ok(maintenance.getDrainingTasks().asMap()).build();
+ public Response getHosts() {
+ return storage.weaklyConsistentRead(new Work.Quiet<Response>() {
+ @Override public Response apply(StoreProvider storeProvider) {
+ Multimap<MaintenanceMode, String> hostsByMode =
+ Multimaps.transformValues(
+ Multimaps.index(storeProvider.getAttributeStore().getHostAttributes(), GET_MODE),
+ HOST_NAME);
+
+ Map<MaintenanceMode, Object> hosts = Maps.newHashMap();
+ hosts.put(DRAINED, ImmutableSet.copyOf(hostsByMode.get(DRAINED)));
+ hosts.put(SCHEDULED, ImmutableSet.copyOf(hostsByMode.get(SCHEDULED)));
+ hosts.put(DRAINING, getTasksByHosts(storeProvider, hostsByMode.get(DRAINING)).asMap());
+ return Response.ok(hosts).build();
+ }
+ });
+ }
+
+ private Multimap<String, String> getTasksByHosts(StoreProvider provider, Iterable<String> hosts) {
+ ImmutableSet.Builder<IScheduledTask> drainingTasks = ImmutableSet.builder();
+ for (String host : hosts) {
+ drainingTasks.addAll(provider.getTaskStore().fetchTasks(Query.slaveScoped(host).active()));
+ }
+ return Multimaps.transformValues(
+ Multimaps.index(drainingTasks.build(), TASK_TO_HOST),
+ Tasks.SCHEDULED_TO_ID);
}
+
+ private static final Function<IScheduledTask, String> TASK_TO_HOST =
+ new Function<IScheduledTask, String>() {
+ @Override public String apply(IScheduledTask task) {
+ return task.getAssignedTask().getSlaveHost();
+ }
+ };
+
+ private static final Function<HostAttributes, String> HOST_NAME =
+ new Function<HostAttributes, String>() {
+ @Override public String apply(HostAttributes attributes) {
+ return attributes.getHost();
+ }
+ };
+
+ private static final Function<HostAttributes, MaintenanceMode> GET_MODE =
+ new Function<HostAttributes, MaintenanceMode>() {
+ @Override public MaintenanceMode apply(HostAttributes attrs) {
+ return attrs.getMode();
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4eb26e74/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
----------------------------------------------------------------------
diff --git a/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java b/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
index fb12d38..d447f52 100644
--- a/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
+++ b/src/main/java/com/twitter/aurora/scheduler/state/MaintenanceController.java
@@ -25,11 +25,7 @@ import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.FluentIterable;
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableMultimap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Multimaps;
import com.google.common.collect.Sets;
import com.google.common.eventbus.Subscribe;
@@ -109,24 +105,10 @@ public interface MaintenanceController {
*/
Set<HostStatus> endMaintenance(Set<String> hosts);
- /**
- * Fetches a mapping from names of DRAINING hosts to the IDs of tasks that are currently being
- * drained from the hosts.
- * This is intended for display/debugging only.
- *
- * @return Draining task IDs, mapped by host name.
- */
- Multimap<String, String> getDrainingTasks();
-
class MaintenanceControllerImpl implements MaintenanceController, EventSubscriber {
private final Storage storage;
private final StateManager stateManager;
private final Closure<PubsubEvent> eventSink;
- // Access on views, or non-atomic operations on this map must synchronize on the map itself.
- // Please be careful to avoid securing any external locks when locking on this data structure,
- // however.
- private final Multimap<String, String> drainingTasksByHost =
- Multimaps.synchronizedMultimap(HashMultimap.<String, String>create());
@Inject
public MaintenanceControllerImpl(
@@ -154,7 +136,6 @@ public interface MaintenanceController {
if (activeTasks.isEmpty()) {
emptyHosts.add(host);
} else {
- drainingTasksByHost.putAll(host, activeTasks);
callback.execute(query);
}
}
@@ -208,13 +189,12 @@ public interface MaintenanceController {
@Override public void execute(MutableStoreProvider store) {
// If the task _was_ associated with a draining host, and it was the last task on the
// host.
- boolean drained;
- synchronized (drainingTasksByHost) {
- drained = drainingTasksByHost.remove(host, change.getTaskId())
- && !drainingTasksByHost.containsKey(host);
- }
- if (drained) {
- setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
+ Optional<HostAttributes> attributes = store.getAttributeStore().getHostAttributes(host);
+ if (attributes.isPresent() && attributes.get().getMode() == DRAINING) {
+ Query.Builder builder = Query.slaveScoped(host).active();
+ if (store.getTaskStore().fetchTasks(builder).isEmpty()) {
+ setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
+ }
}
}
});
@@ -297,21 +277,11 @@ public interface MaintenanceController {
public Set<HostStatus> endMaintenance(final Set<String> hosts) {
return storage.write(new MutateWork.Quiet<Set<HostStatus>>() {
@Override public Set<HostStatus> apply(MutableStoreProvider storeProvider) {
- synchronized (drainingTasksByHost) {
- drainingTasksByHost.keys().removeAll(hosts);
- }
return setMaintenanceMode(storeProvider, hosts, MaintenanceMode.NONE);
}
});
}
- @Override
- public Multimap<String, String> getDrainingTasks() {
- synchronized (drainingTasksByHost) {
- return ImmutableMultimap.copyOf(drainingTasksByHost);
- }
- }
-
private Set<HostStatus> setMaintenanceMode(
MutableStoreProvider storeProvider,
Set<String> hosts,
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/4eb26e74/src/test/java/com/twitter/aurora/scheduler/state/MaintenanceControllerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/com/twitter/aurora/scheduler/state/MaintenanceControllerImplTest.java b/src/test/java/com/twitter/aurora/scheduler/state/MaintenanceControllerImplTest.java
index 8acb716..e6e60bf 100644
--- a/src/test/java/com/twitter/aurora/scheduler/state/MaintenanceControllerImplTest.java
+++ b/src/test/java/com/twitter/aurora/scheduler/state/MaintenanceControllerImplTest.java
@@ -15,10 +15,14 @@
*/
package com.twitter.aurora.scheduler.state;
+import java.util.HashSet;
import java.util.Set;
import com.google.common.base.Optional;
+import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
import com.google.inject.AbstractModule;
import com.google.inject.Guice;
import com.google.inject.Injector;
@@ -44,9 +48,6 @@ import com.twitter.aurora.scheduler.storage.testing.StorageTestUtil;
import com.twitter.common.base.Closure;
import com.twitter.common.testing.easymock.EasyMockTest;
-import static org.easymock.EasyMock.expect;
-import static org.junit.Assert.assertEquals;
-
import static com.twitter.aurora.gen.MaintenanceMode.DRAINED;
import static com.twitter.aurora.gen.MaintenanceMode.DRAINING;
import static com.twitter.aurora.gen.MaintenanceMode.NONE;
@@ -54,6 +55,8 @@ import static com.twitter.aurora.gen.MaintenanceMode.SCHEDULED;
import static com.twitter.aurora.gen.ScheduleStatus.FINISHED;
import static com.twitter.aurora.gen.ScheduleStatus.RUNNING;
import static com.twitter.aurora.scheduler.state.MaintenanceController.MaintenanceControllerImpl;
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertEquals;
public class MaintenanceControllerImplTest extends EasyMockTest {
@@ -103,15 +106,19 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
ScheduledTask task = makeTask(HOST_A, "taskA");
expectMaintenanceModeChange(HOST_A, SCHEDULED);
- expectMaintenanceModeChange(HOST_A, DRAINING);
expectFetchTasksByHost(HOST_A, ImmutableSet.<ScheduledTask>of(task));
- expectMaintenanceModeChange(HOST_A, DRAINED);
- expectMaintenanceModeChange(HOST_A, NONE);
expect(stateManager.changeState(
Query.slaveScoped(HOST_A).active(),
ScheduleStatus.RESTARTING,
MaintenanceControllerImpl.DRAINING_MESSAGE))
.andReturn(1);
+ expectMaintenanceModeChange(HOST_A, DRAINING);
+ expect(storageUtil.attributeStore.getHostAttributes(HOST_A))
+ .andReturn(Optional.of(new HostAttributes().setHost(HOST_A).setMode(DRAINING)));
+ // TaskA is FINISHED and therefore no longer active
+ expectFetchTasksByHost(HOST_A, ImmutableSet.<ScheduledTask>of());
+ expectMaintenanceModeChange(HOST_A, DRAINED);
+ expectMaintenanceModeChange(HOST_A, NONE);
control.replay();
@@ -148,6 +155,8 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
public void testEndEarly() {
expectMaintenanceModeChange(HOST_A, SCHEDULED);
expectMaintenanceModeChange(HOST_A, NONE);
+ expect(storageUtil.attributeStore.getHostAttributes(HOST_A))
+ .andReturn(Optional.of(new HostAttributes().setHost(HOST_A).setMode(NONE)));
control.replay();
@@ -170,10 +179,14 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
.andReturn(ImmutableSet.of(
new HostAttributes().setHost(HOST_A).setMode(DRAINING),
new HostAttributes().setHost(HOST_B).setMode(DRAINING)));
+ expectFetchTasksByHost(HOST_A, ImmutableSet.of(taskA));
expectFetchTasksByHost(HOST_B, ImmutableSet.<ScheduledTask>of());
expectMaintenanceModeChange(HOST_B, DRAINED);
expectMaintenanceModeChange(HOST_A, DRAINING);
- expectFetchTasksByHost(HOST_A, ImmutableSet.of(taskA));
+ expect(storageUtil.attributeStore.getHostAttributes(HOST_A))
+ .andReturn(Optional.of(new HostAttributes().setHost(HOST_A).setMode(DRAINING)));
+ // TaskA is FINISHED and therefore no longer active
+ expectFetchTasksByHost(HOST_A, ImmutableSet.<ScheduledTask>of());
expectMaintenanceModeChange(HOST_A, DRAINED);
control.replay();