You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@aurora.apache.org by ma...@apache.org on 2015/03/14 02:28:05 UTC
incubator-aurora git commit: Adding more logging into
MaintenanceController.
Repository: incubator-aurora
Updated Branches:
refs/heads/master 128e5540e -> 7d68df391
Adding more logging into MaintenanceController.
Bugs closed: AURORA-1170
Reviewed at https://reviews.apache.org/r/32014/
Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/7d68df39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/7d68df39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/7d68df39
Branch: refs/heads/master
Commit: 7d68df39114d3fd9cea1df00ae45baae005474e3
Parents: 128e554
Author: Maxim Khutornenko <ma...@apache.org>
Authored: Fri Mar 13 18:18:28 2015 -0700
Committer: Maxim Khutornenko <ma...@apache.org>
Committed: Fri Mar 13 18:18:28 2015 -0700
----------------------------------------------------------------------
config/legacy_untested_classes.txt | 2 -
.../scheduler/state/MaintenanceController.java | 7 ++-
.../apache/aurora/admin/host_maintenance.py | 6 ++-
.../state/MaintenanceControllerImplTest.java | 45 +++++++++++++-------
.../aurora/admin/test_host_maintenance.py | 9 +++-
5 files changed, 47 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/config/legacy_untested_classes.txt
----------------------------------------------------------------------
diff --git a/config/legacy_untested_classes.txt b/config/legacy_untested_classes.txt
index 06e130f..4d090a8 100644
--- a/config/legacy_untested_classes.txt
+++ b/config/legacy_untested_classes.txt
@@ -55,8 +55,6 @@ org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$4
org/apache/aurora/scheduler/log/mesos/MesosLogStreamModule$5
org/apache/aurora/scheduler/mesos/DriverFactoryImpl
org/apache/aurora/scheduler/mesos/LibMesosLoadingModule
-org/apache/aurora/scheduler/state/MaintenanceController$MaintenanceControllerImpl$4
-org/apache/aurora/scheduler/state/MaintenanceController$MaintenanceControllerImpl$8
org/apache/aurora/scheduler/stats/AsyncStatsModule$OfferAdapter$1
org/apache/aurora/scheduler/storage/log/LogStorage$RecoveryFailedException
org/apache/aurora/scheduler/storage/mem/Util
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
index b6f642e..09be4dc 100644
--- a/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
+++ b/src/main/java/org/apache/aurora/scheduler/state/MaintenanceController.java
@@ -166,8 +166,9 @@ public interface MaintenanceController {
LOG.info(String.format("Moving host %s into DRAINED", host));
setMaintenanceMode(store, ImmutableSet.of(host), DRAINED);
} else {
- LOG.info(
- String.format("Host %s is DRAINING with active tasks: %s", host, activeTasks));
+ LOG.info(String.format("Host %s is DRAINING with active tasks: %s",
+ host,
+ Tasks.ids(activeTasks)));
}
}
}
@@ -268,9 +269,11 @@ public interface MaintenanceController {
AttributeStore.Mutable store = storeProvider.getAttributeStore();
ImmutableSet.Builder<HostStatus> statuses = ImmutableSet.builder();
for (String host : hosts) {
+ LOG.info(String.format("Setting maintenance mode to %s for host %s", mode, host));
Optional<IHostAttributes> toSave = AttributeStore.Util.mergeMode(store, host, mode);
if (toSave.isPresent()) {
store.saveHostAttributes(toSave.get());
+ LOG.info("Updated host attributes: " + toSave.get());
HostStatus status = new HostStatus().setHost(host).setMode(mode);
statuses.add(status);
}
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/main/python/apache/aurora/admin/host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/main/python/apache/aurora/admin/host_maintenance.py b/src/main/python/apache/aurora/admin/host_maintenance.py
index 8fa5182..c31e2a6 100644
--- a/src/main/python/apache/aurora/admin/host_maintenance.py
+++ b/src/main/python/apache/aurora/admin/host_maintenance.py
@@ -71,11 +71,13 @@ class HostMaintenance(object):
self._wait_event.wait(self.STATUS_POLL_INTERVAL.as_(Time.SECONDS))
statuses = self.check_status(list(not_drained_hostnames))
- not_drained_hostnames = set([h[0] for h in statuses if h[1] != 'DRAINED'])
+ not_drained_hostnames = set(h[0] for h in statuses if h[1] != 'DRAINED')
total_wait += self.STATUS_POLL_INTERVAL
if not_drained_hostnames and total_wait > self.MAX_STATUS_WAIT:
- log.warning('Failed to move all hosts into DRAINED within %s' % self.MAX_STATUS_WAIT)
+ log.warning('Failed to move all hosts into DRAINED within %s:\n%s' %
+ (self.MAX_STATUS_WAIT,
+ '\n'.join("\tHost:%s\tStatus:%s" % h for h in sorted(statuses) if h[1] != 'DRAINED')))
break
return not_drained_hostnames
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java b/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
index bd031a5..7b101bc 100644
--- a/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
+++ b/src/test/java/org/apache/aurora/scheduler/state/MaintenanceControllerImplTest.java
@@ -46,7 +46,7 @@ import static org.apache.aurora.gen.MaintenanceMode.DRAINED;
import static org.apache.aurora.gen.MaintenanceMode.DRAINING;
import static org.apache.aurora.gen.MaintenanceMode.NONE;
import static org.apache.aurora.gen.MaintenanceMode.SCHEDULED;
-import static org.apache.aurora.gen.ScheduleStatus.FINISHED;
+import static org.apache.aurora.gen.ScheduleStatus.KILLED;
import static org.apache.aurora.gen.ScheduleStatus.RUNNING;
import static org.apache.aurora.scheduler.state.MaintenanceController.MaintenanceControllerImpl;
import static org.easymock.EasyMock.expect;
@@ -97,21 +97,23 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
@Test
public void testMaintenanceCycle() {
- ScheduledTask task = makeTask(HOST_A, "taskA");
+ ScheduledTask task1 = makeTask(HOST_A, "taskA");
+ ScheduledTask task2 = makeTask(HOST_A, "taskB");
expectMaintenanceModeChange(HOST_A, SCHEDULED);
- expectFetchTasksByHost(HOST_A, ImmutableSet.of(task));
- expect(stateManager.changeState(
- storageUtil.mutableStoreProvider,
- Tasks.id(task),
- Optional.<ScheduleStatus>absent(),
- ScheduleStatus.DRAINING,
- MaintenanceControllerImpl.DRAINING_MESSAGE))
- .andReturn(true);
+ expectFetchTasksByHost(HOST_A, ImmutableSet.of(task1, task2));
+ expectTaskDraining(task1);
+ expectTaskDraining(task2);
expectMaintenanceModeChange(HOST_A, DRAINING);
- expect(storageUtil.attributeStore.getHostAttributes(HOST_A)).andReturn(Optional.of(
- IHostAttributes.build(new HostAttributes().setHost(HOST_A).setMode(DRAINING))));
- // TaskA is FINISHED and therefore no longer active
+ IHostAttributes attributes =
+ IHostAttributes.build(new HostAttributes().setHost(HOST_A).setMode(DRAINING));
+
+ expect(storageUtil.attributeStore.getHostAttributes(HOST_A))
+ .andReturn(Optional.of(attributes)).times(2);
+
+ expect(storageUtil.attributeStore.getHostAttributes()).andReturn(ImmutableSet.of(attributes));
+ expectFetchTasksByHost(HOST_A, ImmutableSet.of(task2));
+ // TaskA is KILLED and therefore no longer active
expectFetchTasksByHost(HOST_A, ImmutableSet.<ScheduledTask>of());
expectMaintenanceModeChange(HOST_A, DRAINED);
expectMaintenanceModeChange(HOST_A, NONE);
@@ -120,8 +122,11 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
assertStatus(HOST_A, SCHEDULED, maintenance.startMaintenance(A));
assertStatus(HOST_A, DRAINING, maintenance.drain(A));
+ assertStatus(HOST_A, DRAINING, maintenance.getStatus(A));
eventSink.post(
- TaskStateChange.transition(IScheduledTask.build(task.setStatus(FINISHED)), RUNNING));
+ TaskStateChange.transition(IScheduledTask.build(task1.setStatus(KILLED)), RUNNING));
+ eventSink.post(
+ TaskStateChange.transition(IScheduledTask.build(task2.setStatus(KILLED)), RUNNING));
assertStatus(HOST_A, NONE, maintenance.endMaintenance(A));
}
@@ -165,7 +170,7 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
// Make sure a later transition on the host does not cause any ill effects that could surface
// from stale internal state.
eventSink.post(TaskStateChange.transition(
- IScheduledTask.build(makeTask(HOST_A, "taskA").setStatus(FINISHED)), RUNNING));
+ IScheduledTask.build(makeTask(HOST_A, "taskA").setStatus(KILLED)), RUNNING));
}
@Test
@@ -181,6 +186,16 @@ public class MaintenanceControllerImplTest extends EasyMockTest {
assertEquals(NONE, maintenance.getMode("unknown"));
}
+ private void expectTaskDraining(ScheduledTask task) {
+ expect(stateManager.changeState(
+ storageUtil.mutableStoreProvider,
+ Tasks.id(task),
+ Optional.<ScheduleStatus>absent(),
+ ScheduleStatus.DRAINING,
+ MaintenanceControllerImpl.DRAINING_MESSAGE))
+ .andReturn(true);
+ }
+
private void expectFetchTasksByHost(String hostName, ImmutableSet<ScheduledTask> tasks) {
expect(storageUtil.taskStore.fetchTasks(Query.slaveScoped(hostName).active()))
.andReturn(IScheduledTask.setFromBuilders(tasks));
http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/7d68df39/src/test/python/apache/aurora/admin/test_host_maintenance.py
----------------------------------------------------------------------
diff --git a/src/test/python/apache/aurora/admin/test_host_maintenance.py b/src/test/python/apache/aurora/admin/test_host_maintenance.py
index b7c13b7..6264d0c 100644
--- a/src/test/python/apache/aurora/admin/test_host_maintenance.py
+++ b/src/test/python/apache/aurora/admin/test_host_maintenance.py
@@ -13,6 +13,7 @@
#
import copy
+import textwrap
import unittest
from contextlib import contextmanager
@@ -103,12 +104,13 @@ class TestHostMaintenance(unittest.TestCase):
(Hosts(set(TEST_HOSTNAMES))),
(Hosts(set([TEST_HOSTNAMES[0]])))]
+ @mock.patch("twitter.common.log.warning", spec=log.warning)
@mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status",
spec=AuroraClientAPI.maintenance_status)
@mock.patch("apache.aurora.client.api.AuroraClientAPI.drain_hosts",
spec=AuroraClientAPI.drain_hosts)
@mock.patch("threading._Event.wait")
- def test_drain_hosts_timed_out_wait(self, _, mock_drain_hosts, mock_maintenance_status):
+ def test_drain_hosts_timed_out_wait(self, _, mock_drain_hosts, mock_maintenance_status, mock_log):
fake_maintenance_status_response = Response(
responseCode=ResponseCode.OK,
result=Result(maintenanceStatusResult=MaintenanceStatusResult(set([
@@ -128,6 +130,11 @@ class TestHostMaintenance(unittest.TestCase):
assert mock_maintenance_status.call_count == 1
mock_drain_hosts.assert_called_once_with(test_hosts)
mock_maintenance_status.assert_called_once_with((Hosts(set(TEST_HOSTNAMES))))
+ assert mock_log.mock_calls == [mock.call(textwrap.dedent("""\
+ Failed to move all hosts into DRAINED within 1 ms:
+ \tHost:us-west-001.example.com\tStatus:SCHEDULED
+ \tHost:us-west-002.example.com\tStatus:SCHEDULED
+ \tHost:us-west-003.example.com\tStatus:SCHEDULED"""))]
@mock.patch("twitter.common.log.warning", spec=log.warning)
@mock.patch("apache.aurora.client.api.AuroraClientAPI.maintenance_status",