[09/13] mesos git commit: Added a test to verify a fix for MESOS-7863.

[bm...@apache.org]

Added a test to verify a fix for MESOS-7863.

This test ensures that the agent does not drop the TASK_KILLED
update when all pending tasks from a framework are killed.
See MESOS-7863 for the motivating issue.

Review: https://reviews.apache.org/r/61646


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/658992ca
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/658992ca
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/658992ca

Branch: refs/heads/master
Commit: 658992cad0174f223d7eaeb30d2eca13845ca6a7
Parents: 45c2444
Author: Benjamin Mahler <bm...@apache.org>
Authored: Fri Aug 4 19:32:12 2017 -0700
Committer: Benjamin Mahler <bm...@apache.org>
Committed: Wed Aug 23 15:09:45 2017 -0700

----------------------------------------------------------------------
 src/tests/slave_tests.cpp | 130 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/658992ca/src/tests/slave_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index e650f31..57679bd 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -32,6 +32,7 @@
 #include <mesos/authentication/http/basic_authenticator_factory.hpp>
 
 #include <process/clock.hpp>
+#include <process/collect.hpp>
 #include <process/future.hpp>
 #include <process/gmock.hpp>
 #include <process/http.hpp>
@@ -4092,6 +4093,135 @@ TEST_F(SlaveTest, KillTaskBetweenRunTaskParts)
 }
 
 
+// This test ensures was added due to MESOS-7863, where the
+// agent previously dropped TASK_KILLED in the cases outlined
+// in the issue.
+TEST_F(SlaveTest, KillMultiplePendingTasks)
+{
+  Try<Owned<cluster::Master>> master = StartMaster();
+  ASSERT_SOME(master);
+
+  MockExecutor exec(DEFAULT_EXECUTOR_ID);
+  TestContainerizer containerizer(&exec);
+
+  StandaloneMasterDetector detector(master.get()->pid);
+
+  MockSlave slave(CreateSlaveFlags(), &detector, &containerizer);
+  spawn(slave);
+
+  MockScheduler sched;
+  MesosSchedulerDriver driver(
+      &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+  EXPECT_CALL(sched, registered(&driver, _, _));
+
+  Future<vector<Offer>> offers;
+  EXPECT_CALL(sched, resourceOffers(&driver, _))
+    .WillOnce(FutureArg<1>(&offers))
+    .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+  driver.start();
+
+  AWAIT_READY(offers);
+  EXPECT_NE(0u, offers->size());
+
+  // We only pause the clock after receiving the offer since the
+  // agent uses a delay to re-register.
+  //
+  // TODO(bmahler): Remove the initial agent delay within the tests.
+  Clock::pause();
+
+  Resources taskResources = Resources::parse("cpus:0.1;mem:32;disk:32").get();
+
+  TaskInfo task1 = createTask(
+      offers->at(0).slave_id(), taskResources, "echo hi");
+
+  TaskInfo task2 = createTask(
+      offers->at(0).slave_id(), taskResources, "echo hi");
+
+  EXPECT_CALL(containerizer, launch(_, _, _, _))
+    .Times(0);
+
+  Future<TaskStatus> status1, status2;
+  EXPECT_CALL(sched, statusUpdate(&driver, _))
+    .WillOnce(FutureArg<1>(&status1))
+    .WillOnce(FutureArg<1>(&status2));
+
+  EXPECT_CALL(slave, runTask(_, _, _, _, _))
+    .WillOnce(Invoke(&slave, &MockSlave::unmocked_runTask))
+    .WillOnce(Invoke(&slave, &MockSlave::unmocked_runTask));
+
+  // Skip what Slave::_run() normally does, save its arguments for
+  // later, tie reaching the critical moment when to kill the task to
+  // a future.
+  Future<bool> future1, future2;
+  FrameworkInfo frameworkInfo1, frameworkInfo2;
+  ExecutorInfo executorInfo1, executorInfo2;
+  Option<TaskGroupInfo> taskGroup1, taskGroup2;
+  Option<TaskInfo> task_1, task_2;
+
+  Future<Nothing> _run1, _run2;
+  EXPECT_CALL(slave, _run(_, _, _, _, _))
+    .WillOnce(DoAll(FutureSatisfy(&_run1),
+                    SaveArg<0>(&future1),
+                    SaveArg<1>(&frameworkInfo1),
+                    SaveArg<2>(&executorInfo1),
+                    SaveArg<3>(&task_1),
+                    SaveArg<4>(&taskGroup1)))
+    .WillOnce(DoAll(FutureSatisfy(&_run2),
+                    SaveArg<0>(&future2),
+                    SaveArg<1>(&frameworkInfo2),
+                    SaveArg<2>(&executorInfo2),
+                    SaveArg<3>(&task_2),
+                    SaveArg<4>(&taskGroup2)));
+
+  driver.launchTasks(offers.get()[0].id(), {task1, task2});
+
+  AWAIT_READY(process::await(_run1, _run2));
+
+  Future<Nothing> killTask1, killTask2;
+  EXPECT_CALL(slave, killTask(_, _))
+    .WillOnce(DoAll(Invoke(&slave, &MockSlave::unmocked_killTask),
+                    FutureSatisfy(&killTask1)))
+    .WillOnce(DoAll(Invoke(&slave, &MockSlave::unmocked_killTask),
+                    FutureSatisfy(&killTask2)));
+
+  Future<Nothing> removeFramework;
+  EXPECT_CALL(slave, removeFramework(_))
+    .WillOnce(DoAll(Invoke(&slave, &MockSlave::unmocked_removeFramework),
+                    FutureSatisfy(&removeFramework)));
+
+  driver.killTask(task1.task_id());
+  driver.killTask(task2.task_id());
+
+  AWAIT_READY(process::await(killTask1, killTask2));
+
+  // We expect the tasks to be killed and framework removed.
+  AWAIT_READY(status1);
+  EXPECT_EQ(TASK_KILLED, status1->state());
+
+  AWAIT_READY(status2);
+  EXPECT_EQ(TASK_KILLED, status2->state());
+
+  AWAIT_READY(removeFramework);
+
+  // The `__run` continuations should have no effect.
+  slave.unmocked__run(
+      future1, frameworkInfo1, executorInfo1, task_1, taskGroup1);
+
+  slave.unmocked__run(
+      future2, frameworkInfo2, executorInfo2, task_2, taskGroup2);
+
+  Clock::settle();
+
+  driver.stop();
+  driver.join();
+
+  terminate(slave);
+  wait(slave);
+}
+
+
 // This test verifies that when the agent gets a `killTask`
 // message for a queued task on a registering executor, a
 // the agent will generate a TASK_KILLED and will shut down