You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by an...@apache.org on 2018/02/26 07:24:28 UTC
[12/12] mesos git commit: Moved `DOCKER` tests to
`DockerContainerizerHealthCheckTest`.
Moved `DOCKER` tests to `DockerContainerizerHealthCheckTest`.
In `health_check_tests.cpp`, the tests matching `*DOCKER*` were moved
to `DockerContainerizerHealthCheckTest::Setup` so that pre-pulling
only happens if the test is actually run. This avoids failing the test
suite due to the pull failing when `docker` isn't installed.
To expand on why this change was necessary: each `DOCKER` test
requires (a) Docker to be available and (b) the images to be pulled.
The `DOCKER` test filter correctly satisfies (a), that is, if `docker`
isn't available, then the test isn't run. However, the previous logic
had the image pulling in `SetUpTestCase`, which was run once for the
whole test suite, and therefore wasn't affected by the `DOCKER`
filter, meaning `docker pull` would run (and fail) if `docker` wasn't
installed, since the suite had tests other than `DOCKER` tests.
So we need the `docker pull` logic to be executed if and only if
Docker is available. We already have logic for checking this on a
per-test basis, the `DOCKER` filter. So the code was moved to `SetUp`,
which is run for each test. The code is in essence idempotent: once
the images are pulled, pulling again is a no-op since they're
cached (and the logging is done with `LOG_FIRST_N(..., 1)`).
Furthermore, the tests were constrained to a suite consisting of all
`DOCKER` tests (the `DockerContainerizerHealthCheckTests`), and it's
reasonable to assume that all tests added to this suite will use the
`DOCKER` filter.
Review: https://reviews.apache.org/r/65727/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/81c724ec
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/81c724ec
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/81c724ec
Branch: refs/heads/master
Commit: 81c724ecf261663f2c6412e83064f3683b785f2e
Parents: 501d176
Author: Akash Gupta <ak...@hotmail.com>
Authored: Sun Feb 25 13:36:55 2018 -0800
Committer: Andrew Schwartzmeyer <an...@schwartzmeyer.com>
Committed: Sun Feb 25 22:13:32 2018 -0800
----------------------------------------------------------------------
src/tests/health_check_tests.cpp | 730 +++++++++++++++++-----------------
1 file changed, 358 insertions(+), 372 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/81c724ec/src/tests/health_check_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/health_check_tests.cpp b/src/tests/health_check_tests.cpp
index 7b8cd0f..6cdfe97 100644
--- a/src/tests/health_check_tests.cpp
+++ b/src/tests/health_check_tests.cpp
@@ -17,6 +17,7 @@
#include <mesos/executor.hpp>
#include <mesos/scheduler.hpp>
+#include <process/collect.hpp>
#include <process/future.hpp>
#include <process/owned.hpp>
#include <process/pid.hpp>
@@ -105,20 +106,6 @@ namespace tests {
class HealthCheckTest : public MesosTest
{
public:
- // Manually pull the images before the tests are run, since the pull can
- // time out the test.
- static void SetUpTestCase()
- {
- Future<Nothing> pull = pullDockerImage(DOCKER_TEST_IMAGE);
-
- LOG(WARNING) << "Downloading " << string(DOCKER_TEST_IMAGE)
- << ". This may take a while...";
-
- // The Windows image is ~200 MB, while the Linux image is ~2MB, so
- // hopefully this is enough time for the Windows image.
- AWAIT_READY_FOR(pull, Minutes(10));
- }
-
vector<TaskInfo> populateTasks(
const string& cmd,
const string& healthCmd,
@@ -576,135 +563,6 @@ TEST_F(HealthCheckTest, ROOT_HealthyTaskWithContainerImage)
#endif // __linux__
-// This test creates a healthy task using the Docker executor and
-// verifies that the healthy status is reported to the scheduler.
-TEST_F(HealthCheckTest, ROOT_DOCKER_DockerHealthyTask)
-{
- Shared<Docker> docker(new MockDocker(
- tests::flags.docker, tests::flags.docker_socket));
-
- Try<Nothing> validateResult = docker->validateVersion(Version(1, 3, 0));
- ASSERT_SOME(validateResult)
- << "-------------------------------------------------------------\n"
- << "We cannot run this test because of 'docker exec' command \n"
- << "require docker version greater than '1.3.0'. You won't be \n"
- << "able to use the docker exec method, but feel free to disable\n"
- << "this test.\n"
- << "-------------------------------------------------------------";
-
- Try<Owned<cluster::Master>> master = StartMaster();
- ASSERT_SOME(master);
-
- slave::Flags agentFlags = CreateSlaveFlags();
-
- Fetcher fetcher(agentFlags);
-
- Try<ContainerLogger*> logger =
- ContainerLogger::create(agentFlags.container_logger);
-
- ASSERT_SOME(logger);
-
- MockDockerContainerizer containerizer(
- agentFlags,
- &fetcher,
- Owned<ContainerLogger>(logger.get()),
- docker);
-
- Owned<MasterDetector> detector = master.get()->createDetector();
- Try<Owned<cluster::Slave>> agent =
- StartSlave(detector.get(), &containerizer, agentFlags);
- ASSERT_SOME(agent);
-
- MockScheduler sched;
- MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
-
- EXPECT_CALL(sched, registered(&driver, _, _));
-
- Future<vector<Offer>> offers;
- EXPECT_CALL(sched, resourceOffers(&driver, _))
- .WillOnce(FutureArg<1>(&offers))
- .WillRepeatedly(Return()); // Ignore subsequent offers.
-
- driver.start();
-
- AWAIT_READY(offers);
- ASSERT_FALSE(offers->empty());
-
- ContainerInfo containerInfo;
- containerInfo.set_type(ContainerInfo::DOCKER);
-
- // TODO(tnachen): Use local image to test if possible.
- ContainerInfo::DockerInfo dockerInfo;
- dockerInfo.set_image(DOCKER_TEST_IMAGE);
- containerInfo.mutable_docker()->CopyFrom(dockerInfo);
-
- vector<TaskInfo> tasks = populateTasks(
- DOCKER_SLEEP_CMD(120),
- "exit 0",
- offers.get()[0],
- 0,
- None(),
- None(),
- containerInfo);
-
- Future<ContainerID> containerId;
- EXPECT_CALL(containerizer, launch(_, _, _, _))
- .WillOnce(DoAll(FutureArg<0>(&containerId),
- Invoke(&containerizer,
- &MockDockerContainerizer::_launch)));
-
- Future<TaskStatus> statusStarting;
- Future<TaskStatus> statusRunning;
- Future<TaskStatus> statusHealthy;
-
- EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&statusStarting))
- .WillOnce(FutureArg<1>(&statusRunning))
- .WillOnce(FutureArg<1>(&statusHealthy));
-
- driver.launchTasks(offers.get()[0].id(), tasks);
-
- AWAIT_READY(containerId);
-
- AWAIT_READY(statusStarting);
- EXPECT_EQ(TASK_STARTING, statusStarting->state());
-
- AWAIT_READY(statusRunning);
- EXPECT_EQ(TASK_RUNNING, statusRunning->state());
-
- AWAIT_READY(statusHealthy);
- EXPECT_EQ(TASK_RUNNING, statusHealthy->state());
- EXPECT_EQ(
- TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- statusHealthy->reason());
- EXPECT_TRUE(statusHealthy->has_healthy());
- EXPECT_TRUE(statusHealthy->healthy());
-
- Future<Option<ContainerTermination>> termination =
- containerizer.wait(containerId.get());
-
- driver.stop();
- driver.join();
-
- AWAIT_READY(termination);
- EXPECT_SOME(termination.get());
-
- agent.get()->terminate();
- agent->reset();
-
- Future<std::list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- // Cleanup all mesos launched containers.
- foreach (const Docker::Container& container, containers.get()) {
- AWAIT_READY_FOR(docker->rm(container.id, true), Seconds(30));
- }
-}
-
-
// Same as above, but use the non-shell version of the health command.
TEST_F(HealthCheckTest, HealthyTaskNonShell)
{
@@ -862,49 +720,20 @@ TEST_F(HealthCheckTest, HealthStatusChange)
}
-// This test creates a task that uses the Docker executor and whose
-// health flaps. It then verifies that the health status updates are
-// sent to the framework scheduler.
-TEST_F(HealthCheckTest, ROOT_DOCKER_DockerHealthStatusChange)
+// This test ensures that a task is killed if the number of maximum
+// health check failures is reached.
+TEST_F(HealthCheckTest, ConsecutiveFailures)
{
- Shared<Docker> docker(new MockDocker(
- tests::flags.docker, tests::flags.docker_socket));
-
- Try<Nothing> validateResult = docker->validateVersion(Version(1, 3, 0));
- ASSERT_SOME(validateResult)
- << "-------------------------------------------------------------\n"
- << "We cannot run this test because of 'docker exec' command \n"
- << "require docker version greater than '1.3.0'. You won't be \n"
- << "able to use the docker exec method, but feel free to disable\n"
- << "this test.\n"
- << "-------------------------------------------------------------";
-
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
- slave::Flags agentFlags = CreateSlaveFlags();
-
- Fetcher fetcher(agentFlags);
-
- Try<ContainerLogger*> logger =
- ContainerLogger::create(agentFlags.container_logger);
-
- ASSERT_SOME(logger);
-
- MockDockerContainerizer containerizer(
- agentFlags,
- &fetcher,
- Owned<ContainerLogger>(logger.get()),
- docker);
-
Owned<MasterDetector> detector = master.get()->createDetector();
- Try<Owned<cluster::Slave>> agent =
- StartSlave(detector.get(), &containerizer, agentFlags);
+ Try<Owned<cluster::Slave>> agent = StartSlave(detector.get());
ASSERT_SOME(agent);
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(&driver, _, _));
@@ -918,72 +747,26 @@ TEST_F(HealthCheckTest, ROOT_DOCKER_DockerHealthStatusChange)
AWAIT_READY(offers);
ASSERT_FALSE(offers->empty());
- ContainerInfo containerInfo;
- containerInfo.set_type(ContainerInfo::DOCKER);
-
- // TODO(tnachen): Use local image to test if possible.
- ContainerInfo::DockerInfo dockerInfo;
- dockerInfo.set_image(DOCKER_TEST_IMAGE);
- containerInfo.mutable_docker()->CopyFrom(dockerInfo);
-
- // Create a temporary file in host and then we could this file to
- // make sure the health check command is run in docker container.
- string tmpPath = path::join(os::getcwd(), "foobar");
- ASSERT_SOME(os::write(tmpPath, "bar"));
-
- // This command fails every other invocation.
- // For all runs i in Nat0, the following case i % 2 applies:
- //
- // Case 0:
- // - Attempt to remove the nonexistent temporary file.
- // - Create the temporary file.
- // - Exit with a non-zero status.
- //
- // Case 1:
- // - Remove the temporary file.
-#ifdef __WINDOWS__
- const string healthCheckCmd =
- "pwsh -Command "
- "Remove-Item -ErrorAction SilentlyContinue \"" + tmpPath + "\"; "
- "if (-Not $?) { "
- "New-Item -ItemType Directory -Force \"" + os::getcwd() + "\"; "
- "Set-Content -Path \"" + tmpPath + "\" -Value foo; "
- "exit 1 "
- "}";
-#else
- const string healthCheckCmd =
- "rm " + tmpPath + " || "
- "(mkdir -p " + os::getcwd() + " && echo foo >" + tmpPath + " && exit 1)";
-#endif // __WINDOWS__
-
vector<TaskInfo> tasks = populateTasks(
- DOCKER_SLEEP_CMD(60),
- healthCheckCmd,
- offers.get()[0],
- 0,
- 3,
- None(),
- containerInfo);
-
- Future<ContainerID> containerId;
- EXPECT_CALL(containerizer, launch(_, _, _, _))
- .WillOnce(DoAll(FutureArg<0>(&containerId),
- Invoke(&containerizer,
- &MockDockerContainerizer::_launch)));
+ SLEEP_COMMAND(120), "exit 1", offers.get()[0], 0, 4);
+ // Expecting four unhealthy updates and one final kill update.
Future<TaskStatus> statusStarting;
Future<TaskStatus> statusRunning;
- Future<TaskStatus> statusUnhealthy;
- Future<TaskStatus> statusHealthy;
- Future<TaskStatus> statusUnhealthyAgain;
+ Future<TaskStatus> status1;
+ Future<TaskStatus> status2;
+ Future<TaskStatus> status3;
+ Future<TaskStatus> status4;
+ Future<TaskStatus> statusKilled;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusStarting))
.WillOnce(FutureArg<1>(&statusRunning))
- .WillOnce(FutureArg<1>(&statusUnhealthy))
- .WillOnce(FutureArg<1>(&statusHealthy))
- .WillOnce(FutureArg<1>(&statusUnhealthyAgain))
- .WillRepeatedly(Return()); // Ignore subsequent updates.
+ .WillOnce(FutureArg<1>(&status1))
+ .WillOnce(FutureArg<1>(&status2))
+ .WillOnce(FutureArg<1>(&status3))
+ .WillOnce(FutureArg<1>(&status4))
+ .WillOnce(FutureArg<1>(&statusKilled));
driver.launchTasks(offers.get()[0].id(), tasks);
@@ -993,132 +776,26 @@ TEST_F(HealthCheckTest, ROOT_DOCKER_DockerHealthStatusChange)
AWAIT_READY(statusRunning);
EXPECT_EQ(TASK_RUNNING, statusRunning->state());
- AWAIT_READY(statusUnhealthy);
- EXPECT_EQ(TASK_RUNNING, statusUnhealthy->state());
+ AWAIT_READY(status1);
+ EXPECT_EQ(TASK_RUNNING, status1->state());
EXPECT_EQ(
TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- statusUnhealthy->reason());
- EXPECT_FALSE(statusUnhealthy->healthy());
+ status1->reason());
+ EXPECT_FALSE(status1->healthy());
- AWAIT_READY(statusHealthy);
- EXPECT_EQ(TASK_RUNNING, statusHealthy->state());
+ AWAIT_READY(status2);
+ EXPECT_EQ(TASK_RUNNING, status2->state());
EXPECT_EQ(
TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- statusHealthy->reason());
- EXPECT_TRUE(statusHealthy->healthy());
+ status2->reason());
+ EXPECT_FALSE(status2->healthy());
- AWAIT_READY(statusUnhealthyAgain);
- EXPECT_EQ(TASK_RUNNING, statusUnhealthyAgain->state());
+ AWAIT_READY(status3);
+ EXPECT_EQ(TASK_RUNNING, status3->state());
EXPECT_EQ(
TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- statusUnhealthyAgain->reason());
- EXPECT_FALSE(statusUnhealthyAgain->healthy());
-
- // Check the temporary file created in host still
- // exists and the content has not changed.
- ASSERT_SOME(os::read(tmpPath));
- EXPECT_EQ("bar", os::read(tmpPath).get());
-
- Future<Option<ContainerTermination>> termination =
- containerizer.wait(containerId.get());
-
- driver.stop();
- driver.join();
-
- AWAIT_READY(termination);
- EXPECT_SOME(termination.get());
-
- agent.get()->terminate();
- agent->reset();
-
- Future<std::list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- // Cleanup all mesos launched containers.
- foreach (const Docker::Container& container, containers.get()) {
- AWAIT_READY_FOR(docker->rm(container.id, true), Seconds(30));
- }
-}
-
-
-// This test ensures that a task is killed if the number of maximum
-// health check failures is reached.
-TEST_F(HealthCheckTest, ConsecutiveFailures)
-{
- Try<Owned<cluster::Master>> master = StartMaster();
- ASSERT_SOME(master);
-
- Owned<MasterDetector> detector = master.get()->createDetector();
- Try<Owned<cluster::Slave>> agent = StartSlave(detector.get());
- ASSERT_SOME(agent);
-
- MockScheduler sched;
- MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
-
- EXPECT_CALL(sched, registered(&driver, _, _));
-
- Future<vector<Offer>> offers;
- EXPECT_CALL(sched, resourceOffers(&driver, _))
- .WillOnce(FutureArg<1>(&offers))
- .WillRepeatedly(Return()); // Ignore subsequent offers.
-
- driver.start();
-
- AWAIT_READY(offers);
- ASSERT_FALSE(offers->empty());
-
- vector<TaskInfo> tasks = populateTasks(
- SLEEP_COMMAND(120), "exit 1", offers.get()[0], 0, 4);
-
- // Expecting four unhealthy updates and one final kill update.
- Future<TaskStatus> statusStarting;
- Future<TaskStatus> statusRunning;
- Future<TaskStatus> status1;
- Future<TaskStatus> status2;
- Future<TaskStatus> status3;
- Future<TaskStatus> status4;
- Future<TaskStatus> statusKilled;
-
- EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&statusStarting))
- .WillOnce(FutureArg<1>(&statusRunning))
- .WillOnce(FutureArg<1>(&status1))
- .WillOnce(FutureArg<1>(&status2))
- .WillOnce(FutureArg<1>(&status3))
- .WillOnce(FutureArg<1>(&status4))
- .WillOnce(FutureArg<1>(&statusKilled));
-
- driver.launchTasks(offers.get()[0].id(), tasks);
-
- AWAIT_READY(statusStarting);
- EXPECT_EQ(TASK_STARTING, statusStarting->state());
-
- AWAIT_READY(statusRunning);
- EXPECT_EQ(TASK_RUNNING, statusRunning->state());
-
- AWAIT_READY(status1);
- EXPECT_EQ(TASK_RUNNING, status1->state());
- EXPECT_EQ(
- TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- status1->reason());
- EXPECT_FALSE(status1->healthy());
-
- AWAIT_READY(status2);
- EXPECT_EQ(TASK_RUNNING, status2->state());
- EXPECT_EQ(
- TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- status2->reason());
- EXPECT_FALSE(status2->healthy());
-
- AWAIT_READY(status3);
- EXPECT_EQ(TASK_RUNNING, status3->state());
- EXPECT_EQ(
- TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
- status3->reason());
- EXPECT_FALSE(status3->healthy());
+ status3->reason());
+ EXPECT_FALSE(status3->healthy());
AWAIT_READY(status4);
EXPECT_EQ(TASK_RUNNING, status4->state());
@@ -2229,28 +1906,27 @@ class DockerContainerizerHealthCheckTest
: public MesosTest,
public ::testing::WithParamInterface<NetworkInfo::Protocol>
{
-public:
- // Manually pull images, so that the pull time isn't counted
- // as part of the task launching.
- static void SetUpTestCase()
- {
- Future<Nothing> httpPull = pullDockerImage(DOCKER_HTTP_IMAGE);
- Future<Nothing> httpsPull = pullDockerImage(DOCKER_HTTPS_IMAGE);
-
- LOG(WARNING) << "Pulling " << string(DOCKER_HTTP_IMAGE) << " and "
- << string(DOCKER_HTTPS_IMAGE) << ". "
- << "This might take a while...";
-
- // The Windows image is ~200 MB, while the Linux image is ~2MB, so
- // hopefully this is enough time for the Windows image. There should
- // be some parallelism too, since we're pulling both simultaneously.
- AWAIT_READY_FOR(httpPull, Minutes(10));
- AWAIT_READY_FOR(httpsPull, Minutes(10));
- }
-
protected:
virtual void SetUp()
{
+ Future<std::tuple<Nothing, Nothing, Nothing>> pulls = process::collect(
+ pullDockerImage(DOCKER_TEST_IMAGE),
+ pullDockerImage(DOCKER_HTTP_IMAGE),
+ pullDockerImage(DOCKER_HTTPS_IMAGE));
+
+ // The pull should only need to happen once since we don't delete the
+ // image. So, we only log the warning once.
+ LOG_FIRST_N(WARNING, 1) << "Pulling " << string(DOCKER_TEST_IMAGE) << ", "
+ << string(DOCKER_HTTP_IMAGE) << " and "
+ << string(DOCKER_HTTPS_IMAGE) << ". "
+ << "This might take a while...";
+
+ // The Windows images are ~200 MB, while the Linux images are ~2MB, so
+ // hopefully this is enough time for the Windows images. There should
+ // be some parallelism too, since we're pulling them simultaneously and
+ // they share the same base Windows layer.
+ AWAIT_READY_FOR(pulls, Minutes(10));
+
createDockerIPv6UserNetwork();
}
@@ -2665,6 +2341,316 @@ TEST_P(
EXPECT_SOME(termination.get());
}
+// This test creates a healthy task using the Docker executor and
+// verifies that the healthy status is reported to the scheduler.
+TEST_F(DockerContainerizerHealthCheckTest, ROOT_DOCKER_DockerHealthyTask)
+{
+ Shared<Docker> docker(
+ new MockDocker(tests::flags.docker, tests::flags.docker_socket));
+
+ Try<Nothing> validateResult = docker->validateVersion(Version(1, 3, 0));
+ ASSERT_SOME(validateResult)
+ << "-------------------------------------------------------------\n"
+ << "We cannot run this test because of 'docker exec' command \n"
+ << "require docker version greater than '1.3.0'. You won't be \n"
+ << "able to use the docker exec method, but feel free to disable\n"
+ << "this test.\n"
+ << "-------------------------------------------------------------";
+
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags agentFlags = CreateSlaveFlags();
+
+ Fetcher fetcher(agentFlags);
+
+ Try<ContainerLogger*> logger =
+ ContainerLogger::create(agentFlags.container_logger);
+
+ ASSERT_SOME(logger);
+
+ MockDockerContainerizer containerizer(
+ agentFlags,
+ &fetcher,
+ Owned<ContainerLogger>(logger.get()),
+ docker);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> agent =
+ StartSlave(detector.get(), &containerizer, agentFlags);
+ ASSERT_SOME(agent);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers->empty());
+
+ TaskInfo task = createTask(offers.get()[0], DOCKER_SLEEP_CMD(120));
+
+ // TODO(tnachen): Use local image to test if possible.
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+ containerInfo.mutable_docker()->set_image(DOCKER_TEST_IMAGE);
+
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ HealthCheck healthCheck;
+ healthCheck.set_type(HealthCheck::COMMAND);
+ healthCheck.mutable_command()->set_value("exit 0");
+ healthCheck.set_delay_seconds(0);
+ healthCheck.set_interval_seconds(0);
+ healthCheck.set_grace_period_seconds(0);
+
+ task.mutable_health_check()->CopyFrom(healthCheck);
+
+ Future<ContainerID> containerId;
+ EXPECT_CALL(containerizer, launch(_, _, _, _))
+ .WillOnce(DoAll(
+ FutureArg<0>(&containerId),
+ Invoke(&containerizer, &MockDockerContainerizer::_launch)));
+
+ Future<TaskStatus> statusStarting;
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusHealthy;
+
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusStarting))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusHealthy));
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY(containerId);
+
+ AWAIT_READY(statusStarting);
+ EXPECT_EQ(TASK_STARTING, statusStarting->state());
+
+ AWAIT_READY(statusRunning);
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ AWAIT_READY(statusHealthy);
+ EXPECT_EQ(TASK_RUNNING, statusHealthy->state());
+ EXPECT_EQ(
+ TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
+ statusHealthy->reason());
+ EXPECT_TRUE(statusHealthy->has_healthy());
+ EXPECT_TRUE(statusHealthy->healthy());
+
+ Future<Option<ContainerTermination>> termination =
+ containerizer.wait(containerId.get());
+
+ driver.stop();
+ driver.join();
+
+ AWAIT_READY(termination);
+ EXPECT_SOME(termination.get());
+
+ agent.get()->terminate();
+ agent->reset();
+
+ Future<std::list<Docker::Container>> containers =
+ docker->ps(true, slave::DOCKER_NAME_PREFIX);
+
+ AWAIT_READY(containers);
+
+ // Cleanup all mesos launched containers.
+ foreach (const Docker::Container& container, containers.get()) {
+ AWAIT_READY_FOR(docker->rm(container.id, true), Seconds(30));
+ }
+}
+
+// This test creates a task that uses the Docker executor and whose
+// health flaps. It then verifies that the health status updates are
+// sent to the framework scheduler.
+TEST_F(DockerContainerizerHealthCheckTest, ROOT_DOCKER_DockerHealthStatusChange)
+{
+ Shared<Docker> docker(
+ new MockDocker(tests::flags.docker, tests::flags.docker_socket));
+
+ Try<Nothing> validateResult = docker->validateVersion(Version(1, 3, 0));
+ ASSERT_SOME(validateResult)
+ << "-------------------------------------------------------------\n"
+ << "We cannot run this test because of 'docker exec' command \n"
+ << "require docker version greater than '1.3.0'. You won't be \n"
+ << "able to use the docker exec method, but feel free to disable\n"
+ << "this test.\n"
+ << "-------------------------------------------------------------";
+
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags agentFlags = CreateSlaveFlags();
+
+ Fetcher fetcher(agentFlags);
+
+ Try<ContainerLogger*> logger =
+ ContainerLogger::create(agentFlags.container_logger);
+
+ ASSERT_SOME(logger);
+
+ MockDockerContainerizer containerizer(
+ agentFlags,
+ &fetcher,
+ Owned<ContainerLogger>(logger.get()),
+ docker);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> agent =
+ StartSlave(detector.get(), &containerizer, agentFlags);
+ ASSERT_SOME(agent);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers->empty());
+
+ TaskInfo task = createTask(offers.get()[0], DOCKER_SLEEP_CMD(120));
+
+ // TODO(tnachen): Use local image to test if possible.
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+ containerInfo.mutable_docker()->set_image(DOCKER_TEST_IMAGE);
+
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ // Create a temporary file in host and then we could this file to
+ // make sure the health check command is run in docker container.
+ string tmpPath = path::join(os::getcwd(), "foobar");
+ ASSERT_SOME(os::write(tmpPath, "bar"));
+
+ // This command fails every other invocation.
+ // For all runs i in Nat0, the following case i % 2 applies:
+ //
+ // Case 0:
+ // - Attempt to remove the nonexistent temporary file.
+ // - Create the temporary file.
+ // - Exit with a non-zero status.
+ //
+ // Case 1:
+ // - Remove the temporary file.
+#ifdef __WINDOWS__
+ const string healthCheckCmd =
+ "pwsh -Command "
+ "Remove-Item -ErrorAction SilentlyContinue \"" + tmpPath + "\"; "
+ "if (-Not $?) { "
+ "New-Item -ItemType Directory -Force \"" + os::getcwd() + "\"; "
+ "Set-Content -Path \"" + tmpPath + "\" -Value foo; "
+ "exit 1 "
+ "}";
+#else
+ const string healthCheckCmd =
+ "rm " + tmpPath + " || "
+ "(mkdir -p " + os::getcwd() + " && echo foo >" + tmpPath + " && exit 1)";
+#endif // __WINDOWS__
+
+ HealthCheck healthCheck;
+ healthCheck.set_type(HealthCheck::COMMAND);
+ healthCheck.mutable_command()->set_value(healthCheckCmd);
+ healthCheck.set_delay_seconds(0);
+ healthCheck.set_interval_seconds(0);
+ healthCheck.set_grace_period_seconds(0);
+
+ task.mutable_health_check()->CopyFrom(healthCheck);
+
+ Future<ContainerID> containerId;
+ EXPECT_CALL(containerizer, launch(_, _, _, _))
+ .WillOnce(DoAll(
+ FutureArg<0>(&containerId),
+ Invoke(&containerizer, &MockDockerContainerizer::_launch)));
+
+ Future<TaskStatus> statusStarting;
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusUnhealthy;
+ Future<TaskStatus> statusHealthy;
+ Future<TaskStatus> statusUnhealthyAgain;
+
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusStarting))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusUnhealthy))
+ .WillOnce(FutureArg<1>(&statusHealthy))
+ .WillOnce(FutureArg<1>(&statusUnhealthyAgain))
+ .WillRepeatedly(Return()); // Ignore subsequent updates.
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY(statusStarting);
+ EXPECT_EQ(TASK_STARTING, statusStarting->state());
+
+ AWAIT_READY(statusRunning);
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ AWAIT_READY(statusUnhealthy);
+ EXPECT_EQ(TASK_RUNNING, statusUnhealthy->state());
+ EXPECT_EQ(
+ TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
+ statusUnhealthy->reason());
+ EXPECT_FALSE(statusUnhealthy->healthy());
+
+ AWAIT_READY(statusHealthy);
+ EXPECT_EQ(TASK_RUNNING, statusHealthy->state());
+ EXPECT_EQ(
+ TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
+ statusHealthy->reason());
+ EXPECT_TRUE(statusHealthy->healthy());
+
+ AWAIT_READY(statusUnhealthyAgain);
+ EXPECT_EQ(TASK_RUNNING, statusUnhealthyAgain->state());
+ EXPECT_EQ(
+ TaskStatus::REASON_TASK_HEALTH_CHECK_STATUS_UPDATED,
+ statusUnhealthyAgain->reason());
+ EXPECT_FALSE(statusUnhealthyAgain->healthy());
+
+ // Check the temporary file created in host still
+ // exists and the content has not changed.
+ ASSERT_SOME(os::read(tmpPath));
+ EXPECT_EQ("bar", os::read(tmpPath).get());
+
+ Future<Option<ContainerTermination>> termination =
+ containerizer.wait(containerId.get());
+
+ driver.stop();
+ driver.join();
+
+ AWAIT_READY(termination);
+ EXPECT_SOME(termination.get());
+
+ agent.get()->terminate();
+ agent->reset();
+
+ Future<std::list<Docker::Container>> containers =
+ docker->ps(true, slave::DOCKER_NAME_PREFIX);
+
+ AWAIT_READY(containers);
+
+ // Cleanup all mesos launched containers.
+ foreach (const Docker::Container& container, containers.get()) {
+ AWAIT_READY_FOR(docker->rm(container.id, true), Seconds(30));
+ }
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {