You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/05/25 07:31:36 UTC
[01/20] mesos git commit: Recover docker containers that launched in
containers.
Repository: mesos
Updated Branches:
refs/heads/master 9c5c92724 -> 3baa60965
Recover docker containers that launched in containers.
Review: https://reviews.apache.org/r/29336
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7fee619c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7fee619c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7fee619c
Branch: refs/heads/master
Commit: 7fee619cf0f3097aa1c473759831c38b8441d3c1
Parents: 597b8a2
Author: Timothy Chen <tn...@gmail.com>
Authored: Thu Dec 4 07:59:45 2014 +0000
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 232 ++++++++++++++++++--------------
src/slave/containerizer/docker.hpp | 2 +-
2 files changed, 131 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7fee619c/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 0154c4b..50a5b72 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -113,6 +113,11 @@ Option<ContainerID> parse(const Docker::Container& container)
ContainerID id;
id.set_value(parts[1]);
return id;
+ } else if (parts.size() == 3) {
+ // We found a executor or log container.
+ ContainerID id;
+ id.set_value(parts[2]);
+ return id;
}
}
@@ -120,6 +125,26 @@ Option<ContainerID> parse(const Docker::Container& container)
}
+// Launches a docker wait process on given container name.
+// Returns the wait process pid.
+Try<pid_t> launchWaitProcess(const string& docker, const string& name)
+{
+ string command = "exit `" + docker + " wait " + name + "`";
+
+ Try<Subprocess> wait = subprocess(
+ command,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"));
+
+ if (wait.isError()) {
+ return Error("Unable to launch docker wait on executor: " + wait.error());
+ }
+
+ return wait.get().pid();
+}
+
+
Try<DockerContainerizer*> DockerContainerizer::create(
const Flags& flags,
Fetcher* fetcher)
@@ -441,16 +466,21 @@ Future<Nothing> DockerContainerizerProcess::recover(
const Option<SlaveState>& state)
{
LOG(INFO) << "Recovering Docker containers";
- // Get the list of all Docker containers (running and exited) in
- // order to remove any orphans and reconcile checkpointed executors.
- // TODO(tnachen): Remove this when we expect users to have already
- // upgraded to 0.23.
- return docker->ps(true, DOCKER_NAME_PREFIX)
- .then(defer(self(), &Self::_recover, state, lambda::_1));
+ if (state.isSome()) {
+ // Get the list of all Docker containers (running and exited) in
+ // order to remove any orphans and reconcile checkpointed executors.
+ // TODO(tnachen): Remove this when we expect users to have already
+ // upgraded to 0.23.
+ return docker->ps(true, DOCKER_NAME_PREFIX + state.get().id.value())
+ .then(defer(self(), &Self::_recover, state.get(), lambda::_1));
+ }
+
+return Nothing();
}
+
Future<Nothing> DockerContainerizerProcess::_recover(
- const Option<SlaveState>& state,
+ const SlaveState& state,
const list<Docker::Container>& _containers)
{
// Although the slave checkpoints executor pids, before 0.23
@@ -470,107 +500,105 @@ Future<Nothing> DockerContainerizerProcess::_recover(
}
}
- if (state.isSome()) {
- // Collection of pids that we've started reaping in order to
- // detect very unlikely duplicate scenario (see below).
- hashmap<ContainerID, pid_t> pids;
-
- foreachvalue (const FrameworkState& framework, state.get().frameworks) {
- foreachvalue (const ExecutorState& executor, framework.executors) {
- if (executor.info.isNone()) {
- LOG(WARNING) << "Skipping recovery of executor '" << executor.id
- << "' of framework " << framework.id
- << " because its info could not be recovered";
- continue;
- }
-
- if (executor.latest.isNone()) {
- LOG(WARNING) << "Skipping recovery of executor '" << executor.id
- << "' of framework " << framework.id
- << " because its latest run could not be recovered";
- continue;
- }
-
- // We are only interested in the latest run of the executor!
- const ContainerID& containerId = executor.latest.get();
- Option<RunState> run = executor.runs.get(containerId);
- CHECK_SOME(run);
- CHECK_SOME(run.get().id);
- CHECK_EQ(containerId, run.get().id.get());
-
- // We need the pid so the reaper can monitor the executor so
- // skip this executor if it's not present. This is not an
- // error because the slave will try to wait on the container
- // which will return a failed Termination and everything will
- // get cleaned up.
- if (!run.get().forkedPid.isSome()) {
- continue;
- }
-
- if (run.get().completed) {
- VLOG(1) << "Skipping recovery of executor '" << executor.id
+ // Collection of pids that we've started reaping in order to
+ // detect very unlikely duplicate scenario (see below).
+ hashmap<ContainerID, pid_t> pids;
+
+ foreachvalue (const FrameworkState& framework, state.get().frameworks) {
+ foreachvalue (const ExecutorState& executor, framework.executors) {
+ if (executor.info.isNone()) {
+ LOG(WARNING) << "Skipping recovery of executor '" << executor.id
+ << "' of framework " << framework.id
+ << " because its info could not be recovered";
+ continue;
+ }
+
+ if (executor.latest.isNone()) {
+ LOG(WARNING) << "Skipping recovery of executor '" << executor.id
+ << "' of framework " << framework.id
+ << " because its latest run could not be recovered";
+ continue;
+ }
+
+ // We are only interested in the latest run of the executor!
+ const ContainerID& containerId = executor.latest.get();
+ Option<RunState> run = executor.runs.get(containerId);
+ CHECK_SOME(run);
+ CHECK_SOME(run.get().id);
+ CHECK_EQ(containerId, run.get().id.get());
+
+ // We need the pid so the reaper can monitor the executor so
+ // skip this executor if it's not present. This is not an
+ // error because the slave will try to wait on the container
+ // which will return a failed Termination and everything will
+ // get cleaned up.
+ if (!run.get().forkedPid.isSome()) {
+ continue;
+ }
+
+ if (run.get().completed) {
+ VLOG(1) << "Skipping recovery of executor '" << executor.id
+ << "' of framework " << framework.id
+ << " because its latest run "
+ << containerId << " is completed";
+ continue;
+ }
+
+ const ExecutorInfo executorInfo = executor.info.get();
+ if (executorInfo.has_container() &&
+ executorInfo.container().type() != ContainerInfo::DOCKER) {
+ LOG(INFO) << "Skipping recovery of executor '" << executor.id
<< "' of framework " << framework.id
- << " because its latest run "
- << containerId << " is completed";
- continue;
- }
-
- const ExecutorInfo executorInfo = executor.info.get();
- if (executorInfo.has_container() &&
- executorInfo.container().type() != ContainerInfo::DOCKER) {
- LOG(INFO) << "Skipping recovery of executor '" << executor.id
- << "' of framework " << framework.id
- << " because it was not launched from docker containerizer";
- continue;
- }
-
- if (!executorInfo.has_container() &&
- !existingContainers.contains(containerId)) {
- LOG(INFO) << "Skipping recovery of executor '" << executor.id
- << "' of framework " << framework.id
- << " because its executor is not marked as docker "
- << "and the docker container doesn't exist";
- continue;
- }
-
- LOG(INFO) << "Recovering container '" << containerId
- << "' for executor '" << executor.id
- << "' of framework " << framework.id;
-
- // Create and store a container.
- Container* container = new Container(containerId);
- containers_[containerId] = container;
- container->slaveId = state.get().id;
- container->state = Container::RUNNING;
-
- pid_t pid = run.get().forkedPid.get();
-
- container->status.set(process::reap(pid));
-
- container->status.future().get()
- .onAny(defer(self(), &Self::reaped, containerId));
-
- if (pids.containsValue(pid)) {
- // This should (almost) never occur. There is the
- // possibility that a new executor is launched with the same
- // pid as one that just exited (highly unlikely) and the
- // slave dies after the new executor is launched but before
- // it hears about the termination of the earlier executor
- // (also unlikely).
- return Failure(
- "Detected duplicate pid " + stringify(pid) +
- " for container " + stringify(containerId));
- }
-
- pids.put(containerId, pid);
+ << " because it was not launched from docker containerizer";
+ continue;
}
- }
- if (flags.docker_kill_orphans) {
- return __recover(_containers);
+ if (!executorInfo.has_container() &&
+ !existingContainers.contains(containerId)) {
+ LOG(INFO) << "Skipping recovery of executor '" << executor.id
+ << "' of framework " << framework.id
+ << " because its executor is not marked as docker "
+ << "and the docker container doesn't exist";
+ continue;
+ }
+
+ LOG(INFO) << "Recovering container '" << containerId
+ << "' for executor '" << executor.id
+ << "' of framework " << framework.id;
+
+ // Create and store a container.
+ Container* container = new Container(containerId);
+ containers_[containerId] = container;
+ container->slaveId = state.get().id;
+ container->state = Container::RUNNING;
+
+ pid_t pid = run.get().forkedPid.get();
+
+ container->status.set(process::reap(pid));
+
+ container->status.future().get()
+ .onAny(defer(self(), &Self::reaped, containerId));
+
+ if (pids.containsValue(pid)) {
+ // This should (almost) never occur. There is the
+ // possibility that a new executor is launched with the same
+ // pid as one that just exited (highly unlikely) and the
+ // slave dies after the new executor is launched but before
+ // it hears about the termination of the earlier executor
+ // (also unlikely).
+ return Failure(
+ "Detected duplicate pid " + stringify(pid) +
+ " for container " + stringify(containerId));
+ }
+
+ pids.put(containerId, pid);
}
}
+ if (flags.docker_kill_orphans) {
+ return __recover(_containers);
+ }
+
return Nothing();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/7fee619c/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index 16c7775..0eda1c0 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -182,7 +182,7 @@ private:
pid_t pid);
process::Future<Nothing> _recover(
- const Option<state::SlaveState>& state,
+ const state::SlaveState& state,
const std::list<Docker::Container>& containers);
process::Future<Nothing> __recover(
[07/20] mesos git commit: Fix docker container naming and tests.
Posted by tn...@apache.org.
Fix docker container naming and tests.
Review: https://reviews.apache.org/r/29335
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/597b8a25
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/597b8a25
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/597b8a25
Branch: refs/heads/master
Commit: 597b8a2544463f368833616c10f70ecb2a846c17
Parents: 280e465
Author: Timothy Chen <tn...@gmail.com>
Authored: Thu Dec 4 02:15:33 2014 +0000
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 24 ++++----
src/slave/containerizer/docker.hpp | 11 +++-
src/tests/docker_containerizer_tests.cpp | 84 +++++++++++++++------------
3 files changed, 68 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/597b8a25/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index c996afe..0154c4b 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -75,6 +75,9 @@ using state::RunState;
const string DOCKER_NAME_PREFIX = "mesos-";
// Declared in header, see explanation there.
+const string DOCKER_NAME_SEPERATOR = ".";
+
+// Declared in header, see explanation there.
const string DOCKER_SYMLINK_DIRECTORY = "docker/links";
// Parse the ContainerID from a Docker container and return None if
@@ -92,20 +95,20 @@ Option<ContainerID> parse(const Docker::Container& container)
}
if (name.isSome()) {
- // For Mesos version <= 0.21.0, the docker container name format
- // was DOCKER_NAME_PREFIX + containerId, and starting with 0.22.0
- // it is changed to DOCKER_NAME_PREFIX + slaveId + "/" +
- // containerId.
- // To still be backward compatible during upgrade, we still need
- // to load the previous format.
+ // For Mesos version < 0.23.0, the docker container name format
+ // was DOCKER_NAME_PREFIX + containerId, and starting with 0.23.0
+ // it is changed to DOCKER_NAME_PREFIX + slaveId +
+ // DOCKER_NAME_SEPERATOR + containerId.
+ // To be backward compatible during upgrade, we still to support
+ // the previous format.
// TODO(tnachen): Remove this check after deprecation cycle.
- if (!strings::contains(name.get(), "/")) {
+ if (!strings::contains(name.get(), DOCKER_NAME_SEPERATOR)) {
ContainerID id;
id.set_value(name.get());
return id;
}
- vector<string> parts = strings::split(name.get(), "/");
+ vector<string> parts = strings::split(name.get(), DOCKER_NAME_SEPERATOR);
if (parts.size() == 2) {
ContainerID id;
id.set_value(parts[1]);
@@ -537,7 +540,7 @@ Future<Nothing> DockerContainerizerProcess::_recover(
// Create and store a container.
Container* container = new Container(containerId);
containers_[containerId] = container;
-
+ container->slaveId = state.get().id;
container->state = Container::RUNNING;
pid_t pid = run.get().forkedPid.get();
@@ -1063,9 +1066,6 @@ Future<bool> DockerContainerizerProcess::_______launch(
container->status.future().get()
.onAny(defer(self(), &Self::reaped, containerId));
- // TODO(benh): Check failure of Docker::logs.
- docker->logs(container->name(), container->directory);
-
return true;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/597b8a25/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index a54e0d4..16c7775 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -35,6 +35,10 @@ namespace slave {
// created by Mesos from those created manually.
extern const std::string DOCKER_NAME_PREFIX;
+// Seperator used to compose docker container name, which is made up
+// of slave id and container id.
+extern const std::string DOCKER_NAME_SEPERATOR;
+
// Directory that stores all the symlinked sandboxes that is mapped
// into Docker containers. This is a relative directory that will
// joined with the slave path. Only sandbox paths that contains a
@@ -341,17 +345,18 @@ private:
std::string name()
{
- return DOCKER_NAME_PREFIX + slaveId.value() + "/" + stringify(id);
+ return DOCKER_NAME_PREFIX + slaveId.value() + DOCKER_NAME_SEPERATOR +
+ stringify(id);
}
std::string logName()
{
- return name() + "/log";
+ return name() + DOCKER_NAME_SEPERATOR + "log";
}
std::string executorName()
{
- return name() + "/executor";
+ return name() + DOCKER_NAME_SEPERATOR + "executor";
}
std::string image() const
http://git-wip-us.apache.org/repos/asf/mesos/blob/597b8a25/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 36ccc6a..3fa663e 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -119,7 +119,8 @@ public:
stop,
process::Future<Nothing>(
const string&,
- const Duration&, bool));
+ const Duration&,
+ bool));
process::Future<Nothing> _run(
const mesos::ContainerInfo& containerInfo,
@@ -168,11 +169,20 @@ public:
class DockerContainerizerTest : public MesosTest
{
public:
+ static string containerName(
+ const SlaveID& slaveId,
+ const ContainerID& containerId)
+ {
+ return slave::DOCKER_NAME_PREFIX + slaveId.value() +
+ slave::DOCKER_NAME_SEPERATOR + containerId.value();
+ }
+
static bool exists(
const list<Docker::Container>& containers,
+ const SlaveID& slaveId,
const ContainerID& containerId)
{
- string expectedName = slave::DOCKER_NAME_PREFIX + stringify(containerId);
+ string expectedName = containerName(slaveId, containerId);
foreach (const Docker::Container& container, containers) {
// Docker inspect name contains an extra slash in the beginning.
@@ -213,7 +223,7 @@ public:
// Cleanup all mesos launched containers.
foreach (const Docker::Container& container, containers.get()) {
- AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
+ ASSERT_TRUE(docker.get()->rm(container.id, true).await(Seconds(30)));
}
delete docker.get();
@@ -435,6 +445,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -489,7 +501,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
AWAIT_READY(containers);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -565,6 +577,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -620,7 +634,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
AWAIT_READY(containers);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -691,6 +705,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -737,7 +753,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
ASSERT_TRUE(containers.get().size() > 0);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -1067,6 +1083,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -1107,8 +1125,9 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- string containerName = slave::DOCKER_NAME_PREFIX + containerId.get().value();
- Future<Docker::Container> container = docker->inspect(containerName);
+ string name = containerName(slaveId, containerId.get());
+
+ Future<Docker::Container> container = docker->inspect(name);
AWAIT_READY(container);
@@ -1193,11 +1212,20 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
MockDockerContainerizer dockerContainerizer(flags, &fetcher, docker);
+ SlaveID slaveId;
+ slaveId.set_value("s1");
ContainerID containerId;
containerId.set_value("c1");
ContainerID reapedContainerId;
reapedContainerId.set_value("c2");
+ string container1 = containerName(slaveId, containerId);
+ string container2 = containerName(slaveId, reapedContainerId);
+
+ // Clean up artifacts if containers still exists.
+ ASSERT_TRUE(docker->rm(container1, true).await(Seconds(30)));
+ ASSERT_TRUE(docker->rm(container2, true).await(Seconds(30)));
+
Resources resources = Resources::parse("cpus:1;mem:512").get();
ContainerInfo containerInfo;
@@ -1214,7 +1242,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
docker->run(
containerInfo,
commandInfo,
- slave::DOCKER_NAME_PREFIX + stringify(containerId),
+ container1,
flags.work_dir,
flags.docker_sandbox_directory,
resources);
@@ -1223,7 +1251,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
docker->run(
containerInfo,
commandInfo,
- slave::DOCKER_NAME_PREFIX + stringify(reapedContainerId),
+ container2,
flags.work_dir,
flags.docker_sandbox_directory,
resources);
@@ -1232,6 +1260,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
AWAIT_READY(d2);
SlaveState slaveState;
+ slaveState.id = slaveId;
FrameworkState frameworkState;
ExecutorID execId;
@@ -1243,18 +1272,12 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
execState.latest = containerId;
Try<process::Subprocess> wait =
- process::subprocess(
- tests::flags.docker + " wait " +
- slave::DOCKER_NAME_PREFIX +
- stringify(containerId));
+ process::subprocess(tests::flags.docker + " wait " + container1);
ASSERT_SOME(wait);
Try<process::Subprocess> reaped =
- process::subprocess(
- tests::flags.docker + " wait " +
- slave::DOCKER_NAME_PREFIX +
- stringify(reapedContainerId));
+ process::subprocess(tests::flags.docker + " wait " + container2);
ASSERT_SOME(reaped);
@@ -1268,15 +1291,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
slaveState.frameworks.put(frameworkId, frameworkState);
- // We need to capture and await on the stop future so that we can
- // ensure there is no child process at the end of the test.
- // The stop future is being awaited at teardown.
- Future<Nothing> stop;
- EXPECT_CALL(*mockDocker, stop(_, _, _))
- .WillOnce(FutureResult(
- &stop,
- Invoke((MockDocker*) docker.get(), &MockDocker::_stop)));
-
Future<Nothing> recover = dockerContainerizer.recover(slaveState);
AWAIT_READY(recover);
@@ -1288,14 +1302,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
AWAIT_FAILED(dockerContainerizer.wait(reapedContainerId));
- dockerContainerizer.destroy(containerId);
-
- AWAIT_READY(termination);
-
AWAIT_READY(reaped.get().status());
- AWAIT_READY_FOR(stop, Seconds(30));
-
Shutdown();
}
@@ -1935,6 +1943,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -2015,7 +2025,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
AWAIT_READY(containers);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer2->wait(containerId.get());
@@ -2208,7 +2218,7 @@ TEST_F(DockerContainerizerTest,
AWAIT_READY(containers);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId.get(), containerId.get()));
driver.stop();
driver.join();
@@ -2417,6 +2427,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("test:colon");
@@ -2463,7 +2475,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
ASSERT_TRUE(containers.get().size() > 0);
- ASSERT_TRUE(exists(containers.get(), containerId.get()));
+ ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
[10/20] mesos git commit: Add executor for docker containerizer
Posted by tn...@apache.org.
Add executor for docker containerizer
Review: https://reviews.apache.org/r/29329
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/24415b72
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/24415b72
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/24415b72
Branch: refs/heads/master
Commit: 24415b72ef7eecfe3dd9022e5d58b6087aa10a6a
Parents: 8f31db8
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Nov 25 18:25:00 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/Makefile.am | 5 +
src/docker/executor.cpp | 358 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 363 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/24415b72/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 34755cf..1e56ae5 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -822,6 +822,11 @@ mesos_usage_SOURCES = usage/main.cpp
mesos_usage_CPPFLAGS = $(MESOS_CPPFLAGS)
mesos_usage_LDADD = libmesos.la $(LDADD)
+pkglibexec_PROGRAMS += mesos-docker-executor
+mesos_docker_executor_SOURCES = docker/executor.cpp
+mesos_docker_executor_CPPFLAGS = $(MESOS_CPPFLAGS)
+mesos_docker_executor_LDADD = libmesos.la
+
bin_PROGRAMS += mesos-log
mesos_log_SOURCES = log/main.cpp
mesos_log_CPPFLAGS = $(MESOS_CPPFLAGS)
http://git-wip-us.apache.org/repos/asf/mesos/blob/24415b72/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
new file mode 100644
index 0000000..1a5ab86
--- /dev/null
+++ b/src/docker/executor.cpp
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+
+#include <string>
+
+#include <mesos/mesos.hpp>
+#include <mesos/executor.hpp>
+
+#include <process/process.hpp>
+#include <process/protobuf.hpp>
+#include <process/subprocess.hpp>
+#include <process/reap.hpp>
+
+#include <stout/flags.hpp>
+#include <stout/os.hpp>
+
+#include "common/status_utils.hpp"
+
+#include "logging/logging.hpp"
+
+using std::cerr;
+using std::cout;
+using std::endl;
+using std::string;
+
+namespace mesos {
+namespace internal {
+
+using namespace mesos;
+using namespace process;
+
+
+// Executor that is responsible to execute a docker container, and
+// redirect log output to configured stdout and stderr files.
+// Similar to the CommandExecutor, it is only responsible to launch
+// one container and exits afterwards.
+// The executor also assumes it is launched from the
+// DockerContainerizer, which already calls setsid before launching
+// the executor.
+class DockerExecutorProcess : public ProtobufProcess<DockerExecutorProcess>
+{
+public:
+ DockerExecutorProcess(const string& docker, const string& container)
+ : launched(false),
+ docker(docker),
+ container(container),
+ pid(-1) {}
+
+ virtual ~DockerExecutorProcess() {}
+
+ void registered(
+ ExecutorDriver* _driver,
+ const ExecutorInfo& executorInfo,
+ const FrameworkInfo& frameworkInfo,
+ const SlaveInfo& slaveInfo)
+ {
+ cout << "Registered docker executor on " << slaveInfo.hostname() << endl;
+ driver = _driver;
+ }
+
+ void reregistered(
+ ExecutorDriver* driver,
+ const SlaveInfo& slaveInfo)
+ {
+ cout << "Re-registered docker executor on " << slaveInfo.hostname() << endl;
+ }
+
+ void disconnected(ExecutorDriver* driver) {}
+
+ void launchTask(ExecutorDriver* driver, const TaskInfo& task)
+ {
+ if (launched) {
+ TaskStatus status;
+ status.mutable_task_id()->MergeFrom(task.task_id());
+ status.set_state(TASK_FAILED);
+ status.set_message(
+ "Attempted to run multiple tasks using a \"docker\" executor");
+
+ driver->sendStatusUpdate(status);
+ return;
+ }
+
+ cout << "Starting task " << task.task_id().value() << endl;
+
+ Try<Subprocess> subprocess = process::subprocess(
+ "exit `" + docker + " wait + " + container + "`",
+ Subprocess::PATH("/dev/null"),
+ Subprocess::FD(STDOUT_FILENO),
+ Subprocess::FD(STDERR_FILENO));
+
+ if (subprocess.isError()) {
+ cerr << "Couldn't launch docker wait process: " << subprocess.error();
+ abort();
+ }
+
+ pid = subprocess.get().pid();
+
+ process::reap(pid)
+ .onAny(defer(self(),
+ &Self::reaped,
+ driver,
+ task.task_id(),
+ pid,
+ lambda::_1));
+
+ TaskStatus status;
+ status.mutable_task_id()->MergeFrom(task.task_id());
+ status.set_state(TASK_RUNNING);
+ driver->sendStatusUpdate(status);
+
+ launched = true;
+ }
+
+ void killTask(ExecutorDriver* driver, const TaskID& taskId)
+ {
+ shutdown(driver);
+ }
+
+ void frameworkMessage(ExecutorDriver* driver, const string& data) {}
+
+ void shutdown(ExecutorDriver* driver)
+ {
+ cout << "Shutting down" << endl;
+
+ if (pid > 0 && !killed) {
+ ::kill(pid, SIGKILL);
+ killed = true;
+ }
+ }
+
+ virtual void error(ExecutorDriver* driver, const string& message) {}
+
+private:
+ void reaped(
+ ExecutorDriver* driver,
+ const TaskID& taskId,
+ pid_t pid,
+ const Future<Option<int>>& status)
+ {
+ TaskState state;
+ string message;
+ if (!status.isReady()) {
+ state = TASK_FAILED;
+ message =
+ "Failed to get exit status for Docker executor: " +
+ (status.isFailed() ? status.failure() : "future discarded");
+ } else if (status.get().isNone()) {
+ state = TASK_FAILED;
+ message = "Failed to get exit status for Docker executor";
+ } else {
+ int s = status.get().get();
+
+ // Subprocess status is gathered from waitpid, therefore we can
+ // get the exit status from WIFEXITED.
+ CHECK(WIFEXITED(s) || WIFSIGNALED(s)) << "status code: " << s;
+
+ if (WIFEXITED(s) && WEXITSTATUS(s) == 0) {
+ state = TASK_FINISHED;
+ } else if (killed) {
+ // Send TASK_KILLED if the task was killed as a result of
+ // killTask() or shutdown().
+ state = TASK_KILLED;
+ } else {
+ state = TASK_FAILED;
+ }
+
+ message = "Docker " + WSTRINGIFY(s);
+ }
+
+ cout << message << " (pid: " << pid << ")" << endl;
+
+ TaskStatus taskStatus;
+ taskStatus.mutable_task_id()->MergeFrom(taskId);
+ taskStatus.set_state(state);
+ taskStatus.set_message(message);
+
+ driver->sendStatusUpdate(taskStatus);
+
+ // A hack for now ... but we need to wait until the status update
+ // is sent to the slave before we shut ourselves down.
+ os::sleep(Seconds(1));
+ driver->stop();
+ }
+
+
+ bool launched;
+ string docker;
+ string container;
+ pid_t pid;
+ bool killed;
+ Option<ExecutorDriver*> driver;
+};
+
+
+class DockerExecutor : public Executor
+{
+public:
+ DockerExecutor(const string& docker, const string& container)
+ {
+ process = new DockerExecutorProcess(docker, container);
+ spawn(process);
+ }
+
+ virtual ~DockerExecutor()
+ {
+ terminate(process);
+ wait(process);
+ delete process;
+ }
+
+ virtual void registered(
+ ExecutorDriver* driver,
+ const ExecutorInfo& executorInfo,
+ const FrameworkInfo& frameworkInfo,
+ const SlaveInfo& slaveInfo)
+ {
+ dispatch(process,
+ &DockerExecutorProcess::registered,
+ driver,
+ executorInfo,
+ frameworkInfo,
+ slaveInfo);
+ }
+
+ virtual void reregistered(
+ ExecutorDriver* driver,
+ const SlaveInfo& slaveInfo)
+ {
+ dispatch(process,
+ &DockerExecutorProcess::reregistered,
+ driver,
+ slaveInfo);
+ }
+
+ virtual void disconnected(ExecutorDriver* driver)
+ {
+ dispatch(process, &DockerExecutorProcess::disconnected, driver);
+ }
+
+ virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task)
+ {
+ dispatch(process, &DockerExecutorProcess::launchTask, driver, task);
+ }
+
+ virtual void killTask(ExecutorDriver* driver, const TaskID& taskId)
+ {
+ dispatch(process, &DockerExecutorProcess::killTask, driver, taskId);
+ }
+
+ virtual void frameworkMessage(ExecutorDriver* driver, const string& data)
+ {
+ dispatch(process, &DockerExecutorProcess::frameworkMessage, driver, data);
+ }
+
+ virtual void shutdown(ExecutorDriver* driver)
+ {
+ dispatch(process, &DockerExecutorProcess::shutdown, driver);
+ }
+
+ virtual void error(ExecutorDriver* driver, const string& data)
+ {
+ dispatch(process, &DockerExecutorProcess::error, driver, data);
+ }
+
+private:
+ DockerExecutorProcess* process;
+};
+
+} // namespace internal {
+} // namespace mesos {
+
+
+void usage(const char* argv0, const flags::FlagsBase& flags)
+{
+ cerr << "Usage: " << os::basename(argv0).get() << " [...]" << endl
+ << endl
+ << "Supported options:" << endl
+ << flags.usage();
+}
+
+
+class Flags : public flags::FlagsBase
+{
+public:
+ Flags()
+ {
+ add(&Flags::container,
+ "container",
+ "The name of the docker container to wait on.");
+
+ add(&Flags::docker,
+ "docker",
+ "The path to the docker cli executable.");
+ }
+
+ Option<string> container;
+ Option<string> docker;
+};
+
+
+int main(int argc, char** argv)
+{
+ Flags flags;
+
+ bool help;
+ flags.add(&help,
+ "help",
+ "Prints this help message",
+ false);
+
+ // Load flags from environment and command line.
+ Try<Nothing> load = flags.load(None(), &argc, &argv);
+
+ if (load.isError()) {
+ cerr << load.error() << endl;
+ usage(argv[0], flags);
+ return -1;
+ }
+
+ if (help) {
+ usage(argv[0], flags);
+ return -1;
+ }
+
+ if (flags.docker.isNone()) {
+ LOG(WARNING) << "Expected docker executable path";
+ usage(argv[0], flags);
+ return 0;
+ }
+
+ if (flags.container.isNone()) {
+ LOG(WARNING) << "Expected container name";
+ usage(argv[0], flags);
+ return 0;
+ }
+
+ mesos::internal::DockerExecutor executor(
+ flags.docker.get(), flags.container.get());
+ mesos::MesosExecutorDriver driver(&executor);
+ return driver.run() == mesos::DRIVER_STOPPED ? 0 : 1;
+}
[09/20] mesos git commit: Add option to disable docker containerizer
killing orphans.
Posted by tn...@apache.org.
Add option to disable docker containerizer killing orphans.
Review: https://reviews.apache.org/r/29328
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8f31db87
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8f31db87
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8f31db87
Branch: refs/heads/master
Commit: 8f31db8794b7b2e20b9980b4e190ca62fc2e3481
Parents: 5acbfd2
Author: Timothy Chen <tn...@apache.org>
Authored: Mon Nov 24 17:17:07 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 6 +++++-
src/slave/flags.cpp | 5 +++++
src/slave/flags.hpp | 1 +
3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8f31db87/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index a37b45d..92f0e85 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -562,9 +562,13 @@ Future<Nothing> DockerContainerizerProcess::_recover(
pids.put(containerId, pid);
}
}
+
+ if (flags.docker_kill_orphans) {
+ return __recover(_containers);
+ }
}
- return __recover(_containers);
+ return Nothing();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/8f31db87/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index b5e2518..c1b22e8 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -292,6 +292,11 @@ mesos::internal::slave::Flags::Flags()
"(e.g., 3days, 2weeks, etc).\n",
DOCKER_REMOVE_DELAY);
+ add(&Flags::docker_kill_orphans,
+ "docker_kill_orphans",
+ "Enable docker containerizer to kill orphaned containers",
+ true);
+
add(&Flags::default_container_info,
"default_container_info",
"JSON formatted ContainerInfo that will be included into\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/8f31db87/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 5c57478..3302b71 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -84,6 +84,7 @@ public:
Duration docker_remove_delay;
Option<ContainerInfo> default_container_info;
Duration docker_stop_timeout;
+ bool docker_kill_orphans;
#ifdef WITH_NETWORK_ISOLATOR
uint16_t ephemeral_ports_per_container;
Option<std::string> eth0_name;
[04/20] mesos git commit: Re-enable docker recover test.
Posted by tn...@apache.org.
Re-enable docker recover test.
Review: https://reviews.apache.org/r/29331
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3790d74c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3790d74c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3790d74c
Branch: refs/heads/master
Commit: 3790d74c43f738f6095bd7f0eb51b5108cf571e7
Parents: c8da93a
Author: Timothy Chen <tn...@apache.org>
Authored: Wed Nov 26 17:34:45 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/tests/docker_containerizer_tests.cpp | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3790d74c/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 154bf98..36ccc6a 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -1182,14 +1182,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
#endif //__linux__
-// Disabling recover test as the docker rm in recover is async.
-// Even though we wait for the container to finish, when the wait
-// returns docker rm might still be in progress.
-// TODO(tnachen): Re-enable test when we wait for the async kill
-// to finish. One way to do this is to mock the Docker interface
-// and let the mocked docker collect all the remove futures and
-// at the end of the test wait for all of them before the test exits.
-TEST_F(DockerContainerizerTest, DISABLED_ROOT_DOCKER_Recover)
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
{
slave::Flags flags = CreateSlaveFlags();
@@ -1275,6 +1268,15 @@ TEST_F(DockerContainerizerTest, DISABLED_ROOT_DOCKER_Recover)
slaveState.frameworks.put(frameworkId, frameworkState);
+ // We need to capture and await on the stop future so that we can
+ // ensure there is no child process at the end of the test.
+ // The stop future is being awaited at teardown.
+ Future<Nothing> stop;
+ EXPECT_CALL(*mockDocker, stop(_, _, _))
+ .WillOnce(FutureResult(
+ &stop,
+ Invoke((MockDocker*) docker.get(), &MockDocker::_stop)));
+
Future<Nothing> recover = dockerContainerizer.recover(slaveState);
AWAIT_READY(recover);
@@ -1292,6 +1294,8 @@ TEST_F(DockerContainerizerTest, DISABLED_ROOT_DOCKER_Recover)
AWAIT_READY(reaped.get().status());
+ AWAIT_READY_FOR(stop, Seconds(30));
+
Shutdown();
}
[13/20] mesos git commit: Change docker executor launch command log
level.
Posted by tn...@apache.org.
Change docker executor launch command log level.
Review: https://reviews.apache.org/r/29340
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/499035aa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/499035aa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/499035aa
Branch: refs/heads/master
Commit: 499035aa4ea2a25e83c3cb86f6a1f8b417241ed4
Parents: 6b7f0da
Author: Timothy Chen <tn...@gmail.com>
Authored: Fri Dec 19 19:55:20 2014 +0000
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/499035aa/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 58505f9..66e3746 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -779,7 +779,7 @@ Future<pid_t> DockerContainerizerProcess::___launch(
command = path::join(flags.launcher_dir, command);
- VLOG(2) << "Launching docker executor with command: " << command;
+ VLOG(1) << "Launching docker executor with command: " << command;
// Construct the mesos-docker-executor using the "name" we gave the
// container (to distinguish it from Docker containers not created
@@ -884,7 +884,7 @@ Future<Nothing> DockerContainerizerProcess::___launchInContainer(
commandInfo.set_value(command);
commandInfo.set_shell(true);
- VLOG(2) << "Launching docker executor in container with command: " << command;
+ VLOG(1) << "Launching docker executor in container with command: " << command;
return docker->run(
containerInfo,
[06/20] mesos git commit: Add docker_socket to slave flags.
Posted by tn...@apache.org.
Add docker_socket to slave flags.
Review: https://reviews.apache.org/r/29333
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/168ba44a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/168ba44a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/168ba44a
Branch: refs/heads/master
Commit: 168ba44a41319e71498d47bf9409d0cd3b4eb365
Parents: 5c3654d
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Dec 2 18:21:41 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/flags.cpp | 7 +++++++
src/slave/flags.hpp | 1 +
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/168ba44a/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 4d20590..acf30d1 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -305,6 +305,13 @@ mesos::internal::slave::Flags::Flags()
"containerizer to launch executors in docker containers, so they keep\n"
"running when the slave container exits.");
+ add(&Flags::docker_socket,
+ "docker_socket",
+ "The docker UNIX socket path that the docker CLI uses to communicate\n"
+ "to the docker daemon. Mesos needs this to launch docker containers\n"
+ "that can run the docker CLI.\n",
+ "/var/run/docker.sock");
+
add(&Flags::default_container_info,
"default_container_info",
"JSON formatted ContainerInfo that will be included into\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/168ba44a/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 742ebf9..6ca59dc 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -86,6 +86,7 @@ public:
Option<ContainerInfo> default_container_info;
Duration docker_stop_timeout;
bool docker_kill_orphans;
+ std::string docker_socket;
#ifdef WITH_NETWORK_ISOLATOR
uint16_t ephemeral_ports_per_container;
Option<std::string> eth0_name;
[14/20] mesos git commit: Fix docker recovery when launched in
container and add more logging.
Posted by tn...@apache.org.
Fix docker recovery when launched in container and add more logging.
Review: https://reviews.apache.org/r/29341
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a76bed6e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a76bed6e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a76bed6e
Branch: refs/heads/master
Commit: a76bed6ecd6eb01755cc5ef5a5a693b61ca2ef86
Parents: 499035a
Author: Timothy Chen <tn...@apache.org>
Authored: Fri Apr 10 16:02:40 2015 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 56 ++++++++++++++++++++++++---------
src/slave/containerizer/docker.hpp | 2 +-
2 files changed, 43 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a76bed6e/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 66e3746..5bce2ce 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -109,15 +109,10 @@ Option<ContainerID> parse(const Docker::Container& container)
}
vector<string> parts = strings::split(name.get(), DOCKER_NAME_SEPERATOR);
- if (parts.size() == 2) {
+ if (parts.size() == 2 || parts.size() == 3) {
ContainerID id;
id.set_value(parts[1]);
return id;
- } else if (parts.size() == 3) {
- // We found a executor or log container.
- ContainerID id;
- id.set_value(parts[2]);
- return id;
}
}
@@ -131,6 +126,8 @@ Try<pid_t> launchWaitProcess(const string& docker, const string& name)
{
string command = "exit `" + docker + " wait " + name + "`";
+ VLOG(1) << "Launching wait process: " << command;
+
Try<Subprocess> wait = subprocess(
command,
Subprocess::PATH("/dev/null"),
@@ -466,6 +463,7 @@ Future<Nothing> DockerContainerizerProcess::recover(
const Option<SlaveState>& state)
{
LOG(INFO) << "Recovering Docker containers";
+
if (state.isSome()) {
// Get the list of all Docker containers (running and exited) in
// order to remove any orphans and reconcile checkpointed executors.
@@ -774,6 +772,11 @@ Future<pid_t> DockerContainerizerProcess::___launch(
environment[variable.name()] = variable.value();
}
+ // Pass GLOG flag to the executor.
+ if (os::hasenv("GLOG_v")) {
+ environment["GLOG_v"] = os::getenv("GLOG_v");
+ }
+
string command = "mesos-docker-executor --docker=" + flags.docker +
" --container=" + container->name();
@@ -854,6 +857,11 @@ Future<Nothing> DockerContainerizerProcess::___launchInContainer(
environment[variable.name()] = variable.value();
}
+ // Pass GLOG flag to the executor.
+ if (os::hasenv("GLOG_v")) {
+ environment["GLOG_v"] = os::getenv("GLOG_v");
+ }
+
// We are launching a mesos-docker-executor in a docker container so
// that the containerizer can recover the executor container, as we
// are assuming this instance is launched in a docker container and
@@ -863,10 +871,17 @@ Future<Nothing> DockerContainerizerProcess::___launchInContainer(
// Mounting in the docker socket so the executor can communicate to
// the host docker daemon. We are assuming the current instance is
// launching docker containers to the host daemon as well.
- Volume* volume = containerInfo.add_volumes();
- volume->set_host_path(flags.docker_socket);
- volume->set_container_path(flags.docker_socket);
- volume->set_mode(Volume::RO);
+ Volume* dockerSockVolume = containerInfo.add_volumes();
+ dockerSockVolume->set_host_path(flags.docker_socket);
+ dockerSockVolume->set_container_path(flags.docker_socket);
+ dockerSockVolume->set_mode(Volume::RO);
+
+ // Mounting in sandbox so the logs from the executor can be
+ // persisted over container failures.
+ Volume* sandboxVolume = containerInfo.add_volumes();
+ sandboxVolume->set_host_path(container->directory);
+ sandboxVolume->set_container_path(container->directory);
+ sandboxVolume->set_mode(Volume::RW);
ContainerInfo::DockerInfo dockerInfo;
@@ -1461,7 +1476,7 @@ void DockerContainerizerProcess::destroy(
// been configured to launch but we might have recovered containers
// on previous slave run that has configured to launch executor in
// docker.
- docker->stop(container->executorName(), Seconds(0), true);
+ docker->stop(container->executorName(), Seconds(0));
container->state = Container::DESTROYING;
@@ -1538,7 +1553,12 @@ void DockerContainerizerProcess::__destroy(
containers_.erase(containerId);
- delay(flags.docker_remove_delay, self(), &Self::remove, container->name());
+ delay(
+ flags.docker_remove_delay,
+ self(),
+ &Self::remove,
+ container->name(),
+ container->executorName());
delete container;
@@ -1576,7 +1596,12 @@ void DockerContainerizerProcess::___destroy(
containers_.erase(containerId);
- delay(flags.docker_remove_delay, self(), &Self::remove, container->name());
+ delay(
+ flags.docker_remove_delay,
+ self(),
+ &Self::remove,
+ container->name(),
+ container->executorName());
delete container;
}
@@ -1601,9 +1626,12 @@ void DockerContainerizerProcess::reaped(const ContainerID& containerId)
}
-void DockerContainerizerProcess::remove(const string& container)
+void DockerContainerizerProcess::remove(
+ const string& container,
+ const string& executor)
{
docker->rm(container, true);
+ docker->rm(executor, true);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/a76bed6e/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index e683ea4..0bfb0e9 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -271,7 +271,7 @@ private:
void reaped(const ContainerID& containerId);
// Removes the docker container.
- void remove(const std::string& container);
+ void remove(const std::string& container, const std::string& executor);
const Flags flags;
[03/20] mesos git commit: Add docker_mesos_image flag to slave flags.
Posted by tn...@apache.org.
Add docker_mesos_image flag to slave flags.
Review: https://reviews.apache.org/r/29332
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5c3654da
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5c3654da
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5c3654da
Branch: refs/heads/master
Commit: 5c3654daafa5e86fc98d37d79ce74a6f7c52423a
Parents: 3790d74
Author: Timothy Chen <tn...@apache.org>
Authored: Mon Dec 1 15:53:29 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/flags.cpp | 8 ++++++++
src/slave/flags.hpp | 1 +
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5c3654da/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index c1b22e8..4d20590 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -297,6 +297,14 @@ mesos::internal::slave::Flags::Flags()
"Enable docker containerizer to kill orphaned containers",
true);
+ add(&Flags::docker_mesos_image,
+ "docker_mesos_image",
+ "The docker image used to launch this mesos slave instance.\n"
+ "If an image is specified, the docker containerizer assumes the slave\n"
+ "is running in a docker container. This enables the docker\n"
+ "containerizer to launch executors in docker containers, so they keep\n"
+ "running when the slave container exits.");
+
add(&Flags::default_container_info,
"default_container_info",
"JSON formatted ContainerInfo that will be included into\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/5c3654da/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 3302b71..742ebf9 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -80,6 +80,7 @@ public:
std::string containerizers;
Option<std::string> default_container_image;
std::string docker;
+ Option<std::string> docker_mesos_image;
std::string docker_sandbox_directory;
Duration docker_remove_delay;
Option<ContainerInfo> default_container_info;
[05/20] mesos git commit: Add slave id to docker container name
prefix.
Posted by tn...@apache.org.
Add slave id to docker container name prefix.
Review: https://reviews.apache.org/r/29327
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5acbfd2e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5acbfd2e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5acbfd2e
Branch: refs/heads/master
Commit: 5acbfd2e7907029d2979561a3876208941044fde
Parents: 9c5c927
Author: Timothy Chen <tn...@gmail.com>
Authored: Mon Nov 17 17:19:49 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 29 +++++++++++++++++++++++------
src/slave/containerizer/docker.hpp | 2 +-
2 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5acbfd2e/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 408a443..a37b45d 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -72,8 +72,6 @@ using state::RunState;
// Declared in header, see explanation there.
-// TODO(benh): At some point to run multiple slaves we'll need to make
-// the Docker container name creation include the slave ID.
const string DOCKER_NAME_PREFIX = "mesos-";
// Declared in header, see explanation there.
@@ -94,9 +92,25 @@ Option<ContainerID> parse(const Docker::Container& container)
}
if (name.isSome()) {
- ContainerID id;
- id.set_value(name.get());
- return id;
+ // For Mesos version <= 0.21.0, the docker container name format
+ // was DOCKER_NAME_PREFIX + containerId, and starting with 0.22.0
+ // it is changed to DOCKER_NAME_PREFIX + slaveId + "/" +
+ // containerId.
+ // To still be backward compatible during upgrade, we still need
+ // to load the previous format.
+ // TODO(tnachen): Remove this check after deprecation cycle.
+ if (!strings::contains(name.get(), "/")) {
+ ContainerID id;
+ id.set_value(name.get());
+ return id;
+ }
+
+ vector<string> parts = strings::split(name.get(), "/");
+ if (parts.size() == 2) {
+ ContainerID id;
+ id.set_value(parts[1]);
+ return id;
+ }
}
return None();
@@ -286,7 +300,10 @@ Try<Nothing> DockerContainerizerProcess::checkpoint(
Future<Nothing> DockerContainerizer::recover(
const Option<SlaveState>& state)
{
- return dispatch(process.get(), &DockerContainerizerProcess::recover, state);
+ return dispatch(
+ process.get(),
+ &DockerContainerizerProcess::recover,
+ state);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/5acbfd2e/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index ed4ee19..f08520a 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -321,7 +321,7 @@ private:
std::string name()
{
- return DOCKER_NAME_PREFIX + stringify(id);
+ return DOCKER_NAME_PREFIX + slaveId.value() + "/" + stringify(id);
}
std::string image() const
[08/20] mesos git commit: Integrate docker executor into
containerizer.
Posted by tn...@apache.org.
Integrate docker executor into containerizer.
Review: https://reviews.apache.org/r/29330
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c8da93a1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c8da93a1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c8da93a1
Branch: refs/heads/master
Commit: c8da93a1956fbe32018f6d776cff620471b34bc7
Parents: 24415b7
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Nov 25 21:37:51 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c8da93a1/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 92f0e85..c4e636c 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -401,8 +401,8 @@ Future<hashset<ContainerID>> DockerContainerizer::containers()
// A Subprocess async-safe "setup" helper used by
-// DockerContainerizerProcess when launching the mesos-executor that
-// does a 'setsid' and then synchronizes with the parent.
+// DockerContainerizerProcess when launching the mesos-docker-executor
+// that does a 'setsid' and then synchronizes with the parent.
static int setup(const string& directory)
{
// Put child into its own process session to prevent slave suicide
@@ -725,16 +725,18 @@ Future<pid_t> DockerContainerizerProcess::___launch(
environment[variable.name()] = variable.value();
}
- // Construct the mesos-executor "override" to do a 'docker wait'
- // using the "name" we gave the container (to distinguish it from
- // Docker containers not created by Mesos). Note, however, that we
- // don't want the exit status from 'docker wait' but rather the exit
- // status from the container, hence the use of /bin/bash.
- string override =
- "/bin/sh -c 'exit `" + flags.docker + " wait " + container->name() + "`'";
+ string command = "mesos-docker-executor --docker=" + flags.docker +
+ " --container=" + container->name();
+ command = path::join(flags.launcher_dir, command);
+
+ VLOG(2) << "Launching docker executor with command: " << command;
+
+ // Construct the mesos-docker-executor using the "name" we gave the
+ // container (to distinguish it from Docker containers not created
+ // by Mesos).
Try<Subprocess> s = subprocess(
- container->executor.command().value() + " --override " + override,
+ command,
Subprocess::PIPE(),
Subprocess::PATH(path::join(container->directory, "stdout")),
Subprocess::PATH(path::join(container->directory, "stderr")),
@@ -1328,10 +1330,9 @@ void DockerContainerizerProcess::_destroy(
// Do a 'docker rm -f' which we'll then find out about in '_destroy'
// after we've reaped either the container's root process (in the
// event that we had just launched a container for an executor) or
- // the mesos-executor (in the case we launched a container for a
- // task). As a reminder, the mesos-executor exits because it's doing
- // a 'docker wait' on the container using the --override flag of
- // mesos-executor.
+ // the mesos-docker-executor (in the case we launched a container
+ // for a task). As a reminder, the mesos-docker-executor exits
+ // because it's doing a 'docker wait' on the container.
LOG(INFO) << "Running docker stop on container '" << containerId << "'";
[17/20] mesos git commit: Address comments for Docker containerizer
changes.
Posted by tn...@apache.org.
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index e5f24e6..7524803 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -81,17 +81,17 @@ public:
EXPECT_CALL(*this, pull(_, _, _))
.WillRepeatedly(Invoke(this, &MockDocker::_pull));
- EXPECT_CALL(*this, logs(_, _))
- .WillRepeatedly(Invoke(this, &MockDocker::_logs));
-
EXPECT_CALL(*this, stop(_, _, _))
.WillRepeatedly(Invoke(this, &MockDocker::_stop));
- EXPECT_CALL(*this, run(_, _, _, _, _, _, _))
+ EXPECT_CALL(*this, run(_, _, _, _, _, _, _, _, _))
.WillRepeatedly(Invoke(this, &MockDocker::_run));
+
+ EXPECT_CALL(*this, inspect(_, _))
+ .WillRepeatedly(Invoke(this, &MockDocker::_inspect));
}
- MOCK_CONST_METHOD7(
+ MOCK_CONST_METHOD9(
run,
process::Future<Nothing>(
const mesos::ContainerInfo&,
@@ -99,21 +99,17 @@ public:
const std::string&,
const std::string&,
const std::string&,
- const Option<mesos::Resources>& resources,
- const Option<std::map<std::string, std::string>>&));
-
- MOCK_CONST_METHOD2(
- logs,
- process::Future<Nothing>(
- const string&,
- const string&));
+ const Option<mesos::Resources>&,
+ const Option<std::map<std::string, std::string>>&,
+ const Option<std::string>&,
+ const Option<std::string>&));
MOCK_CONST_METHOD3(
pull,
process::Future<Docker::Image>(
- const string& directory,
- const string& image,
- bool force));
+ const string&,
+ const string&,
+ bool));
MOCK_CONST_METHOD3(
stop,
@@ -122,14 +118,22 @@ public:
const Duration&,
bool));
+ MOCK_CONST_METHOD2(
+ inspect,
+ process::Future<Docker::Container>(
+ const string&,
+ const Option<Duration>&));
+
process::Future<Nothing> _run(
const mesos::ContainerInfo& containerInfo,
const mesos::CommandInfo& commandInfo,
const std::string& name,
const std::string& sandboxDirectory,
const std::string& mappedDirectory,
- const Option<mesos::Resources>& resources = None(),
- const Option<std::map<std::string, std::string> >& env = None()) const
+ const Option<mesos::Resources>& resources,
+ const Option<std::map<std::string, std::string>>& env,
+ const Option<std::string>& stdoutPath,
+ const Option<std::string>& stderrPath) const
{
return Docker::run(
containerInfo,
@@ -138,7 +142,9 @@ public:
sandboxDirectory,
mappedDirectory,
resources,
- env);
+ env,
+ stdoutPath,
+ stderrPath);
}
process::Future<Docker::Image> _pull(
@@ -149,19 +155,19 @@ public:
return Docker::pull(directory, image, force);
}
- process::Future<Nothing> _logs(
- const string& container,
- const string& directory) const
- {
- return Docker::logs(container, directory);
- }
-
process::Future<Nothing> _stop(
- const string& container,
+ const string& containerName,
const Duration& timeout,
bool remove) const
{
- return Docker::stop(container, timeout, remove);
+ return Docker::stop(containerName, timeout, remove);
+ }
+
+ process::Future<Docker::Container> _inspect(
+ const string& containerName,
+ const Option<Duration>& retryInterval)
+ {
+ return Docker::inspect(containerName, retryInterval);
}
};
@@ -177,50 +183,37 @@ public:
slave::DOCKER_NAME_SEPERATOR + containerId.value();
}
- static bool exists(
- const process::Shared<Docker>& docker,
- const SlaveID& slaveId,
- const ContainerID& containerId)
- {
- Duration waited = Duration::zero();
- string expectedName = containerName(slaveId, containerId);
-
- do {
- Future<Docker::Container> container = docker->inspect(expectedName);
-
- if (!container.await(Seconds(3))) {
- return false;
- }
+ enum ContainerState {
+ EXISTS,
+ RUNNING
+ };
- if(!container.isFailed()) {
- return true;
- }
-
- os::sleep(Milliseconds(200));
- waited += Milliseconds(200);
- } while (waited < Seconds(5));
-
- return false;
- }
-
- static bool running(
+ static bool exists(
const process::Shared<Docker>& docker,
const SlaveID& slaveId,
- const ContainerID& containerId)
+ const ContainerID& containerId,
+ ContainerState state = ContainerState::EXISTS)
{
Duration waited = Duration::zero();
string expectedName = containerName(slaveId, containerId);
do {
- Future<Docker::Container> container = docker->inspect(expectedName);
+ Future<Docker::Container> inspect = docker->inspect(expectedName);
- if (!container.await(Seconds(3))) {
+ if (!inspect.await(Seconds(3))) {
return false;
}
- if (container.isReady()) {
- if (container.get().pid.isSome()) {
- return true;
+ if (inspect.isReady()) {
+ switch (state) {
+ case ContainerState::RUNNING:
+ if (inspect.get().pid.isSome()) {
+ return true;
+ }
+ // Retry looking for running pid until timeout.
+ break;
+ case ContainerState::EXISTS:
+ return true;
}
}
@@ -231,9 +224,11 @@ public:
return false;
}
- static bool containsLine(string text, string expectedLine)
+ static bool containsLine(
+ const vector<string>& lines,
+ const string& expectedLine)
{
- foreach (const string& line, strings::split(text, "\n")) {
+ foreach (const string& line, lines) {
if (line == expectedLine) {
return true;
}
@@ -253,7 +248,7 @@ public:
// Cleanup all mesos launched containers.
foreach (const Docker::Container& container, containers.get()) {
- ASSERT_TRUE(docker.get()->rm(container.id, true).await(Seconds(30)));
+ AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
}
delete docker.get();
@@ -387,7 +382,7 @@ public:
EXPECT_CALL(*this, fetch(_))
.WillRepeatedly(Invoke(this, &MockDockerContainerizerProcess::_fetch));
- EXPECT_CALL(*this, pull(_, _, _, _))
+ EXPECT_CALL(*this, pull(_))
.WillRepeatedly(Invoke(this, &MockDockerContainerizerProcess::_pull));
}
@@ -395,30 +390,18 @@ public:
fetch,
process::Future<Nothing>(const ContainerID& containerId));
- MOCK_METHOD4(
+ MOCK_METHOD1(
pull,
- process::Future<Nothing>(
- const ContainerID& containerId,
- const std::string& directory,
- const std::string& image,
- bool forcePullImage));
+ process::Future<Nothing>(const ContainerID& containerId));
process::Future<Nothing> _fetch(const ContainerID& containerId)
{
return DockerContainerizerProcess::fetch(containerId);
}
- process::Future<Nothing> _pull(
- const ContainerID& containerId,
- const std::string& directory,
- const std::string& image,
- bool forcePullImage)
+ process::Future<Nothing> _pull(const ContainerID& containerId)
{
- return DockerContainerizerProcess::pull(
- containerId,
- directory,
- image,
- forcePullImage);
+ return DockerContainerizerProcess::pull(containerId);
}
};
@@ -435,15 +418,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher;
@@ -495,6 +469,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("tnachen/test-executor");
@@ -536,10 +511,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
AWAIT_READY(termination);
- ASSERT_FALSE(running(docker, slaveId, containerId.get()));
-
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
+ ASSERT_FALSE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
Shutdown();
}
@@ -558,15 +531,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher;
@@ -618,6 +582,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("tnachen/test-executor");
dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
@@ -660,10 +625,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
AWAIT_READY(termination);
- ASSERT_FALSE(running(docker, slaveId, containerId.get()));
-
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
+ ASSERT_FALSE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
Shutdown();
}
@@ -723,6 +686,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -760,7 +724,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
AWAIT_READY(termination);
- ASSERT_FALSE(running(docker, slaveId, containerId.get()));
+ ASSERT_FALSE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
Shutdown();
}
@@ -819,6 +784,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -845,7 +811,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+ ASSERT_TRUE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
Future<TaskStatus> statusKilled;
EXPECT_CALL(sched, statusUpdate(&driver, _))
@@ -861,7 +828,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
AWAIT_READY(termination);
- ASSERT_FALSE(running(docker, slaveId, containerId.get()));
+ ASSERT_FALSE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
driver.stop();
driver.join();
@@ -924,6 +892,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -961,6 +930,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage)
do {
Future<ResourceStatistics> usage =
dockerContainerizer.usage(containerId.get());
+ // TODO(tnachen): Replace await with AWAIT_COMPLETED once
+ // implemented.
ASSERT_TRUE(usage.await(Seconds(3)));
if (usage.isReady()) {
@@ -1057,6 +1028,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1085,13 +1057,14 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+ ASSERT_TRUE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
string name = containerName(slaveId, containerId.get());
- Future<Docker::Container> container = docker->inspect(name);
+ Future<Docker::Container> inspect = docker->inspect(name);
- AWAIT_READY(container);
+ AWAIT_READY(inspect);
Try<Resources> newResources = Resources::parse("cpus:1;mem:128");
@@ -1108,7 +1081,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
ASSERT_SOME(cpuHierarchy);
ASSERT_SOME(memoryHierarchy);
- Option<pid_t> pid = container.get().pid;
+ Option<pid_t> pid = inspect.get().pid;
ASSERT_SOME(pid);
Result<string> cpuCgroup = cgroups::cpu::cgroup(pid.get());
@@ -1195,6 +1168,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1220,11 +1194,10 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
flags.docker_sandbox_directory,
resources);
- AWAIT_READY(d1);
- AWAIT_READY(d2);
-
- ASSERT_TRUE(running(docker, slaveId, containerId));
- ASSERT_TRUE(running(docker, slaveId, reapedContainerId));
+ ASSERT_TRUE(
+ exists(docker, slaveId, containerId, ContainerState::RUNNING));
+ ASSERT_TRUE(
+ exists(docker, slaveId, reapedContainerId, ContainerState::RUNNING));
Future<Docker::Container> inspect = docker->inspect(container2);
AWAIT_READY(inspect);
@@ -1384,6 +1357,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Logs)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1421,16 +1395,20 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Logs)
// Now check that the proper output is in stderr and stdout (which
// might also contain other things, hence the use of a UUID).
Try<string> read = os::read(path::join(directory.get(), "stderr"));
-
ASSERT_SOME(read);
- EXPECT_TRUE(containsLine(read.get(), "err" + uuid));
- EXPECT_FALSE(containsLine(read.get(), "out" + uuid));
- read = os::read(path::join(directory.get(), "stdout"));
+ vector<string> lines = strings::split(read.get(), "\n");
+ EXPECT_TRUE(containsLine(lines, "err" + uuid));
+ EXPECT_FALSE(containsLine(lines, "out" + uuid));
+
+ read = os::read(path::join(directory.get(), "stdout"));
ASSERT_SOME(read);
- EXPECT_TRUE(containsLine(read.get(), "out" + uuid));
- EXPECT_FALSE(containsLine(read.get(), "err" + uuid));
+
+ lines = strings::split(read.get(), "\n");
+
+ EXPECT_TRUE(containsLine(lines, "out" + uuid));
+ EXPECT_FALSE(containsLine(lines, "err" + uuid));
driver.stop();
driver.join();
@@ -1504,6 +1482,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("mesosphere/inky");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1539,18 +1518,21 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
Try<string> read = os::read(path::join(directory.get(), "stdout"));
-
ASSERT_SOME(read);
+ vector<string> lines = strings::split(read.get(), "\n");
+
// Since we're not passing any command value, we're expecting the
// default entry point to be run which is 'echo' with the default
// command from the image which is 'inky'.
- EXPECT_TRUE(containsLine(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(lines, "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(containsLine(read.get(), "inky"));
+ lines = strings::split(read.get(), "\n");
+
+ EXPECT_FALSE(containsLine(lines, "inky"));
driver.stop();
driver.join();
@@ -1624,6 +1606,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Override)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("mesosphere/inky");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1660,19 +1643,23 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Override)
// Now check that the proper output is in stderr and stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
-
ASSERT_SOME(read);
+ vector<string> lines = strings::split(read.get(), "\n");
+
// We expect the passed in command value to override the image's
// default command, thus we should see the value of 'uuid' in the
// output instead of the default command which is 'inky'.
- EXPECT_TRUE(containsLine(read.get(), uuid));
- EXPECT_FALSE(containsLine(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(lines, uuid));
+ EXPECT_FALSE(containsLine(lines, "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(containsLine(read.get(), "inky"));
- EXPECT_FALSE(containsLine(read.get(), uuid));
+
+ lines = strings::split(read.get(), "\n");
+
+ EXPECT_FALSE(containsLine(lines, "inky"));
+ EXPECT_FALSE(containsLine(lines, uuid));
driver.stop();
driver.join();
@@ -1747,6 +1734,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Args)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("mesosphere/inky");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1783,19 +1771,23 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Args)
// Now check that the proper output is in stderr and stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
-
ASSERT_SOME(read);
+ vector<string> lines = strings::split(read.get(), "\n");
+
// We expect the passed in command arguments to override the image's
// default command, thus we should see the value of 'uuid' in the
// output instead of the default command which is 'inky'.
- EXPECT_TRUE(containsLine(read.get(), uuid));
- EXPECT_FALSE(containsLine(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(lines, uuid));
+ EXPECT_FALSE(containsLine(lines, "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(containsLine(read.get(), "inky"));
- EXPECT_FALSE(containsLine(read.get(), uuid));
+
+ lines = strings::split(read.get(), "\n");
+
+ EXPECT_FALSE(containsLine(lines, "inky"));
+ EXPECT_FALSE(containsLine(lines, uuid));
driver.stop();
driver.join();
@@ -1868,6 +1860,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -1969,15 +1962,6 @@ TEST_F(DockerContainerizerTest,
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
Fetcher fetcher;
MockDockerContainerizer* dockerContainerizer1 =
@@ -2031,6 +2015,7 @@ TEST_F(DockerContainerizerTest,
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("mesosphere/test-executor");
@@ -2119,9 +2104,6 @@ TEST_F(DockerContainerizerTest,
driver.stop();
driver.join();
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
delete dockerContainerizer2;
}
@@ -2195,6 +2177,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
@@ -2234,7 +2217,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+ ASSERT_TRUE(
+ exists(docker, slaveId, containerId.get(), ContainerState::RUNNING));
string uuid = UUID::random().toString();
@@ -2250,16 +2234,22 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
// Now check that the proper output is in stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
-
ASSERT_SOME(read);
+ const vector<string> lines = strings::split(read.get(), "\n");
+
// We expect the uuid that is sent to host port to be written
// to stdout by the docker container running nc -l.
- EXPECT_TRUE(containsLine(read.get(), uuid));
+ EXPECT_TRUE(containsLine(lines, uuid));
+
+ Future<containerizer::Termination> termination =
+ dockerContainerizer.wait(containerId.get());
driver.stop();
driver.join();
+ AWAIT_READY(termination);
+
Shutdown();
}
@@ -2321,6 +2311,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -2427,6 +2418,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DestroyWhileFetching)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -2497,7 +2489,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DestroyWhilePulling)
Promise<Nothing> promise;
// We want to pause the fetch call to simulate a long fetch time.
- EXPECT_CALL(*process, pull(_, _, _, _))
+ EXPECT_CALL(*process, pull(_))
.WillOnce(Return(promise.future()));
Try<PID<Slave> > slave = StartSlave(&dockerContainerizer);
@@ -2537,6 +2529,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DestroyWhilePulling)
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image("busybox");
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
@@ -2785,9 +2778,9 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_FetchFailure)
}
-// When the docker run fails we should send the scheduler a status
+// When the docker pull fails we should send the scheduler a status
// update with message the shows the actual error.
-TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerRunFailure)
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerPullFailure)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -2861,8 +2854,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerRunFailure)
Invoke(&dockerContainerizer,
&MockDockerContainerizer::_launch)));
- EXPECT_CALL(*mockDocker, run(_, _, _, _, _, _, _))
- .WillOnce(Return(Failure("some error from docker run")));
+ EXPECT_CALL(*mockDocker, pull(_, _, _))
+ .WillOnce(Return(Failure("some error from docker pull")));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -2874,7 +2867,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerRunFailure)
AWAIT_READY(statusFailed);
EXPECT_EQ(TASK_FAILED, statusFailed.get().state());
- EXPECT_EQ("Failed to launch container: some error from docker run",
+ EXPECT_EQ("Failed to launch container: some error from docker pull",
statusFailed.get().message());
// TODO(jaybuff): When MESOS-2035 is addressed we should validate
@@ -2889,9 +2882,9 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerRunFailure)
}
-// When the docker pull fails we should send the scheduler a status
-// update with message the shows the actual error.
-TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerPullFailure)
+// When the docker executor container fails to launch, docker inspect
+// future that is in a retry loop should be discarded.
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerInspectDiscard)
{
Try<PID<Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -2901,6 +2894,15 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerPullFailure)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
+ Future<Docker::Container> inspect;
+ EXPECT_CALL(*mockDocker, inspect(_, _))
+ .WillOnce(FutureResult(&inspect,
+ Invoke((MockDocker*) docker.get(),
+ &MockDocker::_inspect)));
+
+ EXPECT_CALL(*mockDocker, run(_, _, _, _, _, _, _, _, _))
+ .WillOnce(Return(Failure("Run failed")));
+
Fetcher fetcher;
// The docker containerizer will free the process, so we must
@@ -2942,31 +2944,36 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerPullFailure)
task.mutable_slave_id()->CopyFrom(offer.slave_id());
task.mutable_resources()->CopyFrom(offer.resources());
+ ExecutorInfo executorInfo;
+ ExecutorID executorId;
+ executorId.set_value("e1");
+ executorInfo.mutable_executor_id()->CopyFrom(executorId);
+
CommandInfo command;
- command.set_value("ls");
+ command.set_value("test-executor");
+ executorInfo.mutable_command()->CopyFrom(command);
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
+ // TODO(tnachen): Use local image to test if possible.
ContainerInfo::DockerInfo dockerInfo;
- dockerInfo.set_image("busybox");
+ dockerInfo.set_image("tnachen/test-executor");
+
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+ executorInfo.mutable_container()->CopyFrom(containerInfo);
- task.mutable_command()->CopyFrom(command);
- task.mutable_container()->CopyFrom(containerInfo);
+ task.mutable_executor()->CopyFrom(executorInfo);
Future<TaskStatus> statusFailed;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusFailed));
Future<ContainerID> containerId;
- EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _, _))
+ EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _))
.WillOnce(DoAll(FutureArg<0>(&containerId),
Invoke(&dockerContainerizer,
- &MockDockerContainerizer::_launch)));
-
- EXPECT_CALL(*mockDocker, pull(_, _, _))
- .WillOnce(Return(Failure("some error from docker pull")));
+ &MockDockerContainerizer::_launchExecutor)));
vector<TaskInfo> tasks;
tasks.push_back(task);
@@ -2976,19 +2983,15 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_DockerPullFailure)
AWAIT_READY_FOR(containerId, Seconds(60));
AWAIT_READY(statusFailed);
-
EXPECT_EQ(TASK_FAILED, statusFailed.get().state());
- EXPECT_EQ("Failed to launch container: some error from docker pull",
- statusFailed.get().message());
- // TODO(jaybuff): When MESOS-2035 is addressed we should validate
- // that statusFailed.get().reason() is correctly set here.
+ AWAIT_DISCARDED(inspect);
driver.stop();
driver.join();
- // We expect the executor to have exited, and if not in Shutdown
- // the test will fail because of the executor process still running.
+ // We expect the inspect to have exited, and if not in Shutdown
+ // the test will fail because of the inspect process still running.
Shutdown();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/tests/docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_tests.cpp b/src/tests/docker_tests.cpp
index 5520c58..acf1e3c 100644
--- a/src/tests/docker_tests.cpp
+++ b/src/tests/docker_tests.cpp
@@ -85,7 +85,9 @@ TEST(DockerTest, ROOT_DOCKER_interface)
"/mnt/mesos/sandbox",
resources);
- AWAIT_READY(status);
+ Future<Docker::Container> inspect =
+ docker->inspect(containerName, Seconds(1));
+ AWAIT_READY(inspect);
// Should be able to see the container now.
containers = docker->ps();
@@ -99,13 +101,10 @@ TEST(DockerTest, ROOT_DOCKER_interface)
}
EXPECT_TRUE(found);
- Future<Docker::Container> container = docker->inspect(containerName);
- AWAIT_READY(container);
-
// Test some fields of the container.
- EXPECT_NE("", container.get().id);
- EXPECT_EQ("/" + containerName, container.get().name);
- EXPECT_SOME(container.get().pid);
+ EXPECT_NE("", inspect.get().id);
+ EXPECT_EQ("/" + containerName, inspect.get().name);
+ EXPECT_SOME(inspect.get().pid);
// Stop the container.
status = docker->stop(containerName);
@@ -133,20 +132,20 @@ TEST(DockerTest, ROOT_DOCKER_interface)
// Check the container's info, both id and name should remain the
// same since we haven't removed it, but the pid should be none
// since it's not running.
- container = docker->inspect(containerName);
- AWAIT_READY(container);
+ inspect = docker->inspect(containerName);
+ AWAIT_READY(inspect);
- EXPECT_NE("", container.get().id);
- EXPECT_EQ("/" + containerName, container.get().name);
- EXPECT_NONE(container.get().pid);
+ EXPECT_NE("", inspect.get().id);
+ EXPECT_EQ("/" + containerName, inspect.get().name);
+ EXPECT_NONE(inspect.get().pid);
// Remove the container.
status = docker->rm(containerName);
AWAIT_READY(status);
// Should not be able to inspect the container.
- container = docker->inspect(containerName);
- AWAIT_FAILED(container);
+ inspect = docker->inspect(containerName);
+ AWAIT_FAILED(inspect);
// Also, now we should not be able to see the container by invoking
// ps(true).
@@ -166,7 +165,8 @@ TEST(DockerTest, ROOT_DOCKER_interface)
"/mnt/mesos/sandbox",
resources);
- AWAIT_READY(status);
+ inspect = docker->inspect(containerName, Seconds(1));
+ AWAIT_READY(inspect);
// Verify that the container is there.
containers = docker->ps();
[19/20] mesos git commit: Add documentation for new docker flags.
Posted by tn...@apache.org.
Add documentation for new docker flags.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9f91e07e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9f91e07e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9f91e07e
Branch: refs/heads/master
Commit: 9f91e07e4d90ff0cab2f362d8425770d6a919c16
Parents: a76bed6
Author: Timothy Chen <tn...@gmail.com>
Authored: Fri Jan 16 23:07:19 2015 +0000
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
docs/configuration.md | 31 +++++++++++++++++++++++++++++++
docs/docker-containerizer.md | 19 ++++++++++++-------
2 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9f91e07e/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index 54c4e31..0a7629a 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -859,6 +859,37 @@ file:///path/to/file (where file contains one of the above)</code></pre>
</tr>
<tr>
<td>
+ --[no-]docker_kill_orphans
+ </td>
+ <td>
+ Enable docker containerizer to kill orphaned containers
+ (default: true)
+ </td>
+ </tr>
+ <tr>
+ <td>
+ --docker_sock=VALUE
+ </td>
+ <td>
+ The docker UNIX socket path that the CLI uses to communicate to the
+ daemon. We need this to launch docker containers that can run docker
+ CLI.
+ (default: /var/run/docker.sock)
+ </td>
+ </tr>
+ <tr>
+ <td>
+ --docker_mesos_image=VALUE
+ </td>
+ <td>
+ The docker image used to launch this mesos slave instance.
+ If a image is specified, the docker containerizer assumes the slave
+ is running in a docker container, and launches executors with
+ docker containers in order to recover them when the slave recovers.
+ </td>
+ </tr>
+ <tr>
+ <td>
--docker_sandbox_directory=VALUE
</td>
<td>
http://git-wip-us.apache.org/repos/asf/mesos/blob/9f91e07e/docs/docker-containerizer.md
----------------------------------------------------------------------
diff --git a/docs/docker-containerizer.md b/docs/docker-containerizer.md
index a5438b7..da9c533 100644
--- a/docs/docker-containerizer.md
+++ b/docs/docker-containerizer.md
@@ -39,18 +39,17 @@ Currently the Docker Containerizer when launching as task will do the following:
1, Fetch all the files specified in the CommandInfo into the sandbox.
2, Pull the docker image from the remote repository.
-3, Run the docker image with the configured DockerInfo options, and map the sandbox directory into the Docker container and set the directory mapping to the MESOS_SANDBOX environment variable.
-4. Stream the docker logs into the stdout/stderr files in the sandbox.
-5. Launch the Command Executor to perform a docker wait on the container.
-6. On container exit or containerizer destroy, stop and remove the docker container.
+3, Run the docker image with the Docker executor, and map the sandbox directory into the Docker container and set the directory mapping to the MESOS_SANDBOX environment variable.
+ The executor will also stream the container logs into stdout/stderr files in the sandbox.
+4. On container exit or containerizer destroy, stop and remove the docker container.
-The Docker Containerizer launches all containers with the "mesos-" prefix (ie: mesos-abcdefghji), and also assumes all containers with the "mesos-" prefix is managed by the slave and is free to stop or kill the containers.
+The Docker Containerizer launches all containers with the "mesos-" prefix plus the slave id (ie: mesos-slave1-abcdefghji), and also assumes all containers with the "mesos-" prefix is managed by the slave and is free to stop or kill the containers.
When launching the docker image as an Executor, the only difference is that it skips launching a command executor but just reaps on the docker container executor pid.
-Note that we currently default to host networking when running a docker image, to easier support running a docker image asn an Executor.
+Note that we currently default to host networking when running a docker image, to easier support running a docker image as an Executor.
-Also since we explicitly attempt to pull the image on launch, if the docker image is only installed locally but not avaialble on the remote repository the launch will fail as well.
+The containerizer also supports optional force pulling of the image, and if disabled the docker image will only be updated again if it's not available on the host.
## Private Docker repository
@@ -65,3 +64,9 @@ To run a docker image with the default command (ie: docker run image), the Comma
To run a docker image with an entrypoint defined, the CommandInfo's shell option must be set to false.
If shell option is set to true the Docker Containerizer will run the user's command wrapped with /bin/sh -c which will also become parameters to the image entrypoint.
+
+## Recover Docker containers on slave recovery
+
+The Docker containerizer supports recovering Docker containers when the slave restarts, which supports both when the slave is running in a Docker container or not.
+
+With the docker_mesos_image flag enabled, the Docker containerizer assumes the containerizer is running in a container itself and modifies the mechanism it recovers and launches docker containers accordingly.
[02/20] mesos git commit: Add option to launch docker containers with
helper containers.
Posted by tn...@apache.org.
Add option to launch docker containers with helper containers.
Review: https://reviews.apache.org/r/29334
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/280e4659
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/280e4659
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/280e4659
Branch: refs/heads/master
Commit: 280e4659b922d84c5c405a176b8d5db2f9203928
Parents: 168ba44
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Dec 2 18:23:17 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:50 2015 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 212 +++++++++++++++++++++++++++++---
src/slave/containerizer/docker.hpp | 32 ++++-
2 files changed, 226 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/280e4659/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index c4e636c..c996afe 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -649,11 +649,29 @@ Future<bool> DockerContainerizerProcess::launch(
<< "' (and executor '" << executorInfo.executor_id()
<< "') of framework '" << executorInfo.framework_id() << "'";
- return container.get()->launch = fetch(containerId)
+ Future<Nothing> future = fetch(containerId)
.then(defer(self(), &Self::_launch, containerId))
- .then(defer(self(), &Self::__launch, containerId))
- .then(defer(self(), &Self::___launch, containerId))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1));
+ .then(defer(self(), &Self::__launch, containerId));
+
+ if (flags.docker_mesos_image.isNone()) {
+ // Launch executor and logs with subprocess.
+ return container.get()->launch = future
+ .then(defer(self(), &Self::___launch, containerId))
+ .then(defer(self(), &Self::______launch, containerId, lambda::_1))
+ .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
+ .onFailed(defer(self(), &Self::destroy, containerId, true));
+ }
+
+ // Launch executor and logs with docker containers.
+ return container.get()->launch = future
+ .then(defer(self(), &Self::___launchInContainer, containerId))
+ .then(defer(
+ self(),
+ &Self::____launchInContainer,
+ containerId))
+ .then(defer(self(), &Self::______launch, containerId, lambda::_1))
+ .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
+ .onFailed(defer(self(), &Self::destroy, containerId, true));
}
@@ -777,6 +795,79 @@ Future<pid_t> DockerContainerizerProcess::___launch(
}
+Future<Nothing> DockerContainerizerProcess::___launchInContainer(
+ const ContainerID& containerId)
+{
+ // After we do Docker::run we shouldn't remove a container until
+ // after we set Container::status.
+ CHECK(containers_.contains(containerId));
+ CHECK(flags.docker_mesos_image.isSome());
+
+ Container* container = containers_[containerId];
+
+ // Prepare environment variables for the executor.
+ map<string, string> environment = executorEnvironment(
+ container->executor,
+ container->directory,
+ container->slaveId,
+ container->slavePid,
+ container->checkpoint,
+ flags.recovery_timeout);
+
+ // Include any enviroment variables from ExecutorInfo.
+ foreach (const Environment::Variable& variable,
+ container->executor.command().environment().variables()) {
+ environment[variable.name()] = variable.value();
+ }
+
+ ContainerInfo containerInfo;
+ ContainerInfo::DockerInfo dockerInfo;
+
+ dockerInfo.set_image(flags.docker_mesos_image.get());
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ string command = "mesos-docker-executor --docker=" + flags.docker +
+ " --container=" + container->name();
+
+ command = path::join(flags.launcher_dir, command);
+
+ CommandInfo commandInfo;
+ commandInfo.set_value(command);
+ commandInfo.set_shell(true);
+
+ VLOG(2) << "Launching docker executor in container with command: " << command;
+
+ return docker->run(
+ containerInfo,
+ commandInfo,
+ container->executorName(),
+ container->directory,
+ flags.docker_sandbox_directory,
+ None(),
+ environment);
+}
+
+
+// Launches a docker wait process on given container name.
+// Returns the wait process pid.
+Try<pid_t> launchWaitProcess(const string& docker, const string& name)
+{
+ string command = "exit `" + docker + " wait " + name + "`";
+
+ Try<Subprocess> wait = subprocess(
+ command,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"));
+
+ if (wait.isError()) {
+ return Error("Unable to launch docker wait on executor: " + wait.error());
+ }
+
+ return wait.get().pid();
+}
+
+
Future<bool> DockerContainerizerProcess::launch(
const ContainerID& containerId,
const ExecutorInfo& executorInfo,
@@ -828,20 +919,17 @@ Future<bool> DockerContainerizerProcess::launch(
.then(defer(self(), &Self::__launch, containerId))
.then(defer(self(), &Self::____launch, containerId))
.then(defer(self(), &Self::_____launch, containerId, lambda::_1))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1));
+ .then(defer(self(), &Self::______launch, containerId, lambda::_1))
+ .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
}
Future<Docker::Container> DockerContainerizerProcess::____launch(
const ContainerID& containerId)
{
- // After we do Docker::run we shouldn't remove a container until
- // after we set Container::status.
CHECK(containers_.contains(containerId));
- Container* container = containers_[containerId];
-
- return docker->inspect(container->name());
+ return docker->inspect(containers_[containerId]->name());
}
@@ -859,12 +947,6 @@ Future<pid_t> DockerContainerizerProcess::_____launch(
return Failure("Unable to get executor pid after launch");
}
- // TODO(tnachen): We might not be able to checkpoint if the slave
- // dies before it can checkpoint while the executor is still
- // running. Optinally we can consider recording the slave id and
- // executor id as part of the docker container name so we can
- // recover from this.
-
Try<Nothing> checkpointed = checkpoint(containerId, pid.get());
if (checkpointed.isError()) {
@@ -876,7 +958,96 @@ Future<pid_t> DockerContainerizerProcess::_____launch(
}
-Future<bool> DockerContainerizerProcess::______launch(
+Future<pid_t> DockerContainerizerProcess::______launch(
+ const ContainerID& containerId,
+ pid_t pid)
+{
+ CHECK(containers_.contains(containerId));
+
+ Container* container = containers_[containerId];
+
+ docker->logs(container->name(), container->directory);
+
+ return pid;
+}
+
+
+Future<pid_t> DockerContainerizerProcess::______launchInContainer(
+ const ContainerID& containerId,
+ pid_t pid)
+{
+ CHECK(containers_.contains(containerId));
+ CHECK(flags.docker_mesos_image.isSome());
+
+ Container* container = containers_[containerId];
+
+ // We are launching a docker container to read the logs from
+ // a given launched container into the sandbox stdout and stderr
+ // files. This requires the docker container to run docker logs,
+ // which requires us mounting in the docker socket and docker
+ // CLI binary.
+ ContainerInfo containerInfo;
+ Volume* volume = containerInfo.add_volumes();
+ volume->set_host_path(flags.docker_socket);
+ volume->set_container_path(flags.docker_socket);
+ volume->set_mode(Volume::RO);
+
+ volume = containerInfo.add_volumes();
+ volume->set_host_path(flags.docker);
+ volume->set_container_path(flags.docker);
+ volume->set_mode(Volume::RO);
+
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image(flags.docker_mesos_image.get());
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ string command = flags.docker + " logs --follow " + container->name() +
+ " 2>> " +
+ path::join(flags.docker_sandbox_directory, "stderr") +
+ " 1>> " +
+ path::join(flags.docker_sandbox_directory, "stdout");
+
+ VLOG(1) << "Running docker logs in container with command: " << command;
+
+ CommandInfo commandInfo;
+ commandInfo.set_value(command);
+ commandInfo.set_shell(true);
+
+ docker->run(
+ containerInfo,
+ commandInfo,
+ container->logName(),
+ container->directory,
+ flags.docker_sandbox_directory);
+
+ return pid;
+}
+
+
+Future<pid_t> DockerContainerizerProcess::____launchInContainer(
+ const ContainerID& containerId)
+{
+ CHECK(containers_.contains(containerId));
+
+ Try<pid_t> waitPid =
+ launchWaitProcess(flags.docker, containers_[containerId]->executorName());
+
+ if (waitPid.isError()) {
+ return Failure(waitPid.error());
+ }
+
+ Try<Nothing> checkpointed = checkpoint(containerId, waitPid.get());
+
+ if (checkpointed.isError()) {
+ return Failure(
+ "Failed to checkpoint executor's pid: " + checkpointed.error());
+ }
+
+ return waitPid.get();
+}
+
+
+Future<bool> DockerContainerizerProcess::_______launch(
const ContainerID& containerId,
pid_t pid)
{
@@ -1287,6 +1458,13 @@ void DockerContainerizerProcess::destroy(
CHECK(container->state == Container::RUNNING);
+ // Remove the executor and log docker containers. They might not
+ // been configured to launch but we might have recovered containers
+ // on previous slave run that has configured to launch executor in
+ // docker.
+ docker->rm(container->logName(), true);
+ docker->rm(container->executorName(), true);
+
container->state = Container::DESTROYING;
if (container->executorPid.isSome()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/280e4659/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index f08520a..a54e0d4 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -196,6 +196,16 @@ private:
const ContainerID& containerId);
// NOTE: This continuation is only applicable when launching a
+ // container for a task.
+ process::Future<Nothing> ___launchInContainer(
+ const ContainerID& containerId);
+
+ // NOTE: This continuation is only applicable when launching a
+ // container for a task.
+ process::Future<pid_t> ____launchInContainer(
+ const ContainerID& containerId);
+
+ // NOTE: This continuation is only applicable when launching a
// container for an executor.
process::Future<Docker::Container> ____launch(
const ContainerID& containerId);
@@ -206,7 +216,17 @@ private:
const ContainerID& containerId,
const Docker::Container& container);
- process::Future<bool> ______launch(
+ process::Future<pid_t> ______launch(
+ const ContainerID& containerId,
+ pid_t pid);
+
+ // NOTE: This continuation is only applicable when launching a
+ // container for a task.
+ process::Future<pid_t> ______launchInContainer(
+ const ContainerID& containerId,
+ pid_t pid);
+
+ process::Future<bool> _______launch(
const ContainerID& containerId,
pid_t pid);
@@ -324,6 +344,16 @@ private:
return DOCKER_NAME_PREFIX + slaveId.value() + "/" + stringify(id);
}
+ std::string logName()
+ {
+ return name() + "/log";
+ }
+
+ std::string executorName()
+ {
+ return name() + "/executor";
+ }
+
std::string image() const
{
if (task.isSome()) {
[11/20] mesos git commit: Launch docker container and log through
docker executor.
Posted by tn...@apache.org.
Launch docker container and log through docker executor.
Review: https://reviews.apache.org/r/29337
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/88da1f6e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/88da1f6e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/88da1f6e
Branch: refs/heads/master
Commit: 88da1f6e05addadb9a9f559c58888cfefb58f851
Parents: 7fee619
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Dec 16 13:16:42 2014 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri May 22 23:13:51 2015 -0700
----------------------------------------------------------------------
src/docker/docker.cpp | 55 ++++--
src/docker/docker.hpp | 6 +-
src/docker/executor.cpp | 156 +++++++++------
src/slave/containerizer/docker.cpp | 87 +++------
src/slave/containerizer/docker.hpp | 5 -
src/tests/docker_containerizer_tests.cpp | 266 ++++++--------------------
6 files changed, 232 insertions(+), 343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 3a485a2..d7a33dd 100644
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -175,6 +175,13 @@ Try<Docker*> Docker::create(const string& path, bool validate)
}
+void commandDiscarded(const Subprocess& s, const string& cmd)
+{
+ VLOG(1) << "'" << cmd << "' is being discarded";
+ os::killtree(s.pid(), SIGKILL);
+}
+
+
Try<Docker::Container> Docker::Container::create(const JSON::Object& json)
{
map<string, JSON::Value>::const_iterator entry =
@@ -280,7 +287,8 @@ Future<Nothing> Docker::run(
const string& sandboxDirectory,
const string& mappedDirectory,
const Option<Resources>& resources,
- const Option<map<string, string> >& env) const
+ const Option<map<string, string> >& env,
+ bool detached) const
{
if (!containerInfo.has_docker()) {
return Failure("No docker info found in container info");
@@ -291,7 +299,10 @@ Future<Nothing> Docker::run(
vector<string> argv;
argv.push_back(path);
argv.push_back("run");
- argv.push_back("-d");
+
+ if (detached) {
+ argv.push_back("-d");
+ }
if (dockerInfo.privileged()) {
argv.push_back("--privileged");
@@ -474,8 +485,12 @@ Future<Nothing> Docker::run(
path,
argv,
Subprocess::PATH("/dev/null"),
- Subprocess::PIPE(),
- Subprocess::PIPE(),
+ (detached
+ ? Subprocess::PIPE()
+ : Subprocess::PATH(path::join(sandboxDirectory, "stdout"))),
+ (detached
+ ? Subprocess::PIPE()
+ : Subprocess::PATH(path::join(sandboxDirectory, "stderr"))),
None(),
environment);
@@ -483,9 +498,30 @@ Future<Nothing> Docker::run(
return Failure(s.error());
}
- return checkError(cmd, s.get());
+ if (detached) {
+ return checkError(cmd, s.get());
+ }
+
+ return s.get().status()
+ .then(lambda::bind(
+ &Docker::_run,
+ lambda::_1))
+ .onDiscard(lambda::bind(&commandDiscarded, s.get(), cmd));
}
+
+Future<Nothing> Docker::_run(const Option<int>& status)
+{
+ if (status.isNone()) {
+ return Failure("Failed to get exit status.");
+ } else if (status.get() != 0) {
+ return Failure("Container exited on error: " + WSTRINGIFY(status.get()));
+ }
+
+ return Nothing();
+}
+
+
Future<Nothing> Docker::stop(
const string& container,
const Duration& timeout,
@@ -893,14 +929,7 @@ Future<Docker::Image> Docker::__pull(
cmd,
directory,
image))
- .onDiscard(lambda::bind(&Docker::pullDiscarded, s_.get(), cmd));
-}
-
-
-void Docker::pullDiscarded(const Subprocess& s, const string& cmd)
-{
- VLOG(1) << "'" << cmd << "' is being discarded";
- os::killtree(s.pid(), SIGKILL);
+ .onDiscard(lambda::bind(&commandDiscarded, s_.get(), cmd));
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 3ebbc1f..0ba7e0e 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -89,7 +89,8 @@ public:
const std::string& sandboxDirectory,
const std::string& mappedDirectory,
const Option<mesos::Resources>& resources = None(),
- const Option<std::map<std::string, std::string> >& env = None()) const;
+ const Option<std::map<std::string, std::string> >& env = None(),
+ bool detached = true) const;
// Performs 'docker stop -t TIMEOUT CONTAINER'. If remove is true then a rm -f
// will be called when stop failed, otherwise a failure is returned. The
@@ -149,6 +150,9 @@ private:
static process::Future<Container> __inspect(
const std::string& output);
+ static process::Future<Nothing> _run(
+ const Option<int>& status);
+
static process::Future<std::list<Container> > _ps(
const Docker& docker,
const std::string& cmd,
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 1a5ab86..61aeae9 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -27,12 +27,15 @@
#include <process/protobuf.hpp>
#include <process/subprocess.hpp>
#include <process/reap.hpp>
+#include <process/owned.hpp>
#include <stout/flags.hpp>
#include <stout/os.hpp>
#include "common/status_utils.hpp"
+#include "docker/docker.hpp"
+
#include "logging/logging.hpp"
using std::cerr;
@@ -57,11 +60,17 @@ using namespace process;
class DockerExecutorProcess : public ProtobufProcess<DockerExecutorProcess>
{
public:
- DockerExecutorProcess(const string& docker, const string& container)
+ DockerExecutorProcess(
+ const Owned<Docker>& docker,
+ const string& container,
+ const string& sandboxDirectory,
+ const string& mappedDirectory)
: launched(false),
+ killed(false),
docker(docker),
container(container),
- pid(-1) {}
+ sandboxDirectory(sandboxDirectory),
+ mappedDirectory(mappedDirectory) {}
virtual ~DockerExecutorProcess() {}
@@ -99,26 +108,35 @@ public:
cout << "Starting task " << task.task_id().value() << endl;
- Try<Subprocess> subprocess = process::subprocess(
- "exit `" + docker + " wait + " + container + "`",
- Subprocess::PATH("/dev/null"),
- Subprocess::FD(STDOUT_FILENO),
- Subprocess::FD(STDERR_FILENO));
+ // We assume the Docker executor is launched from the
+ // DockerContainerizer, which already calls setsid before
+ // launching the executor.
- if (subprocess.isError()) {
- cerr << "Couldn't launch docker wait process: " << subprocess.error();
- abort();
- }
+ CHECK(task.has_container());
+ CHECK(task.has_command());
+
+ ContainerInfo containerInfo = task.container();
- pid = subprocess.get().pid();
+ CHECK(containerInfo.type() == ContainerInfo::DOCKER);
- process::reap(pid)
- .onAny(defer(self(),
- &Self::reaped,
- driver,
- task.task_id(),
- pid,
- lambda::_1));
+ Future<Nothing> run = docker->run(
+ containerInfo,
+ task.command(),
+ container,
+ sandboxDirectory,
+ mappedDirectory,
+ task.resources(),
+ None(),
+ false);
+
+ run.onAny(defer(
+ self(),
+ &Self::reaped,
+ driver,
+ task.task_id(),
+ lambda::_1));
+
+ dockerRun = run;
TaskStatus status;
status.mutable_task_id()->MergeFrom(task.task_id());
@@ -139,8 +157,9 @@ public:
{
cout << "Shutting down" << endl;
- if (pid > 0 && !killed) {
- ::kill(pid, SIGKILL);
+ if (dockerRun.isSome() && !killed) {
+ Future<Nothing> dockerRun_ = dockerRun.get();
+ dockerRun_.discard();
killed = true;
}
}
@@ -151,41 +170,20 @@ private:
void reaped(
ExecutorDriver* driver,
const TaskID& taskId,
- pid_t pid,
- const Future<Option<int>>& status)
+ const Future<Nothing>& run)
{
TaskState state;
string message;
- if (!status.isReady()) {
- state = TASK_FAILED;
- message =
- "Failed to get exit status for Docker executor: " +
- (status.isFailed() ? status.failure() : "future discarded");
- } else if (status.get().isNone()) {
+ if (killed) {
+ state = TASK_KILLED;
+ } else if (!run.isReady()) {
state = TASK_FAILED;
- message = "Failed to get exit status for Docker executor";
+ message = "Docker container run error: " +
+ (run.isFailed() ? run.failure() : "future discarded");
} else {
- int s = status.get().get();
-
- // Subprocess status is gathered from waitpid, therefore we can
- // get the exit status from WIFEXITED.
- CHECK(WIFEXITED(s) || WIFSIGNALED(s)) << "status code: " << s;
-
- if (WIFEXITED(s) && WEXITSTATUS(s) == 0) {
- state = TASK_FINISHED;
- } else if (killed) {
- // Send TASK_KILLED if the task was killed as a result of
- // killTask() or shutdown().
- state = TASK_KILLED;
- } else {
- state = TASK_FAILED;
- }
-
- message = "Docker " + WSTRINGIFY(s);
+ state = TASK_FINISHED;
}
- cout << message << " (pid: " << pid << ")" << endl;
-
TaskStatus taskStatus;
taskStatus.mutable_task_id()->MergeFrom(taskId);
taskStatus.set_state(state);
@@ -201,10 +199,12 @@ private:
bool launched;
- string docker;
- string container;
- pid_t pid;
bool killed;
+ Owned<Docker> docker;
+ string container;
+ string sandboxDirectory;
+ string mappedDirectory;
+ Option<Future<Nothing>> dockerRun;
Option<ExecutorDriver*> driver;
};
@@ -212,9 +212,18 @@ private:
class DockerExecutor : public Executor
{
public:
- DockerExecutor(const string& docker, const string& container)
+ DockerExecutor(
+ const Owned<Docker>& docker,
+ const string& container,
+ const string& sandboxDirectory,
+ const string& mappedDirectory)
{
- process = new DockerExecutorProcess(docker, container);
+ process = new DockerExecutorProcess(
+ docker,
+ container,
+ sandboxDirectory,
+ mappedDirectory);
+
spawn(process);
}
@@ -303,15 +312,26 @@ public:
{
add(&Flags::container,
"container",
- "The name of the docker container to wait on.");
+ "The name of the docker container to run.\n");
add(&Flags::docker,
"docker",
- "The path to the docker cli executable.");
+ "The path to the docker cli executable.\n");
+
+ add(&Flags::sandbox_directory,
+ "sandbox_directory",
+ "The path to the container sandbox that stores stdout and stderr\n"
+ "files that is being redirected with docker container logs.\n");
+
+ add(&Flags::mapped_directory,
+ "mapped_directory",
+ "The sandbox directory path that is mapped in the docker container.\n");
}
Option<string> container;
Option<string> docker;
+ Option<string> sandbox_directory;
+ Option<string> mapped_directory;
};
@@ -351,8 +371,30 @@ int main(int argc, char** argv)
return 0;
}
+ if (flags.sandbox_directory.isNone()) {
+ LOG(WARNING) << "Expected sandbox directory path";
+ usage(argv[0], flags);
+ return 0;
+ }
+
+ if (flags.mapped_directory.isNone()) {
+ LOG(WARNING) << "Expected mapped sandbox directory path";
+ usage(argv[0], flags);
+ return 0;
+ }
+
+ Try<Docker*> docker = Docker::create(flags.docker.get());
+ if (docker.isError()) {
+ LOG(WARNING) << "Unable to create docker abstraction: " << docker.error();
+ return -1;
+ }
+
mesos::internal::DockerExecutor executor(
- flags.docker.get(), flags.container.get());
+ process::Owned<Docker>(docker.get()),
+ flags.container.get(),
+ flags.sandbox_directory.get(),
+ flags.mapped_directory.get());
+
mesos::MesosExecutorDriver driver(&executor);
return driver.run() == mesos::DRIVER_STOPPED ? 0 : 1;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 50a5b72..58505f9 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -819,7 +819,10 @@ Future<pid_t> DockerContainerizerProcess::___launch(
if (length != sizeof(c)) {
string error = string(strerror(errno));
os::close(s.get().in().get());
- return Failure("Failed to synchronize with child process: " + error);
+ Failure failure("Failed to synchronize with child process: " + error);
+
+ container->run = failure;
+ return failure;
}
return s.get().pid();
@@ -851,14 +854,29 @@ Future<Nothing> DockerContainerizerProcess::___launchInContainer(
environment[variable.name()] = variable.value();
}
+ // We are launching a mesos-docker-executor in a docker container so
+ // that the containerizer can recover the executor container, as we
+ // are assuming this instance is launched in a docker container and
+ // forked processes are killed on exit.
ContainerInfo containerInfo;
+
+ // Mounting in the docker socket so the executor can communicate to
+ // the host docker daemon. We are assuming the current instance is
+ // launching docker containers to the host daemon as well.
+ Volume* volume = containerInfo.add_volumes();
+ volume->set_host_path(flags.docker_socket);
+ volume->set_container_path(flags.docker_socket);
+ volume->set_mode(Volume::RO);
+
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image(flags.docker_mesos_image.get());
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
string command = "mesos-docker-executor --docker=" + flags.docker +
- " --container=" + container->name();
+ " --container=" + container->name() +
+ " --sandbox_directory=" + container->directory +
+ " --mapped_directory=" + flags.docker_sandbox_directory;
command = path::join(flags.launcher_dir, command);
@@ -1003,58 +1021,6 @@ Future<pid_t> DockerContainerizerProcess::______launch(
}
-Future<pid_t> DockerContainerizerProcess::______launchInContainer(
- const ContainerID& containerId,
- pid_t pid)
-{
- CHECK(containers_.contains(containerId));
- CHECK(flags.docker_mesos_image.isSome());
-
- Container* container = containers_[containerId];
-
- // We are launching a docker container to read the logs from
- // a given launched container into the sandbox stdout and stderr
- // files. This requires the docker container to run docker logs,
- // which requires us mounting in the docker socket and docker
- // CLI binary.
- ContainerInfo containerInfo;
- Volume* volume = containerInfo.add_volumes();
- volume->set_host_path(flags.docker_socket);
- volume->set_container_path(flags.docker_socket);
- volume->set_mode(Volume::RO);
-
- volume = containerInfo.add_volumes();
- volume->set_host_path(flags.docker);
- volume->set_container_path(flags.docker);
- volume->set_mode(Volume::RO);
-
- ContainerInfo::DockerInfo dockerInfo;
- dockerInfo.set_image(flags.docker_mesos_image.get());
- containerInfo.mutable_docker()->CopyFrom(dockerInfo);
-
- string command = flags.docker + " logs --follow " + container->name() +
- " 2>> " +
- path::join(flags.docker_sandbox_directory, "stderr") +
- " 1>> " +
- path::join(flags.docker_sandbox_directory, "stdout");
-
- VLOG(1) << "Running docker logs in container with command: " << command;
-
- CommandInfo commandInfo;
- commandInfo.set_value(command);
- commandInfo.set_shell(true);
-
- docker->run(
- containerInfo,
- commandInfo,
- container->logName(),
- container->directory,
- flags.docker_sandbox_directory);
-
- return pid;
-}
-
-
Future<pid_t> DockerContainerizerProcess::____launchInContainer(
const ContainerID& containerId)
{
@@ -1127,6 +1093,11 @@ Future<Nothing> DockerContainerizerProcess::update(
// Store the resources for usage().
container->resources = _resources;
+ if (flags.docker_mesos_image.isSome()) {
+ LOG(INFO) << "Ignoring update as slave is running under container.";
+ return Nothing();
+ }
+
#ifdef __linux__
if (!_resources.cpus().isSome() && !_resources.mem().isSome()) {
LOG(WARNING) << "Ignoring update as no supported resources are present";
@@ -1486,12 +1457,11 @@ void DockerContainerizerProcess::destroy(
CHECK(container->state == Container::RUNNING);
- // Remove the executor and log docker containers. They might not
+ // Remove the executor docker containers. They might not
// been configured to launch but we might have recovered containers
// on previous slave run that has configured to launch executor in
// docker.
- docker->rm(container->logName(), true);
- docker->rm(container->executorName(), true);
+ docker->stop(container->executorName(), Seconds(0), true);
container->state = Container::DESTROYING;
@@ -1537,8 +1507,7 @@ void DockerContainerizerProcess::_destroy(
// after we've reaped either the container's root process (in the
// event that we had just launched a container for an executor) or
// the mesos-docker-executor (in the case we launched a container
- // for a task). As a reminder, the mesos-docker-executor exits
- // because it's doing a 'docker wait' on the container.
+ // for a task).
LOG(INFO) << "Running docker stop on container '" << containerId << "'";
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index 0eda1c0..e683ea4 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -349,11 +349,6 @@ private:
stringify(id);
}
- std::string logName()
- {
- return name() + DOCKER_NAME_SEPERATOR + "log";
- }
-
std::string executorName()
{
return name() + DOCKER_NAME_SEPERATOR + "executor";
http://git-wip-us.apache.org/repos/asf/mesos/blob/88da1f6e/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 3fa663e..378a526 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -178,35 +178,56 @@ public:
}
static bool exists(
- const list<Docker::Container>& containers,
+ const process::Shared<Docker>& docker,
const SlaveID& slaveId,
const ContainerID& containerId)
{
+ Duration waited = Duration::zero();
string expectedName = containerName(slaveId, containerId);
- foreach (const Docker::Container& container, containers) {
- // Docker inspect name contains an extra slash in the beginning.
- if (strings::contains(container.name, expectedName)) {
+ do {
+ Future<Docker::Container> container = docker->inspect(expectedName);
+
+ if (!container.await(Seconds(3))) {
+ return false;
+ }
+
+ if(!container.isFailed()) {
return true;
}
- }
+
+ os::sleep(Milliseconds(200));
+ waited += Milliseconds(200);
+ } while (waited < Seconds(5));
return false;
}
static bool running(
- const list<Docker::Container>& containers,
+ const process::Shared<Docker>& docker,
+ const SlaveID& slaveId,
const ContainerID& containerId)
{
- string expectedName = slave::DOCKER_NAME_PREFIX + stringify(containerId);
+ Duration waited = Duration::zero();
+ string expectedName = containerName(slaveId, containerId);
+
+ do {
+ Future<Docker::Container> container = docker->inspect(expectedName);
- foreach (const Docker::Container& container, containers) {
- // Docker inspect name contains an extra slash in the beginning.
- if (strings::contains(container.name, expectedName)) {
- return container.pid.isSome();
+ if (!container.await(Seconds(3))) {
+ return false;
}
- }
+
+ if (container.isReady()) {
+ if (container.get().pid.isSome()) {
+ return true;
+ }
+ }
+
+ os::sleep(Milliseconds(200));
+ waited += Milliseconds(200);
+ } while (waited < Seconds(5));
return false;
}
@@ -496,12 +517,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -511,11 +527,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
AWAIT_READY(termination);
- containers = docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_FALSE(running(containers.get(), containerId.get()));
+ ASSERT_FALSE(running(docker, slaveId, containerId.get()));
// See above where we assign logs future for more comments.
AWAIT_READY_FOR(logs, Seconds(30));
@@ -629,12 +641,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -644,10 +651,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
AWAIT_READY(termination);
- containers = docker->ps(true, slave::DOCKER_NAME_PREFIX);
- AWAIT_READY(containers);
-
- ASSERT_FALSE(running(containers.get(), containerId.get()));
+ ASSERT_FALSE(running(docker, slaveId, containerId.get()));
// See above where we assign logs future for more comments.
AWAIT_READY_FOR(logs, Seconds(30));
@@ -665,15 +669,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher;
@@ -746,14 +741,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(containers.get().size() > 0);
-
- ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -763,12 +751,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch)
AWAIT_READY(termination);
- containers = docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- ASSERT_FALSE(running(containers.get(), containerId.get()));
-
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
+ ASSERT_FALSE(running(docker, slaveId, containerId.get()));
Shutdown();
}
@@ -782,15 +765,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
slave::Flags flags = CreateSlaveFlags();
Fetcher fetcher;
@@ -822,6 +796,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -874,19 +850,11 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
AWAIT_READY(termination);
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_FALSE(running(containers.get(), containerId.get()));
+ ASSERT_FALSE(running(docker, slaveId, containerId.get()));
driver.stop();
driver.join();
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Shutdown();
}
@@ -903,15 +871,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
Fetcher fetcher;
MockDockerContainerizer dockerContainerizer(flags, &fetcher, docker);
@@ -991,13 +950,15 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage)
do {
Future<ResourceStatistics> usage =
dockerContainerizer.usage(containerId.get());
- AWAIT_READY(usage);
+ ASSERT_TRUE(usage.await(Seconds(3)));
- statistics = usage.get();
+ if (usage.isReady()) {
+ statistics = usage.get();
- if (statistics.cpus_user_time_secs() > 0 &&
- statistics.cpus_system_time_secs() > 0) {
- break;
+ if (statistics.cpus_user_time_secs() > 0 &&
+ statistics.cpus_system_time_secs() > 0) {
+ break;
+ }
}
os::sleep(Milliseconds(200));
@@ -1027,9 +988,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Usage)
driver.stop();
driver.join();
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Shutdown();
}
@@ -1045,15 +1003,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
Fetcher fetcher;
MockDockerContainerizer dockerContainerizer(flags, &fetcher, docker);
@@ -1125,6 +1074,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+ ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+
string name = containerName(slaveId, containerId.get());
Future<Docker::Container> container = docker->inspect(name);
@@ -1193,9 +1144,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Update)
driver.stop();
driver.join();
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Shutdown();
}
#endif //__linux__
@@ -1370,15 +1318,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Logs)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
// We skip stopping the docker container because stopping a container
// even when it terminated might not flush the logs and we end up
// not getting stdout/stderr in our tests.
@@ -1462,9 +1401,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Logs)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
// Now check that the proper output is in stderr and stdout (which
// might also contain other things, hence the use of a UUID).
Try<string> read = os::read(path::join(directory.get(), "stderr"));
@@ -1498,15 +1434,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
// We skip stopping the docker container because stopping a container
// even when it terminated might not flush the logs and we end up
// not getting stdout/stderr in our tests.
@@ -1592,9 +1519,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Try<string> read = os::read(path::join(directory.get(), "stdout"));
ASSERT_SOME(read);
@@ -1627,15 +1551,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Override)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
// We skip stopping the docker container because stopping a container
// even when it terminated might not flush the logs and we end up
// not getting stdout/stderr in our tests.
@@ -1723,9 +1638,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Override)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
// Now check that the proper output is in stderr and stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
@@ -1761,15 +1673,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Args)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
// We skip stopping the docker container because stopping a container
// even when it terminated might not flush the logs and we end up
// not getting stdout/stderr in our tests.
@@ -1858,9 +1761,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Args)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
// Now check that the proper output is in stderr and stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
@@ -1897,15 +1797,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
Fetcher fetcher;
// We put the containerizer on the heap so we can more easily
@@ -2019,13 +1910,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
AWAIT_READY(status);
ASSERT_EQ(TASK_RUNNING, status.get().state());
- // Make sure the container is still running.
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer2->wait(containerId.get());
@@ -2035,9 +1920,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_SlaveRecoveryTaskContainer)
AWAIT_READY(termination);
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Shutdown();
delete dockerContainerizer2;
@@ -2212,13 +2094,7 @@ TEST_F(DockerContainerizerTest,
AWAIT_READY(status);
ASSERT_EQ(TASK_RUNNING, status.get().state());
- // Make sure the container is still running.
- Future<list<Docker::Container>> containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(exists(containers.get(), slaveId.get(), containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId.get(), containerId.get()));
driver.stop();
driver.join();
@@ -2246,15 +2122,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(FutureResult(&logs,
- Invoke((MockDocker*) docker.get(),
- &MockDocker::_logs)));
-
// We skip stopping the docker container because stopping a container
// even when it terminated might not flush the logs and we end up
// not getting stdout/stderr in our tests.
@@ -2290,6 +2157,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -2345,6 +2214,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+ ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+
string uuid = UUID::random().toString();
// Write uuid to docker mapped host port.
@@ -2357,9 +2228,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
AWAIT_READY_FOR(statusFinished, Seconds(60));
EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
// Now check that the proper output is in stdout.
Try<string> read = os::read(path::join(directory.get(), "stdout"));
@@ -2390,14 +2258,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // We need to capture and await on the logs process's future so that
- // we can ensure there is no child process at the end of the test.
- // The logs future is being awaited at teardown.
- Future<Nothing> logs;
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillRepeatedly(FutureResult(
- &logs, Invoke((MockDocker*)docker.get(), &MockDocker::_logs)));
-
Fetcher fetcher;
MockDockerContainerizer dockerContainerizer(flags, &fetcher, docker);
@@ -2468,14 +2328,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
- Future<list<Docker::Container> > containers =
- docker->ps(true, slave::DOCKER_NAME_PREFIX);
-
- AWAIT_READY(containers);
-
- ASSERT_TRUE(containers.get().size() > 0);
-
- ASSERT_TRUE(exists(containers.get(), slaveId, containerId.get()));
+ ASSERT_TRUE(exists(docker, slaveId, containerId.get()));
Future<containerizer::Termination> termination =
dockerContainerizer.wait(containerId.get());
@@ -2485,9 +2338,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_LaunchSandboxWithColon)
AWAIT_READY(termination);
- // See above where we assign logs future for more comments.
- AWAIT_READY_FOR(logs, Seconds(30));
-
Shutdown();
}
[15/20] mesos git commit: Add version() to docker abstraction.
Posted by tn...@apache.org.
Add version() to docker abstraction.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/58963abf
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/58963abf
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/58963abf
Branch: refs/heads/master
Commit: 58963abf01cc33f41164a861d2add6a8b846d971
Parents: 72f62aa
Author: Timothy Chen <tn...@apache.org>
Authored: Thu Jan 22 16:57:38 2015 -0800
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/docker/docker.cpp | 96 ++++++++++++++++++++++++++++------------------
src/docker/docker.hpp | 10 +++++
2 files changed, 68 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/58963abf/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index d7a33dd..08f2023 100644
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -96,8 +96,9 @@ static Future<Nothing> checkError(const string& cmd, const Subprocess& s)
Try<Docker*> Docker::create(const string& path, bool validate)
{
+ Docker* docker = new Docker(path);
if (!validate) {
- return new Docker(path);
+ return docker;
}
#ifdef __linux__
@@ -106,6 +107,7 @@ Try<Docker*> Docker::create(const string& path, bool validate)
Result<string> hierarchy = cgroups::hierarchy("cpu");
if (hierarchy.isNone()) {
+ delete docker;
return Error("Failed to find a mounted cgroups hierarchy "
"for the 'cpu' subsystem; you probably need "
"to mount cgroups manually!");
@@ -113,6 +115,36 @@ Try<Docker*> Docker::create(const string& path, bool validate)
#endif // __linux__
// Validate the version (and that we can use Docker at all).
+ Future<Version> version = docker->version();
+
+ if(!version.await(Seconds(5))) {
+ delete docker;
+ return Error("Timed out getting docker version.");
+ }
+
+ if (version.isFailed()) {
+ delete docker;
+ return Error("Failed to get docker version: " + version.failure());
+ }
+
+ if (version.get() < Version(1, 0, 0)) {
+ delete docker;
+ return Error("Insufficient version of Docker! Please upgrade to >= 1.0.0");
+ }
+
+ return docker;
+}
+
+
+void commandDiscarded(const Subprocess& s, const string& cmd)
+{
+ VLOG(1) << "'" << cmd << "' is being discarded";
+ os::killtree(s.pid(), SIGKILL);
+}
+
+
+Future<Version> Docker::version() const
+{
string cmd = path + " version";
Try<Subprocess> s = subprocess(
@@ -122,63 +154,51 @@ Try<Docker*> Docker::create(const string& path, bool validate)
Subprocess::PIPE());
if (s.isError()) {
- return Error(s.error());
+ return Failure(s.error());
}
- Future<Option<int> > status = s.get().status();
+ return s.get().status()
+ .then(lambda::bind(&Docker::_version, cmd, s.get()));
+}
+
- if (!status.await(Seconds(5))) {
- return Error("Timed out waiting for '" + cmd + "'");
- } else if (status.isFailed()) {
- return Error("Failed to execute '" + cmd + "': " + status.failure());
- } else if (!status.get().isSome() || status.get().get() != 0) {
+Future<Version> Docker::_version(const string& cmd, const Subprocess& s)
+{
+ const Option<int>& status = s.status().get();
+ if (!status.isSome() || status.get() != 0) {
string msg = "Failed to execute '" + cmd + "': ";
- if (status.get().isSome()) {
- msg += WSTRINGIFY(status.get().get());
+ if (status.isSome()) {
+ msg += WSTRINGIFY(status.get());
} else {
msg += "unknown exit status";
}
- return Error(msg);
+ return Failure(msg);
}
- CHECK_SOME(s.get().out());
+ CHECK_SOME(s.out());
- Future<string> output = io::read(s.get().out().get());
+ return io::read(s.out().get())
+ .then(lambda::bind(&Docker::__version, lambda::_1));
+}
- if (!output.await(Seconds(5))) {
- return Error("Timed out reading output from '" + cmd + "'");
- } else if (output.isFailed()) {
- return Error("Failed to read output from '" + cmd + "': " +
- output.failure());
- }
+Future<Version> Docker::__version(const Future<string>& output)
+{
foreach (string line, strings::split(output.get(), "\n")) {
line = strings::trim(line);
if (strings::startsWith(line, "Client version: ")) {
line = line.substr(strlen("Client version: "));
- vector<string> version = strings::split(line, ".");
- if (version.size() < 1) {
- return Error("Failed to parse Docker version '" + line + "'");
- }
- Try<int> major = numify<int>(version[0]);
- if (major.isError()) {
- return Error("Failed to parse Docker major version '" +
- version[0] + "'");
- } else if (major.get() < 1) {
- break;
+ Try<Version> version = Version::parse(line);
+
+ if (version.isError()) {
+ return Failure("Failed to parse docker version: " + version.error());
}
- return new Docker(path);
+
+ return version;
}
}
- return Error("Insufficient version of Docker! Please upgrade to >= 1.0.0");
-}
-
-
-void commandDiscarded(const Subprocess& s, const string& cmd)
-{
- VLOG(1) << "'" << cmd << "' is being discarded";
- os::killtree(s.pid(), SIGKILL);
+ return Failure("Unable to find docker version in output.");
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/58963abf/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 2e2053e..3b6ea64 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -30,6 +30,7 @@
#include <stout/none.hpp>
#include <stout/nothing.hpp>
#include <stout/option.hpp>
+#include <stout/version.hpp>
#include "mesos/resources.hpp"
@@ -80,6 +81,8 @@ public:
: entrypoint(_entrypoint) {}
};
+ // Returns the current docker version.
+ virtual process::Future<Version> version() const;
// Performs 'docker run IMAGE'.
virtual process::Future<Nothing> run(
@@ -136,6 +139,13 @@ protected:
Docker(const std::string& _path) : path(_path) {};
private:
+ static process::Future<Version> _version(
+ const std::string& cmd,
+ const process::Subprocess& s);
+
+ static process::Future<Version> __version(
+ const process::Future<std::string>& output);
+
static process::Future<Nothing> _stop(
const Docker& docker,
const std::string& container,
[16/20] mesos git commit: Skip docker validation in docker executor.
Posted by tn...@apache.org.
Skip docker validation in docker executor.
Review: https://reviews.apache.org/r/29339
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6b7f0da9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6b7f0da9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6b7f0da9
Branch: refs/heads/master
Commit: 6b7f0da932e00662672570ee4cc1fce915f532b4
Parents: 88da1f6
Author: Timothy Chen <tn...@gmail.com>
Authored: Fri Dec 19 19:54:50 2014 +0000
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/docker/executor.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6b7f0da9/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 61aeae9..227b3d1 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -383,7 +383,9 @@ int main(int argc, char** argv)
return 0;
}
- Try<Docker*> docker = Docker::create(flags.docker.get());
+ // We skip validation when creating docker abstraction as we
+ // don't want to be checking for cgroups in the executor.
+ Try<Docker*> docker = Docker::create(flags.docker.get(), false);
if (docker.isError()) {
LOG(WARNING) << "Unable to create docker abstraction: " << docker.error();
return -1;
[20/20] mesos git commit: Simplify docker container launch using
docker 1.5+.
Posted by tn...@apache.org.
Simplify docker container launch using docker 1.5+.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0b5f67ee
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0b5f67ee
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0b5f67ee
Branch: refs/heads/master
Commit: 0b5f67eee081df3fe86e1cda140c436752944e64
Parents: 58963ab
Author: Timothy Chen <tn...@apache.org>
Authored: Fri Apr 10 16:03:30 2015 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/docker/executor.cpp | 2 +-
src/slave/containerizer/docker.cpp | 450 +++++++++-----------------
src/slave/containerizer/docker.hpp | 70 ++--
src/tests/docker_containerizer_tests.cpp | 4 -
4 files changed, 183 insertions(+), 343 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0b5f67ee/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index fda7b2a..847997b 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -126,7 +126,7 @@ public:
container,
sandboxDirectory,
mappedDirectory,
- task.resources(),
+ task.resources() + task.executor().resources(),
None(),
false);
http://git-wip-us.apache.org/repos/asf/mesos/blob/0b5f67ee/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 5bce2ce..952d3eb 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -120,28 +120,6 @@ Option<ContainerID> parse(const Docker::Container& container)
}
-// Launches a docker wait process on given container name.
-// Returns the wait process pid.
-Try<pid_t> launchWaitProcess(const string& docker, const string& name)
-{
- string command = "exit `" + docker + " wait " + name + "`";
-
- VLOG(1) << "Launching wait process: " << command;
-
- Try<Subprocess> wait = subprocess(
- command,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
-
- if (wait.isError()) {
- return Error("Unable to launch docker wait on executor: " + wait.error());
- }
-
- return wait.get().pid();
-}
-
-
Try<DockerContainerizer*> DockerContainerizer::create(
const Flags& flags,
Fetcher* fetcher)
@@ -151,10 +129,26 @@ Try<DockerContainerizer*> DockerContainerizer::create(
return Error(docker.error());
}
- return new DockerContainerizer(
- flags,
- fetcher,
- Shared<Docker>(docker.get()));
+ Shared<Docker> docker_(docker.get());
+
+ if (flags.docker_mesos_image.isSome()) {
+ Future<Version> version = docker_->version();
+ if (!version.await(Seconds(5))) {
+ return Error("Timed out waiting for docker version");
+ }
+
+ if (version.isFailed()) {
+ return Error(version.failure());
+ }
+
+ if (version.get() < Version(1, 5, 0)) {
+ string message = "Docker with mesos images requires docker 1.5+, found ";
+ message += stringify(version.get());
+ return Error(message);
+ }
+ }
+
+ return new DockerContainerizer(flags, fetcher, docker_);
}
@@ -183,6 +177,20 @@ DockerContainerizer::~DockerContainerizer()
}
+string dockerExecutorCommand(
+ const Flags& flags,
+ const string directory,
+ const string container)
+{
+ string command = "mesos-docker-executor --docker=" + flags.docker +
+ " --container=" + container +
+ " --sandbox_directory=" + directory +
+ " --mapped_directory=" + flags.docker_sandbox_directory;
+
+ return path::join(flags.launcher_dir, command);
+}
+
+
Try<DockerContainerizerProcess::Container*>
DockerContainerizerProcess::Container::create(
const ContainerID& id,
@@ -246,7 +254,7 @@ DockerContainerizerProcess::Container::create(
symlinked = true;
}
- return new Container(
+ Container* container = new Container(
id,
taskInfo,
executorInfo,
@@ -257,6 +265,51 @@ DockerContainerizerProcess::Container::create(
checkpoint,
symlinked,
flags);
+
+ if (taskInfo.isSome() && flags.docker_mesos_image.isSome()) {
+ // Override the container and command to launch an executor
+ // in a docker container.
+ ContainerInfo containerInfo;
+
+ // Mounting in the docker socket so the executor can communicate to
+ // the host docker daemon. We are assuming the current instance is
+ // launching docker containers to the host daemon as well.
+ Volume* dockerSockVolume = containerInfo.add_volumes();
+ dockerSockVolume->set_host_path(flags.docker_socket);
+ dockerSockVolume->set_container_path(flags.docker_socket);
+ dockerSockVolume->set_mode(Volume::RO);
+
+ // Mounting in sandbox so the logs from the executor can be
+ // persisted over container failures.
+ Volume* sandboxVolume = containerInfo.add_volumes();
+ sandboxVolume->set_host_path(containerWorkdir);
+ sandboxVolume->set_container_path(containerWorkdir);
+ sandboxVolume->set_mode(Volume::RW);
+
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image(flags.docker_mesos_image.get());
+
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ const string& command = dockerExecutorCommand(
+ flags, containerWorkdir, container->name());
+
+ CommandInfo commandInfo;
+ commandInfo.set_value(command);
+ commandInfo.set_shell(true);
+
+ container->overrideContainer = containerInfo;
+ container->overrideCommand = commandInfo;
+ container->overrideEnvironment = executorEnvironment(
+ executorInfo,
+ containerWorkdir,
+ slaveId,
+ slavePid,
+ checkpoint,
+ flags.recovery_timeout);
+ }
+
+ return container;
}
@@ -345,6 +398,7 @@ Future<bool> DockerContainerizer::launch(
process.get(),
&DockerContainerizerProcess::launch,
containerId,
+ None(),
executorInfo,
directory,
user,
@@ -473,7 +527,7 @@ Future<Nothing> DockerContainerizerProcess::recover(
.then(defer(self(), &Self::_recover, state.get(), lambda::_1));
}
-return Nothing();
+ return Nothing();
}
@@ -502,7 +556,7 @@ Future<Nothing> DockerContainerizerProcess::_recover(
// detect very unlikely duplicate scenario (see below).
hashmap<ContainerID, pid_t> pids;
- foreachvalue (const FrameworkState& framework, state.get().frameworks) {
+ foreachvalue (const FrameworkState& framework, state.frameworks) {
foreachvalue (const ExecutorState& executor, framework.executors) {
if (executor.info.isNone()) {
LOG(WARNING) << "Skipping recovery of executor '" << executor.id
@@ -567,7 +621,7 @@ Future<Nothing> DockerContainerizerProcess::_recover(
// Create and store a container.
Container* container = new Container(containerId);
containers_[containerId] = container;
- container->slaveId = state.get().id;
+ container->slaveId = state.id;
container->state = Container::RUNNING;
pid_t pid = run.get().forkedPid.get();
@@ -632,7 +686,7 @@ Future<Nothing> DockerContainerizerProcess::__recover(
Future<bool> DockerContainerizerProcess::launch(
const ContainerID& containerId,
- const TaskInfo& taskInfo,
+ const Option<TaskInfo>& taskInfo,
const ExecutorInfo& executorInfo,
const string& directory,
const Option<string>& user,
@@ -644,14 +698,20 @@ Future<bool> DockerContainerizerProcess::launch(
return Failure("Container already started");
}
- if (!taskInfo.has_container()) {
+ Option<ContainerInfo> containerInfo;
+
+ if (taskInfo.isSome() && taskInfo.get().has_container()) {
+ containerInfo = taskInfo.get().container();
+ } else if (executorInfo.has_container()) {
+ containerInfo = executorInfo.container();
+ }
+
+ if (containerInfo.isNone()) {
LOG(INFO) << "No container info found, skipping launch";
return false;
}
- ContainerInfo containerInfo = taskInfo.container();
-
- if (containerInfo.type() != ContainerInfo::DOCKER) {
+ if (containerInfo.get().type() != ContainerInfo::DOCKER) {
LOG(INFO) << "Skipping non-docker container";
return false;
}
@@ -673,34 +733,46 @@ Future<bool> DockerContainerizerProcess::launch(
containers_[containerId] = container.get();
- LOG(INFO) << "Starting container '" << containerId
- << "' for task '" << taskInfo.task_id()
- << "' (and executor '" << executorInfo.executor_id()
- << "') of framework '" << executorInfo.framework_id() << "'";
-
- Future<Nothing> future = fetch(containerId)
- .then(defer(self(), &Self::_launch, containerId))
- .then(defer(self(), &Self::__launch, containerId));
-
- if (flags.docker_mesos_image.isNone()) {
- // Launch executor and logs with subprocess.
- return container.get()->launch = future
+ if (taskInfo.isSome()) {
+ LOG(INFO) << "Starting container '" << containerId
+ << "' for task '" << taskInfo.get().task_id()
+ << "' (and executor '" << executorInfo.executor_id()
+ << "') of framework '" << executorInfo.framework_id() << "'";
+ } else {
+ LOG(INFO) << "Starting container '" << containerId
+ << "' for executor '" << executorInfo.executor_id()
+ << "' and framework '" << executorInfo.framework_id() << "'";
+ }
+
+ if (taskInfo.isSome() && flags.docker_mesos_image.isNone()) {
+ // Launching task by forking a subprocess to run docker executor.
+ return container.get()->launch = fetch(containerId)
+ .then(defer(self(), &Self::_launch, containerId))
.then(defer(self(), &Self::___launch, containerId))
.then(defer(self(), &Self::______launch, containerId, lambda::_1))
- .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
.onFailed(defer(self(), &Self::destroy, containerId, true));
}
- // Launch executor and logs with docker containers.
- return container.get()->launch = future
- .then(defer(self(), &Self::___launchInContainer, containerId))
- .then(defer(
- self(),
- &Self::____launchInContainer,
- containerId))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1))
- .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
- .onFailed(defer(self(), &Self::destroy, containerId, true));
+ string containerName = container.get()->name();
+
+ if (flags.docker_mesos_image.isSome()) {
+ // Launch the container with the executor name as we expect the
+ // executor will launch the docker container.
+ containerName = container.get()->executorName();
+ }
+
+ // Launching task or executor by launching a seperate docker
+ // container to run the executor.
+ // We need to do so for launching a task because as the slave
+ // is running in a container (via docker_mesos_image flag)
+ // we want the executor to keep running when the slave container
+ // dies.
+ return container.get()->launch = fetch(containerId)
+ .then(defer(self(), &Self::_launch, containerId))
+ .then(defer(self(), &Self::__launch, containerId, containerName))
+ .then(defer(self(), &Self::____launch, containerId, containerName))
+ .then(defer(self(), &Self::_____launch, containerId, lambda::_1))
+ .then(defer(self(), &Self::______launch, containerId, lambda::_1));
}
@@ -726,25 +798,26 @@ Future<Nothing> DockerContainerizerProcess::_launch(
Future<Nothing> DockerContainerizerProcess::__launch(
- const ContainerID& containerId)
+ const ContainerID& containerId,
+ const string& container)
{
if (!containers_.contains(containerId)) {
return Failure("Container was destroyed while pulling image");
}
- Container* container = containers_[containerId];
+ Container* container_ = containers_[containerId];
- container->state = Container::RUNNING;
+ container_->state = Container::RUNNING;
// Try and start the Docker container.
return docker->run(
- container->container(),
- container->command(),
- container->name(),
- container->directory,
+ container_->container(),
+ container_->command(),
+ container,
+ container_->directory,
flags.docker_sandbox_directory,
- container->resources,
- container->environment());
+ container_->resources,
+ container_->environment());
}
@@ -777,10 +850,8 @@ Future<pid_t> DockerContainerizerProcess::___launch(
environment["GLOG_v"] = os::getenv("GLOG_v");
}
- string command = "mesos-docker-executor --docker=" + flags.docker +
- " --container=" + container->name();
-
- command = path::join(flags.launcher_dir, command);
+ const string& command = dockerExecutorCommand(
+ flags, container->directory, container->name());
VLOG(1) << "Launching docker executor with command: " << command;
@@ -832,168 +903,13 @@ Future<pid_t> DockerContainerizerProcess::___launch(
}
-Future<Nothing> DockerContainerizerProcess::___launchInContainer(
- const ContainerID& containerId)
-{
- // After we do Docker::run we shouldn't remove a container until
- // after we set Container::status.
- CHECK(containers_.contains(containerId));
- CHECK(flags.docker_mesos_image.isSome());
-
- Container* container = containers_[containerId];
-
- // Prepare environment variables for the executor.
- map<string, string> environment = executorEnvironment(
- container->executor,
- container->directory,
- container->slaveId,
- container->slavePid,
- container->checkpoint,
- flags.recovery_timeout);
-
- // Include any enviroment variables from ExecutorInfo.
- foreach (const Environment::Variable& variable,
- container->executor.command().environment().variables()) {
- environment[variable.name()] = variable.value();
- }
-
- // Pass GLOG flag to the executor.
- if (os::hasenv("GLOG_v")) {
- environment["GLOG_v"] = os::getenv("GLOG_v");
- }
-
- // We are launching a mesos-docker-executor in a docker container so
- // that the containerizer can recover the executor container, as we
- // are assuming this instance is launched in a docker container and
- // forked processes are killed on exit.
- ContainerInfo containerInfo;
-
- // Mounting in the docker socket so the executor can communicate to
- // the host docker daemon. We are assuming the current instance is
- // launching docker containers to the host daemon as well.
- Volume* dockerSockVolume = containerInfo.add_volumes();
- dockerSockVolume->set_host_path(flags.docker_socket);
- dockerSockVolume->set_container_path(flags.docker_socket);
- dockerSockVolume->set_mode(Volume::RO);
-
- // Mounting in sandbox so the logs from the executor can be
- // persisted over container failures.
- Volume* sandboxVolume = containerInfo.add_volumes();
- sandboxVolume->set_host_path(container->directory);
- sandboxVolume->set_container_path(container->directory);
- sandboxVolume->set_mode(Volume::RW);
-
- ContainerInfo::DockerInfo dockerInfo;
-
- dockerInfo.set_image(flags.docker_mesos_image.get());
- containerInfo.mutable_docker()->CopyFrom(dockerInfo);
-
- string command = "mesos-docker-executor --docker=" + flags.docker +
- " --container=" + container->name() +
- " --sandbox_directory=" + container->directory +
- " --mapped_directory=" + flags.docker_sandbox_directory;
-
- command = path::join(flags.launcher_dir, command);
-
- CommandInfo commandInfo;
- commandInfo.set_value(command);
- commandInfo.set_shell(true);
-
- VLOG(1) << "Launching docker executor in container with command: " << command;
-
- return docker->run(
- containerInfo,
- commandInfo,
- container->executorName(),
- container->directory,
- flags.docker_sandbox_directory,
- None(),
- environment);
-}
-
-
-// Launches a docker wait process on given container name.
-// Returns the wait process pid.
-Try<pid_t> launchWaitProcess(const string& docker, const string& name)
-{
- string command = "exit `" + docker + " wait " + name + "`";
-
- Try<Subprocess> wait = subprocess(
- command,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
-
- if (wait.isError()) {
- return Error("Unable to launch docker wait on executor: " + wait.error());
- }
-
- return wait.get().pid();
-}
-
-
-Future<bool> DockerContainerizerProcess::launch(
- const ContainerID& containerId,
- const ExecutorInfo& executorInfo,
- const string& directory,
- const Option<string>& user,
- const SlaveID& slaveId,
- const PID<Slave>& slavePid,
- bool checkpoint)
-{
- if (containers_.contains(containerId)) {
- return Failure("Container already started");
- }
-
- if (!executorInfo.has_container()) {
- LOG(INFO) << "No container info found, skipping launch";
- return false;
- }
-
- ContainerInfo containerInfo = executorInfo.container();
-
- if (containerInfo.type() != ContainerInfo::DOCKER) {
- LOG(INFO) << "Skipping non-docker container";
- return false;
- }
-
- Try<Container*> container = Container::create(
- containerId,
- None(),
- executorInfo,
- directory,
- user,
- slaveId,
- slavePid,
- checkpoint,
- flags);
-
- if (container.isError()) {
- return Failure("Failed to create container: " + container.error());
- }
-
- containers_[containerId] = container.get();
-
- LOG(INFO) << "Starting container '" << containerId
- << "' for executor '" << executorInfo.executor_id()
- << "' and framework '" << executorInfo.framework_id() << "'";
-
- return container.get()->launch = fetch(containerId)
- .then(defer(self(), &Self::_launch, containerId))
- .then(defer(self(), &Self::__launch, containerId))
- .then(defer(self(), &Self::____launch, containerId))
- .then(defer(self(), &Self::_____launch, containerId, lambda::_1))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1))
- .then(defer(self(), &Self::_______launch, containerId, lambda::_1))
-}
-
-
Future<Docker::Container> DockerContainerizerProcess::____launch(
- const ContainerID& containerId)
+ const ContainerID& containerId,
+ const std::string& container)
{
CHECK(containers_.contains(containerId));
- return docker->inspect(containers_[containerId]->name());
+ return docker->inspect(container);
}
@@ -1005,6 +921,8 @@ Future<pid_t> DockerContainerizerProcess::_____launch(
// after we set Container::status.
CHECK(containers_.contains(containerId));
+ Container* container_ = containers_[containerId];
+
Option<int> pid = container.pid;
if (!pid.isSome()) {
@@ -1018,48 +936,16 @@ Future<pid_t> DockerContainerizerProcess::_____launch(
"Failed to checkpoint executor's pid: " + checkpointed.error());
}
- return pid.get();
-}
-
-
-Future<pid_t> DockerContainerizerProcess::______launch(
- const ContainerID& containerId,
- pid_t pid)
-{
- CHECK(containers_.contains(containerId));
-
- Container* container = containers_[containerId];
-
- docker->logs(container->name(), container->directory);
-
- return pid;
-}
-
-
-Future<pid_t> DockerContainerizerProcess::____launchInContainer(
- const ContainerID& containerId)
-{
- CHECK(containers_.contains(containerId));
-
- Try<pid_t> waitPid =
- launchWaitProcess(flags.docker, containers_[containerId]->executorName());
-
- if (waitPid.isError()) {
- return Failure(waitPid.error());
- }
-
- Try<Nothing> checkpointed = checkpoint(containerId, waitPid.get());
-
- if (checkpointed.isError()) {
- return Failure(
- "Failed to checkpoint executor's pid: " + checkpointed.error());
- }
+ // TODO(tnachen): We need to handle the slave in container scenario.
+ // Instead of forking a log process, launch a log container for
+ // executor.
+ docker->logs(container_->name(), container_->directory);
- return waitPid.get();
+ return pid.get();
}
-Future<bool> DockerContainerizerProcess::_______launch(
+Future<bool> DockerContainerizerProcess::______launch(
const ContainerID& containerId,
pid_t pid)
{
@@ -1108,11 +994,6 @@ Future<Nothing> DockerContainerizerProcess::update(
// Store the resources for usage().
container->resources = _resources;
- if (flags.docker_mesos_image.isSome()) {
- LOG(INFO) << "Ignoring update as slave is running under container.";
- return Nothing();
- }
-
#ifdef __linux__
if (!_resources.cpus().isSome() && !_resources.mem().isSome()) {
LOG(WARNING) << "Ignoring update as no supported resources are present";
@@ -1472,33 +1353,8 @@ void DockerContainerizerProcess::destroy(
CHECK(container->state == Container::RUNNING);
- // Remove the executor docker containers. They might not
- // been configured to launch but we might have recovered containers
- // on previous slave run that has configured to launch executor in
- // docker.
- docker->stop(container->executorName(), Seconds(0));
-
container->state = Container::DESTROYING;
- if (container->executorPid.isSome()) {
- LOG(INFO) << "Sending SIGTERM to executor with pid: "
- << container->executorPid.get();
- // We need to clean up the executor as the executor might not have
- // received run task due to a failed containerizer update.
- // We also kill the executor first since container->status below
- // is waiting for the executor to finish.
- Try<std::list<os::ProcessTree>> kill =
- os::killtree(container->executorPid.get(), SIGTERM);
-
- if (kill.isError()) {
- // Ignoring the error from killing executor as it can already
- // have exited.
- VLOG(1) << "Ignoring error when killing executor pid "
- << container->executorPid.get() << " in destroy, error: "
- << kill.error();
- }
- }
-
// Otherwise, wait for Docker::run to succeed, in which case we'll
// continue in _destroy (calling Docker::kill) or for Docker::run to
// fail, in which case we'll re-execute this function and cleanup
http://git-wip-us.apache.org/repos/asf/mesos/blob/0b5f67ee/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index 0bfb0e9..2961048 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -128,16 +128,7 @@ public:
virtual process::Future<bool> launch(
const ContainerID& containerId,
- const ExecutorInfo& executorInfo,
- const std::string& directory,
- const Option<std::string>& user,
- const SlaveID& slaveId,
- const process::PID<Slave>& slavePid,
- bool checkpoint);
-
- virtual process::Future<bool> launch(
- const ContainerID& containerId,
- const TaskInfo& taskInfo,
+ const Option<TaskInfo>& taskInfo,
const ExecutorInfo& executorInfo,
const std::string& directory,
const Option<std::string>& user,
@@ -192,27 +183,20 @@ private:
const ContainerID& containerId);
process::Future<Nothing> __launch(
- const ContainerID& containerId);
+ const ContainerID& containerId,
+ const std::string& container);
// NOTE: This continuation is only applicable when launching a
// container for a task.
process::Future<pid_t> ___launch(
const ContainerID& containerId);
- // NOTE: This continuation is only applicable when launching a
- // container for a task.
- process::Future<Nothing> ___launchInContainer(
- const ContainerID& containerId);
-
- // NOTE: This continuation is only applicable when launching a
- // container for a task.
- process::Future<pid_t> ____launchInContainer(
- const ContainerID& containerId);
// NOTE: This continuation is only applicable when launching a
// container for an executor.
process::Future<Docker::Container> ____launch(
- const ContainerID& containerId);
+ const ContainerID& containerId,
+ const std::string& container);
// NOTE: This continuation is only applicable when launching a
// container for an executor.
@@ -220,20 +204,10 @@ private:
const ContainerID& containerId,
const Docker::Container& container);
- process::Future<pid_t> ______launch(
- const ContainerID& containerId,
- pid_t pid);
-
- // NOTE: This continuation is only applicable when launching a
- // container for a task.
- process::Future<pid_t> ______launchInContainer(
+ process::Future<bool> ______launch(
const ContainerID& containerId,
pid_t pid);
- process::Future<bool> _______launch(
- const ContainerID& containerId,
- pid_t pid);
-
void _destroy(
const ContainerID& containerId,
bool killed);
@@ -332,6 +306,14 @@ private:
if (task.isSome()) {
CHECK(resources.contains(task.get().resources()));
}
+
+ overrideEnvironment = executorEnvironment(
+ executor,
+ directory,
+ slaveId,
+ slavePid,
+ checkpoint,
+ flags.recovery_timeout);
}
~Container()
@@ -374,6 +356,10 @@ private:
ContainerInfo container() const
{
+ if (overrideContainer.isSome()) {
+ return overrideContainer.get();
+ }
+
if (task.isSome()) {
return task.get().container();
}
@@ -383,6 +369,10 @@ private:
CommandInfo command() const
{
+ if (overrideCommand.isSome()) {
+ return overrideCommand.get();
+ }
+
if (task.isSome()) {
return task.get().command();
}
@@ -394,14 +384,8 @@ private:
// the Docker container (beyond the those found in CommandInfo).
std::map<std::string, std::string> environment() const
{
- if (task.isNone()) {
- return executorEnvironment(
- executor,
- directory,
- slaveId,
- slavePid,
- checkpoint,
- flags.recovery_timeout);
+ if (overrideEnvironment.isSome()) {
+ return overrideEnvironment.get();
}
return std::map<std::string, std::string>();
@@ -436,7 +420,7 @@ private:
DESTROYING = 4
} state;
- ContainerID id;
+ const ContainerID id;
Option<TaskInfo> task;
ExecutorInfo executor;
@@ -451,6 +435,10 @@ private:
bool symlinked;
Flags flags;
+ Option<ContainerInfo> overrideContainer;
+ Option<CommandInfo> overrideCommand;
+ Option<std::map<std::string, std::string>> overrideEnvironment;
+
// Promise for future returned from wait().
Promise<containerizer::Termination> termination;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0b5f67ee/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 7efaea4..e5f24e6 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -2593,10 +2593,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_ExecutorCleanupWhenLaunchFailed)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
- // Skip logging so we can avoid waiting for it to terminate.
- EXPECT_CALL(*mockDocker, logs(_, _))
- .WillOnce(Return(Nothing()));
-
Fetcher fetcher;
// The docker containerizer will free the process, so we must
[12/20] mesos git commit: Fix docker tests and warnings running in
verbose logging.
Posted by tn...@apache.org.
Fix docker tests and warnings running in verbose logging.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/72f62aa3
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/72f62aa3
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/72f62aa3
Branch: refs/heads/master
Commit: 72f62aa35d4ed060faaa863718cc0e1e416c7e52
Parents: 9f91e07
Author: Timothy Chen <tn...@gmail.com>
Authored: Fri Jan 16 23:09:43 2015 +0000
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
src/docker/docker.hpp | 2 +-
src/docker/executor.cpp | 7 ++-
src/tests/docker_containerizer_tests.cpp | 64 ++++++++++++++++++---------
3 files changed, 49 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/72f62aa3/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 0ba7e0e..2e2053e 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -98,7 +98,7 @@ public:
// time for docker to wait after stopping a container before killing it.
// A value of zero (the default value) is the same as issuing a
// 'docker kill CONTAINER'.
- process::Future<Nothing> stop(
+ virtual process::Future<Nothing> stop(
const std::string& container,
const Duration& timeout = Seconds(0),
bool remove = false) const;
http://git-wip-us.apache.org/repos/asf/mesos/blob/72f62aa3/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 227b3d1..fda7b2a 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -36,6 +36,7 @@
#include "docker/docker.hpp"
+#include "logging/flags.hpp"
#include "logging/logging.hpp"
using std::cerr;
@@ -305,7 +306,7 @@ void usage(const char* argv0, const flags::FlagsBase& flags)
}
-class Flags : public flags::FlagsBase
+class Flags : public mesos::internal::logging::Flags
{
public:
Flags()
@@ -337,6 +338,8 @@ public:
int main(int argc, char** argv)
{
+ GOOGLE_PROTOBUF_VERIFY_VERSION;
+
Flags flags;
bool help;
@@ -354,6 +357,8 @@ int main(int argc, char** argv)
return -1;
}
+ mesos::internal::logging::initialize(argv[0], flags, true); // Catch signals.
+
if (help) {
usage(argv[0], flags);
return -1;
http://git-wip-us.apache.org/repos/asf/mesos/blob/72f62aa3/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 378a526..7efaea4 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -203,7 +203,6 @@ public:
return false;
}
-
static bool running(
const process::Shared<Docker>& docker,
const SlaveID& slaveId,
@@ -232,6 +231,16 @@ public:
return false;
}
+ static bool containsLine(string text, string expectedLine)
+ {
+ foreach (const string& line, strings::split(text, "\n")) {
+ if (line == expectedLine) {
+ return true;
+ }
+ }
+
+ return false;
+ }
virtual void TearDown()
{
@@ -836,6 +845,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Kill)
AWAIT_READY_FOR(statusRunning, Seconds(60));
EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+ ASSERT_TRUE(running(docker, slaveId, containerId.get()));
+
Future<TaskStatus> statusKilled;
EXPECT_CALL(sched, statusUpdate(&driver, _))
.WillOnce(FutureArg<1>(&statusKilled));
@@ -1156,6 +1167,11 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
MockDocker* mockDocker = new MockDocker(tests::flags.docker);
Shared<Docker> docker(mockDocker);
+ Future<string> stoppedContainer;
+ EXPECT_CALL(*mockDocker, stop(_, _, _))
+ .WillOnce(DoAll(FutureArg<0>(&stoppedContainer),
+ Return(Nothing())));
+
Fetcher fetcher;
MockDockerContainerizer dockerContainerizer(flags, &fetcher, docker);
@@ -1207,6 +1223,12 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
AWAIT_READY(d1);
AWAIT_READY(d2);
+ ASSERT_TRUE(running(docker, slaveId, containerId));
+ ASSERT_TRUE(running(docker, slaveId, reapedContainerId));
+
+ Future<Docker::Container> inspect = docker->inspect(container2);
+ AWAIT_READY(inspect);
+
SlaveState slaveState;
slaveState.id = slaveId;
FrameworkState frameworkState;
@@ -1224,11 +1246,6 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
ASSERT_SOME(wait);
- Try<process::Subprocess> reaped =
- process::subprocess(tests::flags.docker + " wait " + container2);
-
- ASSERT_SOME(reaped);
-
FrameworkID frameworkId;
RunState runState;
@@ -1250,7 +1267,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Recover)
AWAIT_FAILED(dockerContainerizer.wait(reapedContainerId));
- AWAIT_READY(reaped.get().status());
+ AWAIT_EQ(inspect.get().id, stoppedContainer);
Shutdown();
}
@@ -1406,14 +1423,14 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Logs)
Try<string> read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_TRUE(strings::contains(read.get(), "err" + uuid));
- EXPECT_FALSE(strings::contains(read.get(), "out" + uuid));
+ EXPECT_TRUE(containsLine(read.get(), "err" + uuid));
+ EXPECT_FALSE(containsLine(read.get(), "out" + uuid));
read = os::read(path::join(directory.get(), "stdout"));
ASSERT_SOME(read);
- EXPECT_TRUE(strings::contains(read.get(), "out" + uuid));
- EXPECT_FALSE(strings::contains(read.get(), "err" + uuid));
+ EXPECT_TRUE(containsLine(read.get(), "out" + uuid));
+ EXPECT_FALSE(containsLine(read.get(), "err" + uuid));
driver.stop();
driver.join();
@@ -1469,6 +1486,8 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
const Offer& offer = offers.get()[0];
+ SlaveID slaveId = offer.slave_id();
+
TaskInfo task;
task.set_name("");
task.mutable_task_id()->set_value("1");
@@ -1526,11 +1545,12 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD)
// Since we're not passing any command value, we're expecting the
// default entry point to be run which is 'echo' with the default
// command from the image which is 'inky'.
- EXPECT_TRUE(strings::contains(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(read.get(), "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(strings::contains(read.get(), "inky"));
+
+ EXPECT_FALSE(containsLine(read.get(), "inky"));
driver.stop();
driver.join();
@@ -1646,13 +1666,13 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Override)
// We expect the passed in command value to override the image's
// default command, thus we should see the value of 'uuid' in the
// output instead of the default command which is 'inky'.
- EXPECT_TRUE(strings::contains(read.get(), uuid));
- EXPECT_FALSE(strings::contains(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(read.get(), uuid));
+ EXPECT_FALSE(containsLine(read.get(), "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(strings::contains(read.get(), "inky"));
- EXPECT_FALSE(strings::contains(read.get(), uuid));
+ EXPECT_FALSE(containsLine(read.get(), "inky"));
+ EXPECT_FALSE(containsLine(read.get(), uuid));
driver.stop();
driver.join();
@@ -1769,13 +1789,13 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Default_CMD_Args)
// We expect the passed in command arguments to override the image's
// default command, thus we should see the value of 'uuid' in the
// output instead of the default command which is 'inky'.
- EXPECT_TRUE(strings::contains(read.get(), uuid));
- EXPECT_FALSE(strings::contains(read.get(), "inky"));
+ EXPECT_TRUE(containsLine(read.get(), uuid));
+ EXPECT_FALSE(containsLine(read.get(), "inky"));
read = os::read(path::join(directory.get(), "stderr"));
ASSERT_SOME(read);
- EXPECT_FALSE(strings::contains(read.get(), "inky"));
- EXPECT_FALSE(strings::contains(read.get(), uuid));
+ EXPECT_FALSE(containsLine(read.get(), "inky"));
+ EXPECT_FALSE(containsLine(read.get(), uuid));
driver.stop();
driver.join();
@@ -2235,7 +2255,7 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
// We expect the uuid that is sent to host port to be written
// to stdout by the docker container running nc -l.
- EXPECT_TRUE(strings::contains(read.get(), uuid));
+ EXPECT_TRUE(containsLine(read.get(), uuid));
driver.stop();
driver.join();
[18/20] mesos git commit: Address comments for Docker containerizer
changes.
Posted by tn...@apache.org.
Address comments for Docker containerizer changes.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3baa6096
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3baa6096
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3baa6096
Branch: refs/heads/master
Commit: 3baa60965407bf0c3eb9c3da1b2ba7c0a4fee968
Parents: 0b5f67e
Author: Timothy Chen <tn...@apache.org>
Authored: Thu Apr 30 00:43:14 2015 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sun May 24 22:27:40 2015 -0700
----------------------------------------------------------------------
docs/configuration.md | 18 +-
src/Makefile.am | 1 +
src/docker/docker.cpp | 293 +++++++++++----------
src/docker/docker.hpp | 86 ++++---
src/docker/executor.cpp | 187 +++++++-------
src/docker/executor.hpp | 69 +++++
src/slave/constants.cpp | 3 +
src/slave/constants.hpp | 8 +
src/slave/containerizer/docker.cpp | 342 +++++++++++++-----------
src/slave/containerizer/docker.hpp | 165 ++++++------
src/slave/flags.cpp | 20 +-
src/tests/docker_containerizer_tests.cpp | 357 +++++++++++++-------------
src/tests/docker_tests.cpp | 30 +--
13 files changed, 872 insertions(+), 707 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index 0a7629a..5a41477 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -862,7 +862,12 @@ file:///path/to/file (where file contains one of the above)</code></pre>
--[no-]docker_kill_orphans
</td>
<td>
- Enable docker containerizer to kill orphaned containers
+ Enable docker containerizer to kill orphaned containers.
+ You should consider setting this to false when you launch multiple
+ slaves in the same OS, to avoid one of the DockerContainerizer removing
+ docker tasks launched by other slaves. However you should also make sure
+ you enable checkpoint for the slave so the same slave id can be reused,
+ otherwise docker tasks on slave restart will not be cleaned up.
(default: true)
</td>
</tr>
@@ -871,9 +876,9 @@ file:///path/to/file (where file contains one of the above)</code></pre>
--docker_sock=VALUE
</td>
<td>
- The docker UNIX socket path that the CLI uses to communicate to the
- daemon. We need this to launch docker containers that can run docker
- CLI.
+ The UNIX socket path to be mounted into the docker executor container to
+ provide docker CLI access to the docker daemon. This must be the path used
+ by the slave's docker image.
(default: /var/run/docker.sock)
</td>
</tr>
@@ -883,9 +888,10 @@ file:///path/to/file (where file contains one of the above)</code></pre>
</td>
<td>
The docker image used to launch this mesos slave instance.
- If a image is specified, the docker containerizer assumes the slave
+ If an image is specified, the docker containerizer assumes the slave
is running in a docker container, and launches executors with
- docker containers in order to recover them when the slave recovers.
+ docker containers in order to recover them when the slave restarts and
+ recovers.
</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 1e56ae5..814468e 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -323,6 +323,7 @@ libmesos_no_3rdparty_la_SOURCES = \
common/values.cpp \
docker/docker.hpp \
docker/docker.cpp \
+ docker/executor.hpp \
exec/exec.cpp \
files/files.cpp \
hook/manager.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 08f2023..ee74da5 100644
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -41,6 +41,8 @@
#include "slave/containerizer/isolators/cgroups/cpushare.hpp"
#include "slave/containerizer/isolators/cgroups/mem.hpp"
+#include "slave/constants.hpp"
+
using namespace mesos;
using namespace mesos::internal::slave;
@@ -110,16 +112,16 @@ Try<Docker*> Docker::create(const string& path, bool validate)
delete docker;
return Error("Failed to find a mounted cgroups hierarchy "
"for the 'cpu' subsystem; you probably need "
- "to mount cgroups manually!");
+ "to mount cgroups manually");
}
#endif // __linux__
// Validate the version (and that we can use Docker at all).
Future<Version> version = docker->version();
- if(!version.await(Seconds(5))) {
+ if (!version.await(DOCKER_VERSION_WAIT_TIMEOUT)) {
delete docker;
- return Error("Timed out getting docker version.");
+ return Error("Timed out getting docker version");
}
if (version.isFailed()) {
@@ -129,7 +131,7 @@ Try<Docker*> Docker::create(const string& path, bool validate)
if (version.get() < Version(1, 0, 0)) {
delete docker;
- return Error("Insufficient version of Docker! Please upgrade to >= 1.0.0");
+ return Error("Insufficient version of Docker. Please upgrade to >= 1.0.0");
}
return docker;
@@ -165,7 +167,7 @@ Future<Version> Docker::version() const
Future<Version> Docker::_version(const string& cmd, const Subprocess& s)
{
const Option<int>& status = s.status().get();
- if (!status.isSome() || status.get() != 0) {
+ if (status.isNone() || status.get() != 0) {
string msg = "Failed to execute '" + cmd + "': ";
if (status.isSome()) {
msg += WSTRINGIFY(status.get());
@@ -198,66 +200,62 @@ Future<Version> Docker::__version(const Future<string>& output)
}
}
- return Failure("Unable to find docker version in output.");
+ return Failure("Unable to find docker version in output");
}
Try<Docker::Container> Docker::Container::create(const JSON::Object& json)
{
- map<string, JSON::Value>::const_iterator entry =
- json.values.find("Id");
- if (entry == json.values.end()) {
+ Result<JSON::String> idValue = json.find<JSON::String>("Id");
+ if (idValue.isNone()) {
return Error("Unable to find Id in container");
+ } else if (idValue.isError()) {
+ return Error("Error finding Id in container: " + idValue.error());
}
- JSON::Value idValue = entry->second;
- if (!idValue.is<JSON::String>()) {
- return Error("Id in container is not a string type");
- }
-
- string id = idValue.as<JSON::String>().value;
+ string id = idValue.get().value;
- entry = json.values.find("Name");
- if (entry == json.values.end()) {
+ Result<JSON::String> nameValue = json.find<JSON::String>("Name");
+ if (nameValue.isNone()) {
return Error("Unable to find Name in container");
+ } else if (nameValue.isError()) {
+ return Error("Error finding Name in container: " + nameValue.error());
}
- JSON::Value nameValue = entry->second;
- if (!nameValue.is<JSON::String>()) {
- return Error("Name in container is not string type");
- }
-
- string name = nameValue.as<JSON::String>().value;
+ string name = nameValue.get().value;
- entry = json.values.find("State");
- if (entry == json.values.end()) {
+ Result<JSON::Object> stateValue = json.find<JSON::Object>("State");
+ if (stateValue.isNone()) {
return Error("Unable to find State in container");
+ } else if (stateValue.isError()) {
+ return Error("Error finding State in container: " + stateValue.error());
}
- JSON::Value stateValue = entry->second;
- if (!stateValue.is<JSON::Object>()) {
- return Error("State in container is not object type");
- }
-
- entry = stateValue.as<JSON::Object>().values.find("Pid");
- if (entry == json.values.end()) {
+ Result<JSON::Number> pidValue = stateValue.get().find<JSON::Number>("Pid");
+ if (pidValue.isNone()) {
return Error("Unable to find Pid in State");
+ } else if (pidValue.isError()) {
+ return Error("Error finding Pid in State: " + pidValue.error());
}
- // TODO(yifan): Reload operator '=' to reuse the value variable above.
- JSON::Value pidValue = entry->second;
- if (!pidValue.is<JSON::Number>()) {
- return Error("Pid in State is not number type");
- }
-
- pid_t pid = pid_t(pidValue.as<JSON::Number>().value);
+ pid_t pid = pid_t(pidValue.get().value);
Option<pid_t> optionalPid;
if (pid != 0) {
optionalPid = pid;
}
- return Docker::Container(id, name, optionalPid);
+ Result<JSON::String> startedAtValue =
+ stateValue.get().find<JSON::String>("StartedAt");
+ if (startedAtValue.isNone()) {
+ return Error("Unable to find StartedAt in State");
+ } else if (startedAtValue.isError()) {
+ return Error("Error finding StartedAt in State: " + startedAtValue.error());
+ }
+
+ bool started = startedAtValue.get().value != "0001-01-01T00:00:00Z";
+
+ return Docker::Container(id, name, optionalPid, started);
}
@@ -307,8 +305,9 @@ Future<Nothing> Docker::run(
const string& sandboxDirectory,
const string& mappedDirectory,
const Option<Resources>& resources,
- const Option<map<string, string> >& env,
- bool detached) const
+ const Option<map<string, string>>& env,
+ const Option<string>& stdoutPath,
+ const Option<string>& stderrPath) const
{
if (!containerInfo.has_docker()) {
return Failure("No docker info found in container info");
@@ -320,10 +319,6 @@ Future<Nothing> Docker::run(
argv.push_back(path);
argv.push_back("run");
- if (detached) {
- argv.push_back("-d");
- }
-
if (dockerInfo.privileged()) {
argv.push_back("--privileged");
}
@@ -381,7 +376,7 @@ Future<Nothing> Docker::run(
argv.push_back(volumeConfig);
}
- // Mapping sandbox directory into the contianer mapped directory.
+ // Mapping sandbox directory into the container mapped directory.
argv.push_back("-v");
argv.push_back(sandboxDirectory + ":" + mappedDirectory);
@@ -501,16 +496,22 @@ Future<Nothing> Docker::run(
// URI downloads.
environment["HOME"] = sandboxDirectory;
+ Subprocess::IO stdoutIo = Subprocess::PIPE();
+ Subprocess::IO stderrIo = Subprocess::PIPE();
+ if (stdoutPath.isSome()) {
+ stdoutIo = Subprocess::PATH(stdoutPath.get());
+ }
+
+ if (stderrPath.isSome()) {
+ stderrIo = Subprocess::PATH(stderrPath.get());
+ }
+
Try<Subprocess> s = subprocess(
path,
argv,
Subprocess::PATH("/dev/null"),
- (detached
- ? Subprocess::PIPE()
- : Subprocess::PATH(path::join(sandboxDirectory, "stdout"))),
- (detached
- ? Subprocess::PIPE()
- : Subprocess::PATH(path::join(sandboxDirectory, "stderr"))),
+ stdoutIo,
+ stderrIo,
None(),
environment);
@@ -518,10 +519,9 @@ Future<Nothing> Docker::run(
return Failure(s.error());
}
- if (detached) {
- return checkError(cmd, s.get());
- }
-
+ // We don't call checkError here to avoid printing the stderr
+ // of the docker container task as docker run with attach forwards
+ // the container's stderr to the client's stderr.
return s.get().status()
.then(lambda::bind(
&Docker::_run,
@@ -533,7 +533,7 @@ Future<Nothing> Docker::run(
Future<Nothing> Docker::_run(const Option<int>& status)
{
if (status.isNone()) {
- return Failure("Failed to get exit status.");
+ return Failure("Failed to get exit status");
} else if (status.get() != 0) {
return Failure("Container exited on error: " + WSTRINGIFY(status.get()));
}
@@ -543,7 +543,7 @@ Future<Nothing> Docker::_run(const Option<int>& status)
Future<Nothing> Docker::stop(
- const string& container,
+ const string& containerName,
const Duration& timeout,
bool remove) const
{
@@ -553,7 +553,8 @@ Future<Nothing> Docker::stop(
stringify(timeoutSecs));
}
- string cmd = path + " stop -t " + stringify(timeoutSecs) + " " + container;
+ string cmd = path + " stop -t " + stringify(timeoutSecs) +
+ " " + containerName;
VLOG(1) << "Running " << cmd;
@@ -571,7 +572,7 @@ Future<Nothing> Docker::stop(
.then(lambda::bind(
&Docker::_stop,
*this,
- container,
+ containerName,
cmd,
s.get(),
remove));
@@ -579,7 +580,7 @@ Future<Nothing> Docker::stop(
Future<Nothing> Docker::_stop(
const Docker& docker,
- const string& container,
+ const string& containerName,
const string& cmd,
const Subprocess& s,
bool remove)
@@ -588,7 +589,7 @@ Future<Nothing> Docker::_stop(
if (remove) {
bool force = !status.isSome() || status.get() != 0;
- return docker.rm(container, force);
+ return docker.rm(containerName, force);
}
return checkError(cmd, s);
@@ -596,10 +597,10 @@ Future<Nothing> Docker::_stop(
Future<Nothing> Docker::rm(
- const string& container,
+ const string& containerName,
bool force) const
{
- const string cmd = path + (force ? " rm -f " : " rm ") + container;
+ const string cmd = path + (force ? " rm -f " : " rm ") + containerName;
VLOG(1) << "Running " << cmd;
@@ -617,9 +618,28 @@ Future<Nothing> Docker::rm(
}
-Future<Docker::Container> Docker::inspect(const string& container) const
+Future<Docker::Container> Docker::inspect(
+ const string& containerName,
+ const Option<Duration>& retryInterval) const
{
- const string cmd = path + " inspect " + container;
+ Owned<Promise<Docker::Container>> promise(new Promise<Docker::Container>());
+
+ const string cmd = path + " inspect " + containerName;
+ _inspect(cmd, promise, retryInterval);
+
+ return promise->future();
+}
+
+
+void Docker::_inspect(
+ const string& cmd,
+ const Owned<Promise<Docker::Container>>& promise,
+ const Option<Duration>& retryInterval)
+{
+ if (promise->future().hasDiscard()) {
+ promise->discard();
+ return;
+ }
VLOG(1) << "Running " << cmd;
@@ -630,48 +650,86 @@ Future<Docker::Container> Docker::inspect(const string& container) const
Subprocess::PIPE());
if (s.isError()) {
- return Failure(s.error());
+ promise->fail(s.error());
+ return;
}
- return s.get().status()
- .then(lambda::bind(&Docker::_inspect, cmd, s.get()));
+ s.get().status()
+ .onAny([=]() { __inspect(cmd, promise, retryInterval, s.get()); });
}
-Future<Docker::Container> Docker::_inspect(
+void Docker::__inspect(
const string& cmd,
+ const Owned<Promise<Docker::Container>>& promise,
+ const Option<Duration>& retryInterval,
const Subprocess& s)
{
+ if (promise->future().hasDiscard()) {
+ promise->discard();
+ return;
+ }
+
// Check the exit status of 'docker inspect'.
CHECK_READY(s.status());
Option<int> status = s.status().get();
if (!status.isSome()) {
- return Failure("No status found from '" + cmd + "'");
+ promise->fail("No status found from '" + cmd + "'");
} else if (status.get() != 0) {
+ if (retryInterval.isSome()) {
+ VLOG(1) << "Retrying inspect with non-zero status code. cmd: '"
+ << cmd << "', interval: " << stringify(retryInterval.get());
+ Clock::timer(retryInterval.get(),
+ [=]() { _inspect(cmd, promise, retryInterval); } );
+ return;
+ }
+
CHECK_SOME(s.err());
- return io::read(s.err().get())
+ io::read(s.err().get())
.then(lambda::bind(
- failure<Docker::Container>,
+ failure<Nothing>,
cmd,
status.get(),
- lambda::_1));
+ lambda::_1))
+ .onAny([=](const Future<Nothing>& future) {
+ CHECK_FAILED(future);
+ promise->fail(future.failure());
+ });
+ return;
}
// Read to EOF.
CHECK_SOME(s.out());
- return io::read(s.out().get())
- .then(lambda::bind(&Docker::__inspect, lambda::_1));
+ io::read(s.out().get())
+ .onAny([=](const Future<string>& output) {
+ ___inspect(cmd, promise, retryInterval, output);
+ });
}
-Future<Docker::Container> Docker::__inspect(const string& output)
+void Docker::___inspect(
+ const string& cmd,
+ const Owned<Promise<Docker::Container>>& promise,
+ const Option<Duration>& retryInterval,
+ const Future<string>& output)
{
- Try<JSON::Array> parse = JSON::parse<JSON::Array>(output);
+ if (promise->future().hasDiscard()) {
+ promise->discard();
+ return;
+ }
+
+ if (!output.isReady()) {
+ promise->fail(output.isFailed() ? output.failure() : "future discarded");
+ return;
+ }
+
+ Try<JSON::Array> parse = JSON::parse<JSON::Array>(output.get());
if (parse.isError()) {
- return Failure("Failed to parse JSON: " + parse.error());
+ promise->fail("Failed to parse JSON: " + parse.error());
+ return;
}
JSON::Array array = parse.get();
@@ -682,69 +740,30 @@ Future<Docker::Container> Docker::__inspect(const string& output)
Docker::Container::create(array.values.front().as<JSON::Object>());
if (container.isError()) {
- return Failure("Unable to create container: " + container.error());
+ promise->fail("Unable to create container: " + container.error());
+ return;
}
- return container.get();
+ if (retryInterval.isSome() && !container.get().started) {
+ VLOG(1) << "Retrying inspect since container not yet started. cmd: '"
+ << cmd << "', interval: " << stringify(retryInterval.get());
+ Clock::timer(retryInterval.get(),
+ [=]() { _inspect(cmd, promise, retryInterval); } );
+ return;
+ }
+
+ promise->set(container.get());
+ return;
}
// TODO(benh): Handle the case where the short container ID was
// not sufficiently unique and 'array.values.size() > 1'.
- return Failure("Failed to find container");
-}
-
-
-Future<Nothing> Docker::logs(
- const std::string& container,
- const std::string& directory) const
-{
- // Redirect the logs into stdout/stderr.
- //
- // TODO(benh): This is an intermediate solution for now until we can
- // reliably stream the logs either from the CLI or from the REST
- // interface directly. The problem is that it's possible that the
- // 'docker logs --follow' command will be started AFTER the
- // container has already terminated, and thus it will continue
- // running forever because the container has stopped. Unfortunately,
- // when we later remove the container that still doesn't cause the
- // 'logs' process to exit. Thus, we wait some period of time until
- // after the container has terminated in order to let any log data
- // get flushed, then we kill the 'logs' process ourselves. A better
- // solution would be to first "create" the container, then start
- // following the logs, then finally "start" the container so that
- // when the container terminates Docker will properly close the log
- // stream and 'docker logs' will exit. For more information, please
- // see: https://github.com/docker/docker/issues/7020
-
- string logs =
- "logs() {\n"
- " " + path + " logs --follow $1 &\n"
- " pid=$!\n"
- " " + path + " wait $1 >/dev/null 2>&1\n"
- " sleep 10\n" // Sleep 10 seconds to make sure the logs are flushed.
- " kill -TERM $pid >/dev/null 2>&1 &\n"
- "}\n"
- "logs " + container;
-
- VLOG(1) << "Running " << logs;
-
- Try<Subprocess> s = subprocess(
- logs,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH(path::join(directory, "stdout")),
- Subprocess::PATH(path::join(directory, "stderr")));
-
- if (s.isError()) {
- return Failure("Unable to launch docker logs: " + s.error());
- }
-
- return s.get().status()
- .then(lambda::bind(&_nothing));
+ promise->fail("Failed to find container");
}
-Future<list<Docker::Container> > Docker::ps(
+Future<list<Docker::Container>> Docker::ps(
bool all,
const Option<string>& prefix) const
{
@@ -772,7 +791,7 @@ Future<list<Docker::Container> > Docker::ps(
}
-Future<list<Docker::Container> > Docker::_ps(
+Future<list<Docker::Container>> Docker::_ps(
const Docker& docker,
const string& cmd,
const Subprocess& s,
@@ -789,7 +808,7 @@ Future<list<Docker::Container> > Docker::_ps(
CHECK_SOME(s.err());
return io::read(s.err().get())
.then(lambda::bind(
- failure<list<Docker::Container> >,
+ failure<list<Docker::Container>>,
cmd,
status.get(),
lambda::_1));
@@ -800,7 +819,7 @@ Future<list<Docker::Container> > Docker::_ps(
}
-Future<list<Docker::Container> > Docker::__ps(
+Future<list<Docker::Container>> Docker::__ps(
const Docker& docker,
const Option<string>& prefix,
const string& output)
@@ -811,7 +830,7 @@ Future<list<Docker::Container> > Docker::__ps(
CHECK(!lines.empty());
lines.erase(lines.begin());
- list<Future<Docker::Container> > futures;
+ list<Future<Docker::Container>> futures;
foreach (const string& line, lines) {
// Inspect the containers that we are interested in depending on
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index 3b6ea64..d06c73a 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -52,21 +52,26 @@ public:
static Try<Container> create(const JSON::Object& json);
// Returns the ID of the container.
- std::string id;
+ const std::string id;
// Returns the name of the container.
- std::string name;
+ const std::string name;
// Returns the pid of the container, or None if the container is
// not running.
- Option<pid_t> pid;
+ const Option<pid_t> pid;
+
+ // Returns if the container has already started. This field is
+ // needed since pid is empty when the container terminates.
+ const bool started;
private:
Container(
- const std::string& _id,
- const std::string& _name,
- const Option<pid_t>& _pid)
- : id(_id), name(_name), pid(_pid) {}
+ const std::string& id,
+ const std::string& name,
+ const Option<pid_t>& pid,
+ bool started)
+ : id(id), name(name), pid(pid), started(started) {}
};
class Image
@@ -74,26 +79,27 @@ public:
public:
static Try<Image> create(const JSON::Object& json);
- Option<std::vector<std::string> > entrypoint;
+ Option<std::vector<std::string>> entrypoint;
private:
- Image(const Option<std::vector<std::string> >& _entrypoint)
+ Image(const Option<std::vector<std::string>>& _entrypoint)
: entrypoint(_entrypoint) {}
};
- // Returns the current docker version.
- virtual process::Future<Version> version() const;
-
// Performs 'docker run IMAGE'.
virtual process::Future<Nothing> run(
const mesos::ContainerInfo& containerInfo,
const mesos::CommandInfo& commandInfo,
- const std::string& name,
+ const std::string& containerName,
const std::string& sandboxDirectory,
const std::string& mappedDirectory,
const Option<mesos::Resources>& resources = None(),
- const Option<std::map<std::string, std::string> >& env = None(),
- bool detached = true) const;
+ const Option<std::map<std::string, std::string>>& env = None(),
+ const Option<std::string>& stdoutPath = None(),
+ const Option<std::string>& stderrPath = None()) const;
+
+ // Returns the current docker version.
+ virtual process::Future<Version> version() const;
// Performs 'docker stop -t TIMEOUT CONTAINER'. If remove is true then a rm -f
// will be called when stop failed, otherwise a failure is returned. The
@@ -102,33 +108,27 @@ public:
// A value of zero (the default value) is the same as issuing a
// 'docker kill CONTAINER'.
virtual process::Future<Nothing> stop(
- const std::string& container,
+ const std::string& containerName,
const Duration& timeout = Seconds(0),
bool remove = false) const;
// Performs 'docker rm (-f) CONTAINER'.
virtual process::Future<Nothing> rm(
- const std::string& container,
+ const std::string& containerName,
bool force = false) const;
- // Performs 'docker inspect CONTAINER'.
+ // Performs 'docker inspect CONTAINER'. If retryInterval is set,
+ // we will keep retrying inspect until the container is started or
+ // the future is discarded.
virtual process::Future<Container> inspect(
- const std::string& container) const;
+ const std::string& containerName,
+ const Option<Duration>& retryInterval = None()) const;
// Performs 'docker ps (-a)'.
- virtual process::Future<std::list<Container> > ps(
+ virtual process::Future<std::list<Container>> ps(
bool all = false,
const Option<std::string>& prefix = None()) const;
- // Performs a 'docker logs --follow' and sends the output into a
- // 'stderr' and 'stdout' file in the specfied directory.
- //
- // TODO(benh): Return the file descriptors, or some struct around
- // them so others can do what they want with stdout/stderr.
- virtual process::Future<Nothing> logs(
- const std::string& container,
- const std::string& directory) const;
-
virtual process::Future<Image> pull(
const std::string& directory,
const std::string& image,
@@ -139,6 +139,9 @@ protected:
Docker(const std::string& _path) : path(_path) {};
private:
+ static process::Future<Nothing> _run(
+ const Option<int>& status);
+
static process::Future<Version> _version(
const std::string& cmd,
const process::Subprocess& s);
@@ -148,29 +151,36 @@ private:
static process::Future<Nothing> _stop(
const Docker& docker,
- const std::string& container,
+ const std::string& containerName,
const std::string& cmd,
const process::Subprocess& s,
bool remove);
- static process::Future<Container> _inspect(
+ static void _inspect(
const std::string& cmd,
- const process::Subprocess& s);
+ const process::Owned<process::Promise<Container>>& promise,
+ const Option<Duration>& retryInterval);
- static process::Future<Container> __inspect(
- const std::string& output);
+ static void __inspect(
+ const std::string& cmd,
+ const process::Owned<process::Promise<Container>>& promise,
+ const Option<Duration>& retryInterval,
+ const process::Subprocess& s);
- static process::Future<Nothing> _run(
- const Option<int>& status);
+ static void ___inspect(
+ const std::string& cmd,
+ const process::Owned<process::Promise<Container>>& promise,
+ const Option<Duration>& retryInterval,
+ const process::Future<std::string>& output);
- static process::Future<std::list<Container> > _ps(
+ static process::Future<std::list<Container>> _ps(
const Docker& docker,
const std::string& cmd,
const process::Subprocess& s,
const Option<std::string>& prefix,
process::Future<std::string> output);
- static process::Future<std::list<Container> > __ps(
+ static process::Future<std::list<Container>> __ps(
const Docker& docker,
const Option<std::string>& prefix,
const std::string& output);
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 847997b..075c6b5 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -35,6 +35,7 @@
#include "common/status_utils.hpp"
#include "docker/docker.hpp"
+#include "docker/executor.hpp"
#include "logging/flags.hpp"
#include "logging/logging.hpp"
@@ -46,6 +47,7 @@ using std::string;
namespace mesos {
namespace internal {
+namespace docker {
using namespace mesos;
using namespace process;
@@ -55,9 +57,9 @@ using namespace process;
// redirect log output to configured stdout and stderr files.
// Similar to the CommandExecutor, it is only responsible to launch
// one container and exits afterwards.
-// The executor also assumes it is launched from the
-// DockerContainerizer, which already calls setsid before launching
-// the executor.
+// The executor assumes that it is launched from the
+// DockerContainerizer, which already sets up when launching the
+// executor that ensures its kept running if the slave exits.
class DockerExecutorProcess : public ProtobufProcess<DockerExecutorProcess>
{
public:
@@ -65,13 +67,15 @@ public:
const Owned<Docker>& docker,
const string& container,
const string& sandboxDirectory,
- const string& mappedDirectory)
- : launched(false),
- killed(false),
+ const string& mappedDirectory,
+ const Duration& stopTimeout)
+ : killed(false),
docker(docker),
container(container),
sandboxDirectory(sandboxDirectory),
- mappedDirectory(mappedDirectory) {}
+ mappedDirectory(mappedDirectory),
+ stopTimeout(stopTimeout),
+ stop(Nothing()) {}
virtual ~DockerExecutorProcess() {}
@@ -92,13 +96,16 @@ public:
cout << "Re-registered docker executor on " << slaveInfo.hostname() << endl;
}
- void disconnected(ExecutorDriver* driver) {}
+ void disconnected(ExecutorDriver* driver)
+ {
+ cout << "Disconnected from the slave" << endl;
+ }
void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
- if (launched) {
+ if (run.isSome()) {
TaskStatus status;
- status.mutable_task_id()->MergeFrom(task.task_id());
+ status.mutable_task_id()->CopyFrom(task.task_id());
status.set_state(TASK_FAILED);
status.set_message(
"Attempted to run multiple tasks using a \"docker\" executor");
@@ -109,46 +116,43 @@ public:
cout << "Starting task " << task.task_id().value() << endl;
- // We assume the Docker executor is launched from the
- // DockerContainerizer, which already calls setsid before
- // launching the executor.
-
CHECK(task.has_container());
CHECK(task.has_command());
- ContainerInfo containerInfo = task.container();
-
- CHECK(containerInfo.type() == ContainerInfo::DOCKER);
+ CHECK(task.container().type() == ContainerInfo::DOCKER);
- Future<Nothing> run = docker->run(
- containerInfo,
+ // We're adding task and executor resources to launch docker since
+ // the DockerContainerizer updates the container cgroup limits
+ // directly and it expects it to be the sum of both task and
+ // executor resources. This does leave to a bit of unaccounted
+ // resources for running this executor, but we are assuming
+ // this is just a very small amount of overcommit.
+ run = docker->run(
+ task.container(),
task.command(),
container,
sandboxDirectory,
mappedDirectory,
task.resources() + task.executor().resources(),
None(),
- false);
-
- run.onAny(defer(
+ path::join(sandboxDirectory, "stdout"),
+ path::join(sandboxDirectory, "stderr"))
+ .onAny(defer(
self(),
&Self::reaped,
driver,
task.task_id(),
lambda::_1));
- dockerRun = run;
-
TaskStatus status;
- status.mutable_task_id()->MergeFrom(task.task_id());
+ status.mutable_task_id()->CopyFrom(task.task_id());
status.set_state(TASK_RUNNING);
driver->sendStatusUpdate(status);
-
- launched = true;
}
void killTask(ExecutorDriver* driver, const TaskID& taskId)
{
+ cout << "Killing docker task" << endl;
shutdown(driver);
}
@@ -158,14 +162,19 @@ public:
{
cout << "Shutting down" << endl;
- if (dockerRun.isSome() && !killed) {
- Future<Nothing> dockerRun_ = dockerRun.get();
- dockerRun_.discard();
+ if (run.isSome() && !killed) {
+ // The docker daemon might still be in progress starting the
+ // container, therefore we kill both the docker run process
+ // and also ask the daemon to stop the container.
+
+ // Making a mutable copy of the future so we can call discard.
+ Future<Nothing>(run.get()).discard();
+ stop = docker->stop(container, stopTimeout);
killed = true;
}
}
- virtual void error(ExecutorDriver* driver, const string& message) {}
+ void error(ExecutorDriver* driver, const string& message) {}
private:
void reaped(
@@ -173,9 +182,22 @@ private:
const TaskID& taskId,
const Future<Nothing>& run)
{
+ stop.onAny(defer(self(), &Self::_reaped, driver, taskId, run, lambda::_1));
+ }
+
+ void _reaped(
+ ExecutorDriver* driver,
+ const TaskID& taskId,
+ const Future<Nothing>& run,
+ const Future<Nothing>& stop)
+ {
TaskState state;
string message;
- if (killed) {
+ if (!stop.isReady()) {
+ state = TASK_FAILED;
+ message = "Unable to stop docker container, error: " +
+ (stop.isFailed() ? stop.failure() : "future discarded");
+ } else if (killed) {
state = TASK_KILLED;
} else if (!run.isReady()) {
state = TASK_FAILED;
@@ -186,7 +208,7 @@ private:
}
TaskStatus taskStatus;
- taskStatus.mutable_task_id()->MergeFrom(taskId);
+ taskStatus.mutable_task_id()->CopyFrom(taskId);
taskStatus.set_state(state);
taskStatus.set_message(message);
@@ -194,18 +216,21 @@ private:
// A hack for now ... but we need to wait until the status update
// is sent to the slave before we shut ourselves down.
+ // TODO(tnachen): Remove this hack and also the same hack in the
+ // command executor when we have the new HTTP APIs to wait until
+ // an ack.
os::sleep(Seconds(1));
driver->stop();
}
-
- bool launched;
bool killed;
Owned<Docker> docker;
string container;
string sandboxDirectory;
string mappedDirectory;
- Option<Future<Nothing>> dockerRun;
+ Duration stopTimeout;
+ Option<Future<Nothing>> run;
+ Future<Nothing> stop;
Option<ExecutorDriver*> driver;
};
@@ -217,22 +242,23 @@ public:
const Owned<Docker>& docker,
const string& container,
const string& sandboxDirectory,
- const string& mappedDirectory)
+ const string& mappedDirectory,
+ const Duration& stopTimeout)
{
- process = new DockerExecutorProcess(
+ process = Owned<DockerExecutorProcess>(new DockerExecutorProcess(
docker,
container,
sandboxDirectory,
- mappedDirectory);
+ mappedDirectory,
+ stopTimeout));
- spawn(process);
+ spawn(process.get());
}
virtual ~DockerExecutor()
{
- terminate(process);
- wait(process);
- delete process;
+ terminate(process.get());
+ wait(process.get());
}
virtual void registered(
@@ -241,7 +267,7 @@ public:
const FrameworkInfo& frameworkInfo,
const SlaveInfo& slaveInfo)
{
- dispatch(process,
+ dispatch(process.get(),
&DockerExecutorProcess::registered,
driver,
executorInfo,
@@ -253,7 +279,7 @@ public:
ExecutorDriver* driver,
const SlaveInfo& slaveInfo)
{
- dispatch(process,
+ dispatch(process.get(),
&DockerExecutorProcess::reregistered,
driver,
slaveInfo);
@@ -261,38 +287,43 @@ public:
virtual void disconnected(ExecutorDriver* driver)
{
- dispatch(process, &DockerExecutorProcess::disconnected, driver);
+ dispatch(process.get(), &DockerExecutorProcess::disconnected, driver);
}
virtual void launchTask(ExecutorDriver* driver, const TaskInfo& task)
{
- dispatch(process, &DockerExecutorProcess::launchTask, driver, task);
+ dispatch(process.get(), &DockerExecutorProcess::launchTask, driver, task);
}
virtual void killTask(ExecutorDriver* driver, const TaskID& taskId)
{
- dispatch(process, &DockerExecutorProcess::killTask, driver, taskId);
+ dispatch(process.get(), &DockerExecutorProcess::killTask, driver, taskId);
}
virtual void frameworkMessage(ExecutorDriver* driver, const string& data)
{
- dispatch(process, &DockerExecutorProcess::frameworkMessage, driver, data);
+ dispatch(process.get(),
+ &DockerExecutorProcess::frameworkMessage,
+ driver,
+ data);
}
virtual void shutdown(ExecutorDriver* driver)
{
- dispatch(process, &DockerExecutorProcess::shutdown, driver);
+ dispatch(process.get(), &DockerExecutorProcess::shutdown, driver);
}
virtual void error(ExecutorDriver* driver, const string& data)
{
- dispatch(process, &DockerExecutorProcess::error, driver, data);
+ dispatch(process.get(), &DockerExecutorProcess::error, driver, data);
}
private:
- DockerExecutorProcess* process;
+ Owned<DockerExecutorProcess> process;
};
+
+} // namespace docker {
} // namespace internal {
} // namespace mesos {
@@ -306,41 +337,11 @@ void usage(const char* argv0, const flags::FlagsBase& flags)
}
-class Flags : public mesos::internal::logging::Flags
-{
-public:
- Flags()
- {
- add(&Flags::container,
- "container",
- "The name of the docker container to run.\n");
-
- add(&Flags::docker,
- "docker",
- "The path to the docker cli executable.\n");
-
- add(&Flags::sandbox_directory,
- "sandbox_directory",
- "The path to the container sandbox that stores stdout and stderr\n"
- "files that is being redirected with docker container logs.\n");
-
- add(&Flags::mapped_directory,
- "mapped_directory",
- "The sandbox directory path that is mapped in the docker container.\n");
- }
-
- Option<string> container;
- Option<string> docker;
- Option<string> sandbox_directory;
- Option<string> mapped_directory;
-};
-
-
int main(int argc, char** argv)
{
GOOGLE_PROTOBUF_VERIFY_VERSION;
- Flags flags;
+ mesos::internal::docker::Flags flags;
bool help;
flags.add(&help,
@@ -357,6 +358,8 @@ int main(int argc, char** argv)
return -1;
}
+ std::cout << stringify(flags) << std::endl;
+
mesos::internal::logging::initialize(argv[0], flags, true); // Catch signals.
if (help) {
@@ -364,6 +367,8 @@ int main(int argc, char** argv)
return -1;
}
+ std::cout << stringify(flags) << std::endl;
+
if (flags.docker.isNone()) {
LOG(WARNING) << "Expected docker executable path";
usage(argv[0], flags);
@@ -388,19 +393,27 @@ int main(int argc, char** argv)
return 0;
}
- // We skip validation when creating docker abstraction as we
- // don't want to be checking for cgroups in the executor.
+ if (flags.stop_timeout.isNone()) {
+ LOG(WARNING) << "Expected stop timeout";
+ usage(argv[0], flags);
+ return 0;
+ }
+
+ // The 2nd argument for docker create is set to false so we skip
+ // validation when creating a docker abstraction, as the slave
+ // should have already validated docker.
Try<Docker*> docker = Docker::create(flags.docker.get(), false);
if (docker.isError()) {
LOG(WARNING) << "Unable to create docker abstraction: " << docker.error();
return -1;
}
- mesos::internal::DockerExecutor executor(
+ mesos::internal::docker::DockerExecutor executor(
process::Owned<Docker>(docker.get()),
flags.container.get(),
flags.sandbox_directory.get(),
- flags.mapped_directory.get());
+ flags.mapped_directory.get(),
+ flags.stop_timeout.get());
mesos::MesosExecutorDriver driver(&executor);
return driver.run() == mesos::DRIVER_STOPPED ? 0 : 1;
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/docker/executor.hpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.hpp b/src/docker/executor.hpp
new file mode 100644
index 0000000..fa13b6e
--- /dev/null
+++ b/src/docker/executor.hpp
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __DOCKER_EXECUTOR_HPP__
+#define __DOCKER_EXECUTOR_HPP__
+
+#include <stdio.h>
+
+#include <process/process.hpp>
+
+#include "logging/flags.hpp"
+
+namespace mesos {
+namespace internal {
+namespace docker {
+
+struct Flags : public mesos::internal::logging::Flags
+{
+ Flags() {
+ add(&container,
+ "container",
+ "The name of the docker container to run.\n");
+
+ add(&docker,
+ "docker",
+ "The path to the docker executable.\n");
+
+ add(&sandbox_directory,
+ "sandbox_directory",
+ "The path to the container sandbox holding stdout and stderr files\n"
+ "into which docker container logs will be redirected.");
+
+ add(&mapped_directory,
+ "mapped_directory",
+ "The sandbox directory path that is mapped in the docker container.\n");
+
+ add(&stop_timeout,
+ "stop_timeout",
+ "The duration for docker to wait after stopping a running container\n"
+ "before it kills that container.");
+ }
+
+ Option<std::string> container;
+ Option<std::string> docker;
+ Option<std::string> sandbox_directory;
+ Option<std::string> mapped_directory;
+ Option<Duration> stop_timeout;
+};
+
+} // namespace docker {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __DOCKER_EXECUTOR_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/slave/constants.cpp
----------------------------------------------------------------------
diff --git a/src/slave/constants.cpp b/src/slave/constants.cpp
index 2a99b11..d8d2f98 100644
--- a/src/slave/constants.cpp
+++ b/src/slave/constants.cpp
@@ -50,6 +50,9 @@ const std::string DEFAULT_PORTS = "[31000-32000]";
const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER = 1024;
#endif
const Duration DOCKER_REMOVE_DELAY = Hours(6);
+const Duration DOCKER_INSPECT_DELAY = Seconds(1);
+// TODO(tnachen): Make this a flag.
+const Duration DOCKER_VERSION_WAIT_TIMEOUT = Seconds(5);
const std::string DEFAULT_AUTHENTICATEE = "crammd5";
Duration MASTER_PING_TIMEOUT()
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/slave/constants.hpp
----------------------------------------------------------------------
diff --git a/src/slave/constants.hpp b/src/slave/constants.hpp
index fd1c1ab..206d439 100644
--- a/src/slave/constants.hpp
+++ b/src/slave/constants.hpp
@@ -97,6 +97,14 @@ extern const uint16_t DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER;
// Default duration that docker containers will be removed after exit.
extern const Duration DOCKER_REMOVE_DELAY;
+// Default duration to wait before retry inspecting a docker
+// container.
+extern const Duration DOCKER_INSPECT_DELAY;
+
+// Default duration that docker containerizer will wait to check
+// docker version.
+extern const Duration DOCKER_VERSION_WAIT_TIMEOUT;
+
// Name of the default, CRAM-MD5 authenticatee.
extern const std::string DEFAULT_AUTHENTICATEE;
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 952d3eb..bd58f94 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -35,8 +35,6 @@
#include "common/status_utils.hpp"
-#include "docker/docker.hpp"
-
#ifdef __linux__
#include "linux/cgroups.hpp"
#endif // __linux__
@@ -48,7 +46,6 @@
#include "slave/containerizer/docker.hpp"
#include "slave/containerizer/fetcher.hpp"
-
#include "slave/containerizer/isolators/cgroups/constants.hpp"
#include "usage/usage.hpp"
@@ -99,8 +96,8 @@ Option<ContainerID> parse(const Docker::Container& container)
// was DOCKER_NAME_PREFIX + containerId, and starting with 0.23.0
// it is changed to DOCKER_NAME_PREFIX + slaveId +
// DOCKER_NAME_SEPERATOR + containerId.
- // To be backward compatible during upgrade, we still to support
- // the previous format.
+ // To be backward compatible during upgrade, we still have to
+ // support the previous format.
// TODO(tnachen): Remove this check after deprecation cycle.
if (!strings::contains(name.get(), DOCKER_NAME_SEPERATOR)) {
ContainerID id;
@@ -124,16 +121,16 @@ Try<DockerContainerizer*> DockerContainerizer::create(
const Flags& flags,
Fetcher* fetcher)
{
- Try<Docker*> docker = Docker::create(flags.docker);
- if (docker.isError()) {
- return Error(docker.error());
+ Try<Docker*> create = Docker::create(flags.docker);
+ if (create.isError()) {
+ return Error("Failed to create docker: " + create.error());
}
- Shared<Docker> docker_(docker.get());
+ Shared<Docker> docker(create.get());
if (flags.docker_mesos_image.isSome()) {
- Future<Version> version = docker_->version();
- if (!version.await(Seconds(5))) {
+ Future<Version> version = docker->version();
+ if (!version.await(DOCKER_VERSION_WAIT_TIMEOUT)) {
return Error("Timed out waiting for docker version");
}
@@ -148,7 +145,7 @@ Try<DockerContainerizer*> DockerContainerizer::create(
}
}
- return new DockerContainerizer(flags, fetcher, docker_);
+ return new DockerContainerizer(flags, fetcher, docker);
}
@@ -177,17 +174,18 @@ DockerContainerizer::~DockerContainerizer()
}
-string dockerExecutorCommand(
- const Flags& flags,
- const string directory,
- const string container)
+docker::Flags dockerFlags(
+ const Flags& flags,
+ const string& name,
+ const string& directory)
{
- string command = "mesos-docker-executor --docker=" + flags.docker +
- " --container=" + container +
- " --sandbox_directory=" + directory +
- " --mapped_directory=" + flags.docker_sandbox_directory;
-
- return path::join(flags.launcher_dir, command);
+ docker::Flags dockerFlags;
+ dockerFlags.container = name;
+ dockerFlags.docker = flags.docker;
+ dockerFlags.sandbox_directory = directory;
+ dockerFlags.mapped_directory = flags.docker_sandbox_directory;
+ dockerFlags.stop_timeout = flags.docker_stop_timeout;
+ return dockerFlags;
}
@@ -254,34 +252,26 @@ DockerContainerizerProcess::Container::create(
symlinked = true;
}
- Container* container = new Container(
- id,
- taskInfo,
- executorInfo,
- containerWorkdir,
- user,
- slaveId,
- slavePid,
- checkpoint,
- symlinked,
- flags);
-
+ Option<ContainerInfo> containerInfo = None();
+ Option<CommandInfo> commandInfo = None();
+ Option<std::map<string, string>> environment = None();
+ bool launchesExecutorContainer = false;
if (taskInfo.isSome() && flags.docker_mesos_image.isSome()) {
// Override the container and command to launch an executor
// in a docker container.
- ContainerInfo containerInfo;
+ ContainerInfo newContainerInfo;
// Mounting in the docker socket so the executor can communicate to
// the host docker daemon. We are assuming the current instance is
// launching docker containers to the host daemon as well.
- Volume* dockerSockVolume = containerInfo.add_volumes();
+ Volume* dockerSockVolume = newContainerInfo.add_volumes();
dockerSockVolume->set_host_path(flags.docker_socket);
dockerSockVolume->set_container_path(flags.docker_socket);
dockerSockVolume->set_mode(Volume::RO);
// Mounting in sandbox so the logs from the executor can be
// persisted over container failures.
- Volume* sandboxVolume = containerInfo.add_volumes();
+ Volume* sandboxVolume = newContainerInfo.add_volumes();
sandboxVolume->set_host_path(containerWorkdir);
sandboxVolume->set_container_path(containerWorkdir);
sandboxVolume->set_mode(Volume::RW);
@@ -289,27 +279,48 @@ DockerContainerizerProcess::Container::create(
ContainerInfo::DockerInfo dockerInfo;
dockerInfo.set_image(flags.docker_mesos_image.get());
- containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+ newContainerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ docker::Flags dockerExecutorFlags = dockerFlags(
+ flags,
+ Container::name(slaveId, stringify(id)),
+ containerWorkdir);
- const string& command = dockerExecutorCommand(
- flags, containerWorkdir, container->name());
+ CommandInfo newCommandInfo;
+ // TODO(tnachen): Pass flags directly into docker run.
+ newCommandInfo.set_value(
+ path::join(flags.launcher_dir, "mesos-docker-executor") +
+ " " + stringify(dockerExecutorFlags));
- CommandInfo commandInfo;
- commandInfo.set_value(command);
- commandInfo.set_shell(true);
+ newCommandInfo.set_shell(true);
- container->overrideContainer = containerInfo;
- container->overrideCommand = commandInfo;
- container->overrideEnvironment = executorEnvironment(
+ containerInfo = newContainerInfo;
+ commandInfo = newCommandInfo;
+ environment = executorEnvironment(
executorInfo,
containerWorkdir,
slaveId,
slavePid,
checkpoint,
flags.recovery_timeout);
+ launchesExecutorContainer = true;
}
- return container;
+ return new Container(
+ id,
+ taskInfo,
+ executorInfo,
+ containerWorkdir,
+ user,
+ slaveId,
+ slavePid,
+ checkpoint,
+ symlinked,
+ flags,
+ commandInfo,
+ containerInfo,
+ environment,
+ launchesExecutorContainer);
}
@@ -321,7 +332,7 @@ Future<Nothing> DockerContainerizerProcess::fetch(
return fetcher->fetch(
containerId,
- container->command(),
+ container->command,
container->directory,
None(),
flags);
@@ -329,21 +340,28 @@ Future<Nothing> DockerContainerizerProcess::fetch(
Future<Nothing> DockerContainerizerProcess::pull(
- const ContainerID& containerId,
- const string& directory,
- const string& image,
- bool forcePullImage)
+ const ContainerID& containerId)
{
- Future<Docker::Image> future = docker->pull(directory, image, forcePullImage);
- containers_[containerId]->pull = future;
- return future.then(defer(self(), &Self::_pull, image));
-}
+ if (!containers_.contains(containerId)) {
+ return Failure("Container is already destroyed");
+ }
+ Container* container = containers_[containerId];
+ container->state = Container::PULLING;
-Future<Nothing> DockerContainerizerProcess::_pull(const string& image)
-{
- VLOG(1) << "Docker pull " << image << " completed";
- return Nothing();
+ string image = container->image();
+
+ Future<Docker::Image> future = docker->pull(
+ container->directory,
+ image,
+ container->forcePullImage());
+
+ containers_[containerId]->pull = future;
+
+ return future.then(defer(self(), [=]() {
+ VLOG(1) << "Docker pull " << image << " completed";
+ return Nothing();
+ }));
}
@@ -545,10 +563,16 @@ Future<Nothing> DockerContainerizerProcess::_recover(
// checkpointed container id.
// TODO(tnachen): Remove this explicit reconciliation 0.24.
hashset<ContainerID> existingContainers;
+ // Tracks all the task containers that launched an executor in
+ // a docker container.
+ hashset<ContainerID> executorContainers;
foreach (const Docker::Container& container, _containers) {
Option<ContainerID> id = parse(container);
if (id.isSome()) {
existingContainers.insert(id.get());
+ if (strings::contains(container.name, ".executor")) {
+ executorContainers.insert(id.get());
+ }
}
}
@@ -623,6 +647,8 @@ Future<Nothing> DockerContainerizerProcess::_recover(
containers_[containerId] = container;
container->slaveId = state.id;
container->state = Container::RUNNING;
+ container->launchesExecutorContainer =
+ executorContainers.contains(containerId);
pid_t pid = run.get().forkedPid.get();
@@ -747,18 +773,19 @@ Future<bool> DockerContainerizerProcess::launch(
if (taskInfo.isSome() && flags.docker_mesos_image.isNone()) {
// Launching task by forking a subprocess to run docker executor.
return container.get()->launch = fetch(containerId)
- .then(defer(self(), &Self::_launch, containerId))
- .then(defer(self(), &Self::___launch, containerId))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1))
- .onFailed(defer(self(), &Self::destroy, containerId, true));
+ .then(defer(self(), [=]() { return pull(containerId); }))
+ .then(defer(self(), [=]() { return launchExecutorProcess(containerId); }))
+ .then(defer(self(), [=](pid_t pid) {
+ return reapExecutor(containerId, pid);
+ }));
}
string containerName = container.get()->name();
- if (flags.docker_mesos_image.isSome()) {
+ if (container.get()->executorName().isSome()) {
// Launch the container with the executor name as we expect the
// executor will launch the docker container.
- containerName = container.get()->executorName();
+ containerName = container.get()->executorName().get();
}
// Launching task or executor by launching a seperate docker
@@ -768,67 +795,76 @@ Future<bool> DockerContainerizerProcess::launch(
// we want the executor to keep running when the slave container
// dies.
return container.get()->launch = fetch(containerId)
- .then(defer(self(), &Self::_launch, containerId))
- .then(defer(self(), &Self::__launch, containerId, containerName))
- .then(defer(self(), &Self::____launch, containerId, containerName))
- .then(defer(self(), &Self::_____launch, containerId, lambda::_1))
- .then(defer(self(), &Self::______launch, containerId, lambda::_1));
+ .then(defer(self(), [=]() { return pull(containerId); }))
+ .then(defer(self(), [=]() {
+ return launchExecutorContainer(containerId, containerName);
+ }))
+ .then(defer(self(), [=](const Docker::Container& dockerContainer) {
+ return checkpointExecutor(containerId, dockerContainer);
+ }))
+ .then(defer(self(), [=](pid_t pid) {
+ return reapExecutor(containerId, pid);
+ }));
}
-Future<Nothing> DockerContainerizerProcess::_launch(
- const ContainerID& containerId)
+Future<Docker::Container> DockerContainerizerProcess::launchExecutorContainer(
+ const ContainerID& containerId,
+ const string& containerName)
{
- // Doing the fetch might have succeded but we were actually asked to
- // destroy the container, which we did, so don't continue.
if (!containers_.contains(containerId)) {
- return Failure("Container was destroyed while launching");
+ return Failure("Container is already destroyed");
}
Container* container = containers_[containerId];
-
- container->state = Container::PULLING;
-
- return pull(
- containerId,
- container->directory,
- container->image(),
- container->forcePullImage());
+ container->state = Container::RUNNING;
+
+ // Start the executor in a Docker container.
+ // This executor could either be a custom executor specified by an
+ // ExecutorInfo, or the docker executor.
+ Future<Nothing> run = docker->run(
+ container->container,
+ container->command,
+ containerName,
+ container->directory,
+ flags.docker_sandbox_directory,
+ container->resources,
+ container->environment,
+ path::join(container->directory, "stdout"),
+ path::join(container->directory, "stderr"));
+
+ Owned<Promise<Docker::Container>> promise(new Promise<Docker::Container>());
+ // We like to propogate the run failure when run fails so slave can
+ // send this failure back to the scheduler. Otherwise we return
+ // inspect's result or its failure, which should not fail when
+ // the container isn't launched.
+ Future<Docker::Container> inspect =
+ docker->inspect(containerName, slave::DOCKER_INSPECT_DELAY)
+ .onAny([=](Future<Docker::Container> f) {
+ // We cannot associate the promise outside of the callback
+ // because we like to propagate run's failure when
+ // available.
+ promise->associate(f);
+ });
+
+ run.onFailed([=](const string& failure) mutable {
+ inspect.discard();
+ promise->fail(failure);
+ });
+
+ return promise->future();
}
-Future<Nothing> DockerContainerizerProcess::__launch(
- const ContainerID& containerId,
- const string& container)
+Future<pid_t> DockerContainerizerProcess::launchExecutorProcess(
+ const ContainerID& containerId)
{
if (!containers_.contains(containerId)) {
- return Failure("Container was destroyed while pulling image");
+ return Failure("Container is already destroyed");
}
- Container* container_ = containers_[containerId];
-
- container_->state = Container::RUNNING;
-
- // Try and start the Docker container.
- return docker->run(
- container_->container(),
- container_->command(),
- container,
- container_->directory,
- flags.docker_sandbox_directory,
- container_->resources,
- container_->environment());
-}
-
-
-Future<pid_t> DockerContainerizerProcess::___launch(
- const ContainerID& containerId)
-{
- // After we do Docker::run we shouldn't remove a container until
- // after we set Container::status.
- CHECK(containers_.contains(containerId));
-
Container* container = containers_[containerId];
+ container->state = Container::RUNNING;
// Prepare environment variables for the executor.
map<string, string> environment = executorEnvironment(
@@ -850,19 +886,19 @@ Future<pid_t> DockerContainerizerProcess::___launch(
environment["GLOG_v"] = os::getenv("GLOG_v");
}
- const string& command = dockerExecutorCommand(
- flags, container->directory, container->name());
-
- VLOG(1) << "Launching docker executor with command: " << command;
+ vector<string> argv;
+ argv.push_back("mesos-docker-executor");
// Construct the mesos-docker-executor using the "name" we gave the
// container (to distinguish it from Docker containers not created
// by Mesos).
Try<Subprocess> s = subprocess(
- command,
+ path::join(flags.launcher_dir, "mesos-docker-executor"),
+ argv,
Subprocess::PIPE(),
Subprocess::PATH(path::join(container->directory, "stdout")),
Subprocess::PATH(path::join(container->directory, "stderr")),
+ dockerFlags(flags, container->name(), container->directory),
environment,
lambda::bind(&setup, container->directory));
@@ -893,37 +929,23 @@ Future<pid_t> DockerContainerizerProcess::___launch(
if (length != sizeof(c)) {
string error = string(strerror(errno));
os::close(s.get().in().get());
- Failure failure("Failed to synchronize with child process: " + error);
-
- container->run = failure;
- return failure;
+ return Failure("Failed to synchronize with child process: " + error);
}
return s.get().pid();
}
-Future<Docker::Container> DockerContainerizerProcess::____launch(
- const ContainerID& containerId,
- const std::string& container)
-{
- CHECK(containers_.contains(containerId));
-
- return docker->inspect(container);
-}
-
-Future<pid_t> DockerContainerizerProcess::_____launch(
+Future<pid_t> DockerContainerizerProcess::checkpointExecutor(
const ContainerID& containerId,
- const Docker::Container& container)
+ const Docker::Container& dockerContainer)
{
// After we do Docker::run we shouldn't remove a container until
// after we set Container::status.
CHECK(containers_.contains(containerId));
- Container* container_ = containers_[containerId];
-
- Option<int> pid = container.pid;
+ Option<int> pid = dockerContainer.pid;
if (!pid.isSome()) {
return Failure("Unable to get executor pid after launch");
@@ -936,16 +958,11 @@ Future<pid_t> DockerContainerizerProcess::_____launch(
"Failed to checkpoint executor's pid: " + checkpointed.error());
}
- // TODO(tnachen): We need to handle the slave in container scenario.
- // Instead of forking a log process, launch a log container for
- // executor.
- docker->logs(container_->name(), container_->directory);
-
return pid.get();
}
-Future<bool> DockerContainerizerProcess::______launch(
+Future<bool> DockerContainerizerProcess::reapExecutor(
const ContainerID& containerId,
pid_t pid)
{
@@ -1355,6 +1372,25 @@ void DockerContainerizerProcess::destroy(
container->state = Container::DESTROYING;
+ if (killed && container->executorPid.isSome()) {
+ LOG(INFO) << "Sending SIGTERM to executor with pid: "
+ << container->executorPid.get();
+ // We need to clean up the executor as the executor might not have
+ // received run task due to a failed containerizer update.
+ // We also kill the executor first since container->status below
+ // is waiting for the executor to finish.
+ Try<std::list<os::ProcessTree>> kill =
+ os::killtree(container->executorPid.get(), SIGTERM);
+
+ if (kill.isError()) {
+ // Ignoring the error from killing executor as it can already
+ // have exited.
+ VLOG(1) << "Ignoring error when killing executor pid "
+ << container->executorPid.get() << " in destroy, error: "
+ << kill.error();
+ }
+ }
+
// Otherwise, wait for Docker::run to succeed, in which case we'll
// continue in _destroy (calling Docker::kill) or for Docker::run to
// fail, in which case we'll re-execute this function and cleanup
@@ -1374,17 +1410,19 @@ void DockerContainerizerProcess::_destroy(
CHECK(container->state == Container::DESTROYING);
- // Do a 'docker rm -f' which we'll then find out about in '_destroy'
+ // Do a 'docker stop' which we'll then find out about in '_destroy'
// after we've reaped either the container's root process (in the
// event that we had just launched a container for an executor) or
// the mesos-docker-executor (in the case we launched a container
// for a task).
-
LOG(INFO) << "Running docker stop on container '" << containerId << "'";
- docker->stop(container->name(),
- flags.docker_stop_timeout)
- .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1));
+ if (killed) {
+ docker->stop(container->name(), flags.docker_stop_timeout)
+ .onAny(defer(self(), &Self::__destroy, containerId, killed, lambda::_1));
+ } else {
+ __destroy(containerId, killed, Nothing());
+ }
}
@@ -1397,7 +1435,7 @@ void DockerContainerizerProcess::__destroy(
Container* container = containers_[containerId];
- if (!kill.isReady()) {
+ if (!kill.isReady() && !container->status.future().isReady()) {
// TODO(benh): This means we've failed to do a Docker::kill, which
// means it's possible that the container is still going to be
// running after we return! We either need to have a periodic
@@ -1422,7 +1460,7 @@ void DockerContainerizerProcess::__destroy(
}
// Status must be ready since we did a Docker::kill.
- CHECK_READY(containers_[containerId]->status.future());
+ CHECK_READY(container->status.future());
container->status.future().get()
.onAny(defer(self(), &Self::___destroy, containerId, killed, lambda::_1));
@@ -1483,11 +1521,13 @@ void DockerContainerizerProcess::reaped(const ContainerID& containerId)
void DockerContainerizerProcess::remove(
- const string& container,
- const string& executor)
+ const string& containerName,
+ const Option<string>& executor)
{
- docker->rm(container, true);
- docker->rm(executor, true);
+ docker->rm(containerName, true);
+ if (executor.isSome()) {
+ docker->rm(executor.get(), true);
+ }
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index 2961048..d2cca4b 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -21,9 +21,11 @@
#include <process/shared.hpp>
+#include <stout/flags.hpp>
#include <stout/hashset.hpp>
#include "docker/docker.hpp"
+#include "docker/executor.hpp"
#include "slave/containerizer/containerizer.hpp"
@@ -36,7 +38,7 @@ namespace slave {
extern const std::string DOCKER_NAME_PREFIX;
// Seperator used to compose docker container name, which is made up
-// of slave id and container id.
+// of slave ID and container ID.
extern const std::string DOCKER_NAME_SEPERATOR;
// Directory that stores all the symlinked sandboxes that is mapped
@@ -152,11 +154,7 @@ public:
virtual process::Future<Nothing> fetch(const ContainerID& containerId);
- virtual process::Future<Nothing> pull(
- const ContainerID& containerId,
- const std::string& directory,
- const std::string& image,
- bool forcePullImage);
+ virtual process::Future<Nothing> pull(const ContainerID& containerId);
virtual process::Future<hashset<ContainerID>> containers();
@@ -166,8 +164,6 @@ private:
const ContainerID& containerId,
const Option<int>& status);
- process::Future<Nothing> _pull(const std::string& image);
-
Try<Nothing> checkpoint(
const ContainerID& containerId,
pid_t pid);
@@ -179,32 +175,21 @@ private:
process::Future<Nothing> __recover(
const std::list<Docker::Container>& containers);
- process::Future<Nothing> _launch(
- const ContainerID& containerId);
-
- process::Future<Nothing> __launch(
+ // Starts the executor in a Docker container.
+ process::Future<Docker::Container> launchExecutorContainer(
const ContainerID& containerId,
- const std::string& container);
+ const std::string& containerName);
- // NOTE: This continuation is only applicable when launching a
- // container for a task.
- process::Future<pid_t> ___launch(
+ // Starts the docker executor with a subprocess.
+ process::Future<pid_t> launchExecutorProcess(
const ContainerID& containerId);
-
- // NOTE: This continuation is only applicable when launching a
- // container for an executor.
- process::Future<Docker::Container> ____launch(
+ process::Future<pid_t> checkpointExecutor(
const ContainerID& containerId,
- const std::string& container);
+ const Docker::Container& dockerContainer);
- // NOTE: This continuation is only applicable when launching a
- // container for an executor.
- process::Future<pid_t> _____launch(
- const ContainerID& containerId,
- const Docker::Container& container);
-
- process::Future<bool> ______launch(
+ // Reaps on the executor pid.
+ process::Future<bool> reapExecutor(
const ContainerID& containerId,
pid_t pid);
@@ -245,7 +230,9 @@ private:
void reaped(const ContainerID& containerId);
// Removes the docker container.
- void remove(const std::string& container, const std::string& executor);
+ void remove(
+ const std::string& containerName,
+ const Option<std::string>& executor);
const Flags flags;
@@ -266,6 +253,12 @@ private:
bool checkpoint,
const Flags& flags);
+ static std::string name(const SlaveID& slaveId, const std::string& id)
+ {
+ return DOCKER_NAME_PREFIX + slaveId.value() + DOCKER_NAME_SEPERATOR +
+ stringify(id);
+ }
+
Container(const ContainerID& id)
: state(FETCHING), id(id) {}
@@ -278,7 +271,11 @@ private:
const process::PID<Slave>& slavePid,
bool checkpoint,
bool symlinked,
- const Flags& flags)
+ const Flags& flags,
+ const Option<CommandInfo>& _command,
+ const Option<ContainerInfo>& _container,
+ const Option<std::map<std::string, std::string>>& _environment,
+ bool launchesExecutorContainer)
: state(FETCHING),
id(id),
task(taskInfo),
@@ -289,7 +286,8 @@ private:
slavePid(slavePid),
checkpoint(checkpoint),
symlinked(symlinked),
- flags(flags)
+ flags(flags),
+ launchesExecutorContainer(launchesExecutorContainer)
{
// NOTE: The task's resources are included in the executor's
// resources in order to make sure when launching the executor
@@ -307,13 +305,33 @@ private:
CHECK(resources.contains(task.get().resources()));
}
- overrideEnvironment = executorEnvironment(
- executor,
- directory,
- slaveId,
- slavePid,
- checkpoint,
- flags.recovery_timeout);
+ if (_command.isSome()) {
+ command = _command.get();
+ } else if (task.isSome()) {
+ command = task.get().command();
+ } else {
+ command = executor.command();
+ }
+
+ if (_container.isSome()) {
+ container = _container.get();
+ } else if (task.isSome()) {
+ container = task.get().container();
+ } else {
+ container = executor.container();
+ }
+
+ if (_environment.isSome()) {
+ environment = _environment.get();
+ } else {
+ environment = executorEnvironment(
+ executor,
+ directory,
+ slaveId,
+ slavePid,
+ checkpoint,
+ flags.recovery_timeout);
+ }
}
~Container()
@@ -327,13 +345,16 @@ private:
std::string name()
{
- return DOCKER_NAME_PREFIX + slaveId.value() + DOCKER_NAME_SEPERATOR +
- stringify(id);
+ return name(slaveId, stringify(id));
}
- std::string executorName()
+ Option<std::string> executorName()
{
- return name() + DOCKER_NAME_SEPERATOR + "executor";
+ if (launchesExecutorContainer) {
+ return name() + DOCKER_NAME_SEPERATOR + "executor";
+ } else {
+ return None();
+ }
}
std::string image() const
@@ -354,43 +375,6 @@ private:
return executor.container().docker().force_pull_image();
}
- ContainerInfo container() const
- {
- if (overrideContainer.isSome()) {
- return overrideContainer.get();
- }
-
- if (task.isSome()) {
- return task.get().container();
- }
-
- return executor.container();
- }
-
- CommandInfo command() const
- {
- if (overrideCommand.isSome()) {
- return overrideCommand.get();
- }
-
- if (task.isSome()) {
- return task.get().command();
- }
-
- return executor.command();
- }
-
- // Returns any extra environment varaibles to set when launching
- // the Docker container (beyond the those found in CommandInfo).
- std::map<std::string, std::string> environment() const
- {
- if (overrideEnvironment.isSome()) {
- return overrideEnvironment.get();
- }
-
- return std::map<std::string, std::string>();
- }
-
// The DockerContainerier needs to be able to properly clean up
// Docker containers, regardless of when they are destroyed. For
// example, if a container gets destroyed while we are fetching,
@@ -421,23 +405,22 @@ private:
} state;
const ContainerID id;
- Option<TaskInfo> task;
- ExecutorInfo executor;
+ const Option<TaskInfo> task;
+ const ExecutorInfo executor;
+ ContainerInfo container;
+ CommandInfo command;
+ std::map<std::string, std::string> environment;
// The sandbox directory for the container. This holds the
// symlinked path if symlinked boolean is true.
- std::string directory;
+ const std::string directory;
- Option<std::string> user;
+ const Option<std::string> user;
SlaveID slaveId;
- process::PID<Slave> slavePid;
+ const process::PID<Slave> slavePid;
bool checkpoint;
bool symlinked;
- Flags flags;
-
- Option<ContainerInfo> overrideContainer;
- Option<CommandInfo> overrideCommand;
- Option<std::map<std::string, std::string>> overrideEnvironment;
+ const Flags flags;
// Promise for future returned from wait().
Promise<containerizer::Termination> termination;
@@ -469,6 +452,10 @@ private:
// container. This is stored so we can clean up the executor
// on destroy.
Option<pid_t> executorPid;
+
+ // Marks if this container launches a executor in a docker
+ // container.
+ bool launchesExecutorContainer;
};
hashmap<ContainerID, Container*> containers_;
http://git-wip-us.apache.org/repos/asf/mesos/blob/3baa6096/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index acf30d1..a8c7c49 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -294,22 +294,28 @@ mesos::internal::slave::Flags::Flags()
add(&Flags::docker_kill_orphans,
"docker_kill_orphans",
- "Enable docker containerizer to kill orphaned containers",
+ "Enable docker containerizer to kill orphaned containers.\n"
+ "You should consider setting this to false when you launch multiple\n"
+ "slaves in the same OS, to avoid one of the DockerContainerizer \n"
+ "removing docker tasks launched by other slaves. However you should\n"
+ "also make sure to enable checkpoint for the slave so the same slave id\n"
+ "can be reused, otherwise docker tasks on slave restart will not be\n"
+ "cleaned up.\n",
true);
add(&Flags::docker_mesos_image,
"docker_mesos_image",
"The docker image used to launch this mesos slave instance.\n"
"If an image is specified, the docker containerizer assumes the slave\n"
- "is running in a docker container. This enables the docker\n"
- "containerizer to launch executors in docker containers, so they keep\n"
- "running when the slave container exits.");
+ "is running in a docker container, and launches executors with\n"
+ "docker containers in order to recover them when the slave restarts and\n"
+ "recovers.\n");
add(&Flags::docker_socket,
"docker_socket",
- "The docker UNIX socket path that the docker CLI uses to communicate\n"
- "to the docker daemon. Mesos needs this to launch docker containers\n"
- "that can run the docker CLI.\n",
+ "The UNIX socket path to be mounted into the docker executor container\n"
+ "to provide docker CLI access to the docker daemon. This must be the\n"
+ "path used by the slave's docker image.\n",
"/var/run/docker.sock");
add(&Flags::default_container_info,