You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2014/09/17 03:30:31 UTC
git commit: Enabled bridge network for Docker Containerizer.
Repository: mesos
Updated Branches:
refs/heads/master 9b5895606 -> 1453a4775
Enabled bridge network for Docker Containerizer.
Review: https://reviews.apache.org/r/25270
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1453a477
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1453a477
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1453a477
Branch: refs/heads/master
Commit: 1453a477511c8f6f22ff16e3dd13d0532e019c5b
Parents: 9b58956
Author: Timothy Chen <tn...@apache.org>
Authored: Tue Sep 16 18:29:36 2014 -0700
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Sep 16 18:29:37 2014 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 17 ++
src/docker/docker.cpp | 61 ++++++-
src/slave/slave.cpp | 3 +-
src/tests/docker_containerizer_tests.cpp | 246 +++++++++++++++++++++++++-
src/tests/docker_tests.cpp | 65 ++++++-
5 files changed, 380 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index dea51f9..be45494 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -855,6 +855,23 @@ message ContainerInfo {
message DockerInfo {
// The docker image that is going to be passed to the registry.
required string image = 1;
+
+ // Network options.
+ enum Network {
+ HOST = 1;
+ BRIDGE = 2;
+ }
+
+ optional Network network = 2 [default = HOST];
+
+ message PortMapping {
+ required uint32 host_port = 1;
+ required uint32 container_port = 2;
+ // Protocol to expose as (ie: tcp, udp).
+ optional string protocol = 3;
+ }
+
+ repeated PortMapping port_mappings = 3;
}
required Type type = 1;
http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index af51ac9..a8becb8 100644
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -253,7 +253,7 @@ Future<Nothing> Docker::run(
argv.push_back("-d");
if (resources.isSome()) {
- // TODO(yifan): Support other resources (e.g. disk, ports).
+ // TODO(yifan): Support other resources (e.g. disk).
Option<double> cpus = resources.get().cpus();
if (cpus.isSome()) {
uint64_t cpuShare =
@@ -311,14 +311,59 @@ Future<Nothing> Docker::run(
const string& image = dockerInfo.image();
- // TODO(tnachen): Support more network options other than host
- // networking that docker provides (ie: BRIDGE). We currently
- // require host networking since if the docker container is
- // expected to be an executor it needs to be able to communicate
- // with the slave by the slave's PID. There can be more future work
- // to allow a bridge to connect but this is not yet implemented.
argv.push_back("--net");
- argv.push_back("host");
+ string network;
+ switch (dockerInfo.network()) {
+ case ContainerInfo::DockerInfo::HOST: network = "host"; break;
+ case ContainerInfo::DockerInfo::BRIDGE: network = "bridge"; break;
+ default: return Failure("Unsupported Network mode: " +
+ stringify(dockerInfo.network()));
+ }
+
+ argv.push_back(network);
+
+ if (dockerInfo.port_mappings().size() > 0) {
+ if (network != "bridge") {
+ return Failure("Port mappings are only supported for bridge network");
+ }
+
+ if (!resources.isSome()) {
+ return Failure("Port mappings require resources");
+ }
+
+ Option<Value::Ranges> portRanges = resources.get().ports();
+
+ if (!portRanges.isSome()) {
+ return Failure("Port mappings require port resources");
+ }
+
+ foreach (const ContainerInfo::DockerInfo::PortMapping& mapping,
+ dockerInfo.port_mappings()) {
+ bool found = false;
+ foreach (const Value::Range& range, portRanges.get().range()) {
+ if (mapping.host_port() >= range.begin() &&
+ mapping.host_port() <= range.end()) {
+ found = true;
+ break;
+ }
+ }
+
+ if (!found) {
+ return Failure("Port [" + stringify(mapping.host_port()) + "] not " +
+ "included in resources");
+ }
+
+ string portMapping = stringify(mapping.host_port()) + ":" +
+ stringify(mapping.container_port());
+
+ if (mapping.has_protocol()) {
+ portMapping += "/" + strings::lower(mapping.protocol());
+ }
+
+ argv.push_back("-p");
+ argv.push_back(portMapping);
+ }
+ }
argv.push_back("--name");
argv.push_back(name);
http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 1b3dc73..28eb028 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -1756,7 +1756,8 @@ void Slave::registerExecutor(
const ExecutorID& executorId)
{
LOG(INFO) << "Got registration for executor '" << executorId
- << "' of framework " << frameworkId;
+ << "' of framework " << frameworkId << " from "
+ << stringify(from);
CHECK(state == RECOVERING || state == DISCONNECTED ||
state == RUNNING || state == TERMINATING)
http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/tests/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_containerizer_tests.cpp b/src/tests/docker_containerizer_tests.cpp
index 8654f9c..1981f49 100644
--- a/src/tests/docker_containerizer_tests.cpp
+++ b/src/tests/docker_containerizer_tests.cpp
@@ -240,7 +240,122 @@ TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor)
containerInfo.set_type(ContainerInfo::DOCKER);
ContainerInfo::DockerInfo dockerInfo;
- dockerInfo.set_image("mesosphere/test-executor");
+ dockerInfo.set_image("tnachen/test-executor");
+
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+ executorInfo.mutable_container()->CopyFrom(containerInfo);
+
+ task.mutable_executor()->CopyFrom(executorInfo);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ Future<ContainerID> containerId;
+ EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _))
+ .WillOnce(DoAll(FutureArg<0>(&containerId),
+ Invoke(&dockerContainerizer,
+ &MockDockerContainerizer::_launchExecutor)));
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished));
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY_FOR(containerId, Seconds(60));
+ AWAIT_READY_FOR(statusRunning, Seconds(60));
+ EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+ AWAIT_READY_FOR(statusFinished, Seconds(60));
+ EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
+
+ Future<list<Docker::Container> > containers =
+ docker.ps(true, slave::DOCKER_NAME_PREFIX);
+
+ AWAIT_READY(containers);
+
+ ASSERT_TRUE(exists(containers.get(), containerId.get()));
+
+ Future<containerizer::Termination> termination =
+ dockerContainerizer.wait(containerId.get());
+
+ driver.stop();
+ driver.join();
+
+ AWAIT_READY(termination);
+
+ containers = docker.ps(true, slave::DOCKER_NAME_PREFIX);
+ AWAIT_READY(containers);
+
+ ASSERT_FALSE(exists(containers.get(), containerId.get()));
+
+ Shutdown();
+}
+
+
+// This test verifies that a custom executor can be launched and
+// registered with the slave with docker bridge network enabled.
+// We're assuming that the custom executor is registering it's public
+// ip instead of 0.0.0.0 or equivelent to the slave as that's the
+// default behavior for libprocess.
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_Launch_Executor_Bridged)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+
+ Docker docker = Docker::create(tests::flags.docker, false).get();
+
+ MockDockerContainerizer dockerContainerizer(flags, docker);
+
+ Try<PID<Slave> > slave = StartSlave(&dockerContainerizer, flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ const Offer& offer = offers.get()[0];
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->CopyFrom(offer.slave_id());
+ task.mutable_resources()->CopyFrom(offer.resources());
+
+ ExecutorInfo executorInfo;
+ ExecutorID executorId;
+ executorId.set_value("e1");
+ executorInfo.mutable_executor_id()->CopyFrom(executorId);
+
+ CommandInfo command;
+ command.set_value("test-executor");
+ executorInfo.mutable_command()->CopyFrom(command);
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("tnachen/test-executor");
+ dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
containerInfo.mutable_docker()->CopyFrom(dockerInfo);
executorInfo.mutable_container()->CopyFrom(containerInfo);
@@ -1630,3 +1745,132 @@ TEST_F(DockerContainerizerTest,
delete dockerContainerizer2;
}
+
+
+// This test verifies that port mapping with bridge network is
+// exposing the host port to the container port, by sending data
+// to the host port and receiving it in the container by listening
+// to the mapped container port.
+TEST_F(DockerContainerizerTest, ROOT_DOCKER_PortMapping)
+{
+ Try<PID<Master> > master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+
+ flags.resources = "cpus:1;mem:1024;ports:[10000-10000]";
+
+ Docker docker = Docker::create(tests::flags.docker, false).get();
+
+ MockDockerContainerizer dockerContainerizer(flags, docker);
+
+ Try<PID<Slave> > slave = StartSlave(&dockerContainerizer, flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get(), DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer> > offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+
+ AWAIT_READY(offers);
+ EXPECT_NE(0u, offers.get().size());
+
+ const Offer& offer = offers.get()[0];
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->CopyFrom(offer.slave_id());
+ task.mutable_resources()->CopyFrom(offer.resources());
+
+ CommandInfo command;
+ command.set_shell(false);
+ command.set_value("nc");
+ command.add_arguments("-l");
+ command.add_arguments("-p");
+ command.add_arguments("1000");
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("busybox");
+ dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
+
+ ContainerInfo::DockerInfo::PortMapping portMapping;
+ portMapping.set_host_port(10000);
+ portMapping.set_container_port(1000);
+
+ dockerInfo.add_port_mappings()->CopyFrom(portMapping);
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ task.mutable_command()->CopyFrom(command);
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ vector<TaskInfo> tasks;
+ tasks.push_back(task);
+
+ Future<ContainerID> containerId;
+ Future<string> directory;
+ EXPECT_CALL(dockerContainerizer, launch(_, _, _, _, _, _, _, _))
+ .WillOnce(DoAll(FutureArg<0>(&containerId),
+ FutureArg<3>(&directory),
+ Invoke(&dockerContainerizer,
+ &MockDockerContainerizer::_launch)));
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished))
+ .WillRepeatedly(DoDefault());
+
+ driver.launchTasks(offers.get()[0].id(), tasks);
+
+ AWAIT_READY_FOR(containerId, Seconds(60));
+ AWAIT_READY(directory);
+ AWAIT_READY_FOR(statusRunning, Seconds(60));
+ EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+
+ string uuid = UUID::random().toString();
+
+ // Write uuid to docker mapped host port.
+ Try<process::Subprocess> s = process::subprocess(
+ "echo " + uuid + " | nc localhost 10000");
+
+ ASSERT_SOME(s);
+ AWAIT_READY(s.get().status());
+
+ string container = slave::DOCKER_NAME_PREFIX + stringify(containerId.get());
+
+ AWAIT_READY_FOR(docker.kill(container, false), Seconds(30));
+
+ AWAIT_READY_FOR(statusFinished, Seconds(60));
+ EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
+
+ // Now check that the proper output is in stdout.
+ Try<string> read = os::read(path::join(directory.get(), "stdout"));
+
+ ASSERT_SOME(read);
+
+ // We expect the uuid that is sent to host port to be written
+ // to stdout by the docker container running nc -l.
+ EXPECT_TRUE(strings::contains(read.get(), uuid));
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/1453a477/src/tests/docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/docker_tests.cpp b/src/tests/docker_tests.cpp
index 826a8c1..e6c228a 100644
--- a/src/tests/docker_tests.cpp
+++ b/src/tests/docker_tests.cpp
@@ -214,8 +214,69 @@ TEST(DockerTest, ROOT_DOCKER_CheckCommandWithShell)
commandInfo,
"testContainer",
"dir",
- "/mnt/mesos/sandbox",
- None());
+ "/mnt/mesos/sandbox");
ASSERT_TRUE(run.isFailed());
}
+
+
+TEST(DockerTest, ROOT_DOCKER_CheckPortResource)
+{
+ string containerName = "mesos-docker-port-resource-test";
+ Docker docker = Docker::create(tests::flags.docker, false).get();
+
+ // Make sure the container is removed.
+ Future<Nothing> remove = docker.rm(containerName, true);
+
+ ASSERT_TRUE(process::internal::await(remove, Seconds(10)));
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("busybox");
+ dockerInfo.set_network(ContainerInfo::DockerInfo::BRIDGE);
+
+ ContainerInfo::DockerInfo::PortMapping portMapping;
+ portMapping.set_host_port(10000);
+ portMapping.set_container_port(80);
+
+ dockerInfo.add_port_mappings()->CopyFrom(portMapping);
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ CommandInfo commandInfo;
+ commandInfo.set_shell(false);
+ commandInfo.set_value("true");
+
+ Resources resources =
+ Resources::parse("ports:[9998-9999];ports:[10001-11000]").get();
+
+ Future<Nothing> run = docker.run(
+ containerInfo,
+ commandInfo,
+ containerName,
+ "dir",
+ "/mnt/mesos/sandbox",
+ resources);
+
+ // Port should be out side of the provided ranges.
+ AWAIT_EXPECTED_FAILED(run);
+
+ resources = Resources::parse("ports:[9998-9999];ports:[10000-11000]").get();
+
+ Try<string> directory = environment->mkdtemp();
+ CHECK_SOME(directory) << "Failed to create temporary directory";
+
+ run = docker.run(
+ containerInfo,
+ commandInfo,
+ containerName,
+ directory.get(),
+ "/mnt/mesos/sandbox",
+ resources);
+
+ AWAIT_READY(run);
+
+ Future<Nothing> status = docker.rm(containerName, true);
+ AWAIT_READY(status);
+}