You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/11/12 02:57:29 UTC
[08/11] mesos git commit: Updated namespace isolators to customize
based on 'ContainerClass'.
Updated namespace isolators to customize based on 'ContainerClass'.
The namespace-related isolators now do different things depending on
whether they are launching a "normal" nested container or a "debug"
nested container. Normal nested containers clone a new mount namespace
as well as a new pid namespace. Debug nested cotnainers do not -- they
simply inherit these namespaces from their parent.
Review: https://reviews.apache.org/r/53354/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0dafc555
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0dafc555
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0dafc555
Branch: refs/heads/master
Commit: 0dafc55529e386e97e6384e1b8e5a19a84899b63
Parents: a012efa
Author: Kevin Klues <kl...@gmail.com>
Authored: Fri Nov 11 17:01:23 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri Nov 11 18:54:11 2016 -0800
----------------------------------------------------------------------
src/slave/containerizer/mesos/containerizer.cpp | 6 -
.../mesos/isolators/filesystem/linux.cpp | 24 ++++
.../mesos/isolators/gpu/isolator.cpp | 21 +++-
.../mesos/isolators/namespaces/pid.cpp | 9 ++
.../mesos/isolators/network/cni/cni.cpp | 56 ++++++---
.../mesos/isolators/volume/image.cpp | 6 +
.../mesos/isolators/volume/sandbox_path.cpp | 7 ++
.../containerizer/mesos/linux_launcher.cpp | 6 +
.../nested_mesos_containerizer_tests.cpp | 117 +++++++++++++++++++
9 files changed, 225 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp
index 3c3bee4..2741497 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -1230,12 +1230,6 @@ Future<bool> MesosContainerizerProcess::_launch(
Option<int> enterNamespaces = None();
Option<int> cloneNamespaces = None();
- // TODO(klueska): Remove the following once we update
- // isolators to fill enterNamespaces for us.
- if (containerId.has_parent()) {
- enterNamespaces = CLONE_NEWUTS | CLONE_NEWNET | CLONE_NEWPID;
- }
-
CHECK_READY(container->launchInfos);
foreach (const Option<ContainerLaunchInfo>& launchInfo,
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp b/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
index 6535375..8f024d0 100644
--- a/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
+++ b/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
@@ -53,6 +53,7 @@ using std::ostringstream;
using std::string;
using std::vector;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerState;
using mesos::slave::ContainerLaunchInfo;
@@ -307,6 +308,29 @@ Future<Option<ContainerLaunchInfo>> LinuxFilesystemIsolatorProcess::prepare(
const ContainerID& containerId,
const ContainerConfig& containerConfig)
{
+ // If we are a nested container in the `DEBUG` class, then we only
+ // use this isolator to indicate that we should enter our parent's
+ // MOUNT namespace. We don't want to clone a new MOUNT namespace or
+ // run any new pre-exec commands in it. For now, we also don't
+ // support provisioning a new filesystem or setting a `rootfs` for
+ // the container. We also don't support mounting any volumes.
+ if (containerId.has_parent() &&
+ containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ if (containerConfig.has_rootfs()) {
+ return Failure("A 'rootfs' cannot be set for DEBUG containers");
+ }
+
+ if (containerConfig.has_container_info() &&
+ containerConfig.container_info().volumes().size() > 0) {
+ return Failure("Volumes not supported for DEBUG containers");
+ }
+
+ ContainerLaunchInfo launchInfo;
+ launchInfo.set_enter_namespaces(CLONE_NEWNS);
+ return launchInfo;
+ }
+
if (infos.contains(containerId)) {
return Failure("Container has already been prepared");
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/gpu/isolator.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/gpu/isolator.cpp b/src/slave/containerizer/mesos/isolators/gpu/isolator.cpp
index a3a140d..a3b9fe7 100644
--- a/src/slave/containerizer/mesos/isolators/gpu/isolator.cpp
+++ b/src/slave/containerizer/mesos/isolators/gpu/isolator.cpp
@@ -53,6 +53,7 @@ using cgroups::devices::Entry;
using docker::spec::v1::ImageManifest;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerLaunchInfo;
using mesos::slave::ContainerLimitation;
@@ -283,13 +284,21 @@ Future<Option<ContainerLaunchInfo>> NvidiaGpuIsolatorProcess::prepare(
const ContainerID& containerId,
const mesos::slave::ContainerConfig& containerConfig)
{
- // If we are a nested container, we don't need to maintain an `Info()`
- // struct about the container since we don't allocate GPUs to it
- // directly (we only allocate GPUs to top-level containers, and they
- // automatically get shared by nested containers). However, we do
- // still need to mount the necessary Nvidia libraries into the
- // container. We call `_prepare()` directly to do this for us.
if (containerId.has_parent()) {
+ // If we are a nested container in the `DEBUG` class, then we
+ // don't need to do anything special to prepare ourselves for GPU
+ // support. All Nvidia volumes will be inherited from our parent.
+ if (containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ return None();
+ }
+
+ // If we are a nested container in a different class, we don't
+ // need to maintain an `Info()` struct about the container (since
+ // we don't directly allocate any GPUs to it), but we do need to
+ // mount the necessary Nvidia libraries into the container (since
+ // we live in a different mount namespace than our parent). We
+ // directly call `_prepare()` to do this for us.
return _prepare(containerConfig);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/namespaces/pid.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/namespaces/pid.cpp b/src/slave/containerizer/mesos/isolators/namespaces/pid.cpp
index 8ebf2e7..5bf5ffb 100644
--- a/src/slave/containerizer/mesos/isolators/namespaces/pid.cpp
+++ b/src/slave/containerizer/mesos/isolators/namespaces/pid.cpp
@@ -25,6 +25,7 @@
using process::Future;
using process::Owned;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerLaunchInfo;
using mesos::slave::Isolator;
@@ -84,6 +85,14 @@ Future<Option<ContainerLaunchInfo>> NamespacesPidIsolatorProcess::prepare(
// If we are a nested container, then we want to enter our
// parent's pid namespace before cloning a new one.
launchInfo.set_enter_namespaces(CLONE_NEWPID);
+
+ // However, if we are a nested container in the `DEBUG` class,
+ // then we don't want to clone a new PID namespace at all, so we
+ // short cirucuit here.
+ if (containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ return launchInfo;
+ }
}
launchInfo.set_clone_namespaces(CLONE_NEWPID);
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index 74bb6eb..3b2d4db 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -54,6 +54,7 @@ using process::Owned;
using process::PID;
using process::Subprocess;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerLaunchInfo;
using mesos::slave::ContainerLimitation;
@@ -590,6 +591,8 @@ Future<Option<ContainerLaunchInfo>> NetworkCniIsolatorProcess::prepare(
// * Cases where the containers don't need a new mount namespace:
// a) Containers (nested or stand alone) join the host network
// without an image.
+ // b) Nested DEBUG containers join the same mount namespace as
+ // their parent.
// * Cases where the container needs a new mount namespace:
// a) Containers (nested or stand alone) join the host network
// with an image.
@@ -610,19 +613,40 @@ Future<Option<ContainerLaunchInfo>> NetworkCniIsolatorProcess::prepare(
infos.put(containerId, info);
}
- // NOTE: No additional namespace needed. The container shares the
- // same network and UTS namesapces with the host. If container has
- // a rootfs, the filesystem/linux isolator will put the container
- // in a new mount namespace.
+ // NOTE: No additional namespaces needed. The container shares the
+ // same network and UTS namespaces with the host. If the container
+ // has a rootfs, the filesystem/linux isolator will put the
+ // container in a new mount namespace. If the *parent* container
+ // has a rootfs, the filesystem/linux isolator will properly set
+ // the MNT namespace to enter. If the parent does not have a
+ // rootfs, it will join the host network and there are no
+ // namespaces it needs to enter.
return None();
} else {
// This is the case where the container is joining a non-host
- // network namespace.
- if (containerConfig.has_rootfs()) {
- Owned<Info> info(new Info(containerNetworks, containerConfig.rootfs()));
- infos.put(containerId, info);
+ // network namespace. Non-DEBUG containers will need a new mount
+ // namespaces to bind mount their network files (/etc/hosts,
+ // /etc/hostname, /etc/resolv.conf) which will be different than
+ // those on the host file system.
+ //
+ // Unlike other isolators, we can't simply rely on the
+ // `filesystem/linux` isolator to give this container a new
+ // mount namespace (because we allow the `filesystem/posix`
+ // isolator to be used here). We must set the clone flags
+ // ourselves explicitly.
+
+ if (containerId.has_parent() &&
+ containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ // Nested DEBUG containers never need a new MOUNT namespace, so
+ // we don't mantain information about them in the `infos` map.
} else {
- infos.put(containerId, Owned<Info>(new Info(containerNetworks)));
+ if (containerConfig.has_rootfs()) {
+ Owned<Info> info(new Info(containerNetworks, containerConfig.rootfs()));
+ infos.put(containerId, info);
+ } else {
+ infos.put(containerId, Owned<Info>(new Info(containerNetworks)));
+ }
}
ContainerLaunchInfo launchInfo;
@@ -659,12 +683,14 @@ Future<Option<ContainerLaunchInfo>> NetworkCniIsolatorProcess::prepare(
}
} else {
// This is a nested container. This shares the parent's network
- // and UTS namespace. We still need to give this container a new
- // mount namespace since its joining a non-host network and
- // therefore we will need to bind mount its network files
- // (/etc/hosts, /etc/hostname, /etc/resolv.conf) which will be
- // different than those on the host file system.
- launchInfo.set_clone_namespaces(CLONE_NEWNS);
+ // and UTS namespace. For non-DEBUG containers it also needs a
+ // new mount namespace.
+ launchInfo.set_enter_namespaces(CLONE_NEWNET | CLONE_NEWUTS);
+
+ if (!containerConfig.has_container_class() ||
+ containerConfig.container_class() != ContainerClass::DEBUG) {
+ launchInfo.set_clone_namespaces(CLONE_NEWNS);
+ }
}
return launchInfo;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/volume/image.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/volume/image.cpp b/src/slave/containerizer/mesos/isolators/volume/image.cpp
index e954182..c89af3e 100644
--- a/src/slave/containerizer/mesos/isolators/volume/image.cpp
+++ b/src/slave/containerizer/mesos/isolators/volume/image.cpp
@@ -43,6 +43,7 @@ using process::Owned;
using process::PID;
using process::Shared;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerLaunchInfo;
using mesos::slave::Isolator;
@@ -110,6 +111,11 @@ Future<Option<ContainerLaunchInfo>> VolumeImageIsolatorProcess::prepare(
continue;
}
+ if (containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ return Failure("Image volume is not supported for DEBUG containers");
+ }
+
// Determine the target of the mount. The mount target
// is determined by 'container_path'.
string target;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/isolators/volume/sandbox_path.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/volume/sandbox_path.cpp b/src/slave/containerizer/mesos/isolators/volume/sandbox_path.cpp
index cb7aa4f..6f7304d 100644
--- a/src/slave/containerizer/mesos/isolators/volume/sandbox_path.cpp
+++ b/src/slave/containerizer/mesos/isolators/volume/sandbox_path.cpp
@@ -36,6 +36,7 @@ using process::Failure;
using process::Future;
using process::Owned;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerConfig;
using mesos::slave::ContainerLaunchInfo;
using mesos::slave::ContainerState;
@@ -127,6 +128,12 @@ Future<Option<ContainerLaunchInfo>> VolumeSandboxPathIsolatorProcess::prepare(
continue;
}
+ if (containerConfig.has_container_class() &&
+ containerConfig.container_class() == ContainerClass::DEBUG) {
+ return Failure(
+ "SANDBOX_PATH volume is not supported for DEBUG containers");
+ }
+
if (!volume.source().has_sandbox_path()) {
return Failure("volume.source.sandbox_path is not specified");
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/slave/containerizer/mesos/linux_launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/linux_launcher.cpp b/src/slave/containerizer/mesos/linux_launcher.cpp
index 2c55d7b..a3409dc 100644
--- a/src/slave/containerizer/mesos/linux_launcher.cpp
+++ b/src/slave/containerizer/mesos/linux_launcher.cpp
@@ -416,6 +416,12 @@ Try<pid_t> LinuxLauncherProcess::fork(
target = container->pid.get();
}
+ // Ensure we didn't pass `enterNamespaces`
+ // if we aren't forking a nested contiainer.
+ if (!containerId.has_parent() && enterNamespaces.isSome()) {
+ return Error("Cannot enter parent namespaces for non-nested container");
+ }
+
int enterFlags = enterNamespaces.isSome() ? enterNamespaces.get() : 0;
int cloneFlags = cloneNamespaces.isSome() ? cloneNamespaces.get() : 0;
http://git-wip-us.apache.org/repos/asf/mesos/blob/0dafc555/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
index e6c690c..20bc9d5 100644
--- a/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
+++ b/src/tests/containerizer/nested_mesos_containerizer_tests.cpp
@@ -58,6 +58,7 @@ using mesos::internal::slave::state::FrameworkState;
using mesos::internal::slave::state::RunState;
using mesos::internal::slave::state::SlaveState;
+using mesos::slave::ContainerClass;
using mesos::slave::ContainerState;
using mesos::slave::ContainerTermination;
@@ -232,6 +233,122 @@ TEST_F(NestedMesosContainerizerTest, ROOT_CGROUPS_LaunchNested)
}
+TEST_F(NestedMesosContainerizerTest,
+ ROOT_CGROUPS_LaunchNestedDebugCheckPidNamespace)
+{
+ slave::Flags flags = CreateSlaveFlags();
+ flags.launcher = "linux";
+ flags.isolation = "cgroups/cpu,filesystem/linux,namespaces/pid";
+
+ Fetcher fetcher;
+
+ Try<MesosContainerizer*> create = MesosContainerizer::create(
+ flags,
+ true,
+ &fetcher);
+
+ ASSERT_SOME(create);
+
+ Owned<MesosContainerizer> containerizer(create.get());
+
+ SlaveState state;
+ state.id = SlaveID();
+
+ AWAIT_READY(containerizer->recover(state));
+
+ ContainerID containerId;
+ containerId.set_value(UUID::random().toString());
+
+ Try<string> directory = environment->mkdtemp();
+ ASSERT_SOME(directory);
+
+ Future<bool> launch = containerizer->launch(
+ containerId,
+ None(),
+ createExecutorInfo("executor", "sleep 1000", "cpus:1"),
+ directory.get(),
+ None(),
+ state.id,
+ map<string, string>(),
+ true); // TODO(benh): Ever want to test not checkpointing?
+
+ AWAIT_ASSERT_TRUE(launch);
+
+ // Now launch nested container.
+ ContainerID nestedContainerId;
+ nestedContainerId.mutable_parent()->CopyFrom(containerId);
+ nestedContainerId.set_value(UUID::random().toString());
+
+ // Launch the nested container with the `ps | wc -l` command and
+ // launch the container without a `ContainerClass`. With this
+ // default setting, we should request to clone a new PID namespace.
+ //
+ // We expect to see exactly 6 lines of output from `ps`.
+ //
+ // 1) The 'ps' header
+ // 2) The init process of the container (i.e. `mesos-containerizer`).
+ // 3) The executor of the container (i.e. `mesos-executor`).
+ // 4) `sh`
+ // 5) `wc -l`
+ // 6) `ps`
+ launch = containerizer->launch(
+ nestedContainerId,
+ createCommandInfo(
+ "PS_LINES=`ps | wc -l`;"
+ "if [ ${PS_LINES} -ne 6 ]; then"
+ " exit ${PS_LINES};"
+ "fi;"),
+ None(),
+ None(),
+ state.id);
+
+ AWAIT_ASSERT_TRUE(launch);
+
+ Future<Option<ContainerTermination>> wait = containerizer->wait(
+ nestedContainerId);
+
+ AWAIT_READY(wait);
+ ASSERT_SOME(wait.get());
+ ASSERT_TRUE(wait.get()->has_status());
+ EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());
+
+ // Relaunch the nested container with the `ps | wc -l` command and
+ // set the container class as `DEBUG`. In this class, we don't
+ // clone a new PID namespace.
+ //
+ // We expect to see much more than 6 lines of output from `ps`.
+ launch = containerizer->launch(
+ nestedContainerId,
+ createCommandInfo(
+ "PS_LINES=`ps | wc -l`;"
+ "if [ ${PS_LINES} -le 6 ]; then"
+ " exit ${PS_LINES};"
+ "fi;"),
+ None(),
+ None(),
+ state.id,
+ ContainerClass::DEBUG);
+
+ AWAIT_ASSERT_TRUE(launch);
+
+ wait = containerizer->wait(nestedContainerId);
+
+ AWAIT_READY(wait);
+ ASSERT_SOME(wait.get());
+ ASSERT_TRUE(wait.get()->has_status());
+ EXPECT_WEXITSTATUS_EQ(0, wait.get()->status());
+
+ wait = containerizer->wait(containerId);
+
+ containerizer->destroy(containerId);
+
+ AWAIT_READY(wait);
+ ASSERT_SOME(wait.get());
+ ASSERT_TRUE(wait.get()->has_status());
+ EXPECT_WTERMSIG_EQ(SIGKILL, wait.get()->status());
+}
+
+
TEST_F(NestedMesosContainerizerTest, ROOT_CGROUPS_DestroyNested)
{
slave::Flags flags = CreateSlaveFlags();