You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/08/02 04:48:39 UTC
[1/5] mesos git commit: Updated NvidiaVolume to mount as 'tmpfs' if
parent fs is 'noexec'.
Repository: mesos
Updated Branches:
refs/heads/1.0.x 094c8976c -> 3ab813b5b
Updated NvidiaVolume to mount as 'tmpfs' if parent fs is 'noexec'.
This patch is in response to an issue we ran into on Ubuntu 14.04,
where '/run' is being mounted as 'noexec' (MESOS-5923). Since our
NvidiaVolume is created below this mount point, we are unable to
execute any binaries we add to this volume. This causes problems, for
example, when trying to execute 'nvidia-smi' from within a container
that has this volume mounted in.
To work around this issue, we detect if any mount point above the path
where we create the volume is marked as 'noexec', and if so, we create
a new 'tmpfs' mount for the volume without 'noexec' set.
Review: https://reviews.apache.org/r/50592/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/dd171d82
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/dd171d82
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/dd171d82
Branch: refs/heads/1.0.x
Commit: dd171d829d52041cfae346d7d3f647b29efcdfec
Parents: 7d66e4c
Author: Kevin Klues <kl...@gmail.com>
Authored: Mon Aug 1 09:06:07 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Aug 1 21:47:21 2016 -0700
----------------------------------------------------------------------
.../mesos/isolators/gpu/volume.cpp | 45 +++++++++++++++++++-
1 file changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/dd171d82/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/gpu/volume.cpp b/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
index 4d87c11..478e106 100644
--- a/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
+++ b/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
@@ -23,6 +23,7 @@
#include <process/owned.hpp>
+#include <stout/adaptor.hpp>
#include <stout/elf.hpp>
#include <stout/error.hpp>
#include <stout/foreach.hpp>
@@ -38,6 +39,7 @@
#include <stout/os/rmdir.hpp>
#include <stout/os/shell.hpp>
+#include "linux/fs.hpp"
#include "linux/ldcache.hpp"
#include "slave/containerizer/mesos/isolators/gpu/nvml.hpp"
@@ -222,6 +224,45 @@ Try<NvidiaVolume> NvidiaVolume::create()
}
}
+ // If the filesystem where we are creating this volume has the
+ // `noexec` bit set, we will not be able to execute any of the
+ // nvidia binaries we place in the volume (e.g. `nvidia-smi`). To
+ // fix this, we mount a `tmpfs` over the volume `hostPath` without
+ // the `noexec` bit set. See MESOS-5923 for more information.
+ Try<fs::MountInfoTable> table = fs::MountInfoTable::read();
+ if (table.isError()) {
+ return Error("Failed to get mount table: " + table.error());
+ }
+
+ Result<string> realpath = os::realpath(hostPath);
+ if (!realpath.isSome()) {
+ return Error("Failed to os::realpath '" + hostPath + "':"
+ " " + (realpath.isError()
+ ? realpath.error()
+ : "No such file or directory"));
+ }
+
+ // Do a reverse search through the list of mounted filesystems to
+ // find the filesystem that is mounted with the longest overlapping
+ // path to our `hostPath` (which may include the `hostPath` itself).
+ // Only mount a new `tmpfs` over the `hostPath` if the filesysem we
+ // find is marked as `noexec`.
+ foreach (const fs::MountInfoTable::Entry& entry,
+ adaptor::reverse(table->entries)) {
+ if (strings::startsWith(realpath.get(), entry.target)) {
+ if (strings::contains(entry.vfsOptions, "noexec")) {
+ Try<Nothing> mnt = fs::mount(
+ "tmpfs", hostPath, "tmpfs", MS_NOSUID | MS_NODEV, "mode=755");
+
+ if (mnt.isError()) {
+ return Error("Failed to mount '" + hostPath + "': " + mnt.error());
+ }
+ }
+
+ break;
+ }
+ }
+
// Create some directories in the volume if they don't yet exist.
string directories[] = {"bin", "lib", "lib64" };
foreach (const string& directory, directories) {
@@ -345,7 +386,7 @@ Try<NvidiaVolume> NvidiaVolume::create()
if (!os::exists(symlinkPath)) {
Try<Nothing> symlink =
- fs::symlink(Path(realpath.get()).basename(), symlinkPath);
+ ::fs::symlink(Path(realpath.get()).basename(), symlinkPath);
if (symlink.isError()) {
return Error("Failed to fs::symlink"
" '" + symlinkPath + "'"
@@ -370,7 +411,7 @@ Try<NvidiaVolume> NvidiaVolume::create()
if (!os::exists(symlinkPath)) {
Try<Nothing> symlink =
- fs::symlink(Path(realpath.get()).basename(), symlinkPath);
+ ::fs::symlink(Path(realpath.get()).basename(), symlinkPath);
if (symlink.isError()) {
return Error("Failed to fs::symlink"
" '" + symlinkPath + "'"
[3/5] mesos git commit: Updated containerizer.cpp to look for
duplicates in '--containerizer'.
Posted by ji...@apache.org.
Updated containerizer.cpp to look for duplicates in '--containerizer'.
Review: https://reviews.apache.org/r/50671/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/62c7c701
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/62c7c701
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/62c7c701
Branch: refs/heads/1.0.x
Commit: 62c7c70100723b4b7a6eeab564bdbfe34a14290b
Parents: dd171d8
Author: Kevin Klues <kl...@gmail.com>
Authored: Mon Aug 1 15:52:40 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Aug 1 21:47:45 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/containerizer.cpp | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/62c7c701/src/slave/containerizer/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/containerizer.cpp b/src/slave/containerizer/containerizer.cpp
index d66356d..f10a08a 100644
--- a/src/slave/containerizer/containerizer.cpp
+++ b/src/slave/containerizer/containerizer.cpp
@@ -211,6 +211,15 @@ Try<Containerizer*> Containerizer::create(
return containerizer.get();
}
+ // Get the set of containerizer types.
+ const vector<string> _types = strings::split(flags.containerizers, ",");
+ const set<string> containerizerTypes(_types.begin(), _types.end());
+
+ if (containerizerTypes.size() != _types.size()) {
+ return Error("Duplicate entries found in --containerizer flag"
+ " '" + flags.containerizers + "'");
+ }
+
// Optionally create the Nvidia components.
Option<NvidiaComponents> nvidia;
@@ -247,7 +256,7 @@ Try<Containerizer*> Containerizer::create(
// Create containerizer(s).
vector<Containerizer*> containerizers;
- foreach (const string& type, strings::split(flags.containerizers, ",")) {
+ foreach (const string& type, containerizerTypes) {
if (type == "mesos") {
Try<MesosContainerizer*> containerizer =
MesosContainerizer::create(flags, local, fetcher, nvidia);
[5/5] mesos git commit: Updated CHANGELOG about 1.0.1 fix (MESOS-5923
and MESOS-5959).
Posted by ji...@apache.org.
Updated CHANGELOG about 1.0.1 fix (MESOS-5923 and MESOS-5959).
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3ab813b5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3ab813b5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3ab813b5
Branch: refs/heads/1.0.x
Commit: 3ab813b5b991eff2f99bd3d228b2bbed65f1dfe2
Parents: b9d52a9
Author: Jie Yu <yu...@gmail.com>
Authored: Mon Aug 1 21:45:33 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Aug 1 21:48:03 2016 -0700
----------------------------------------------------------------------
CHANGELOG | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3ab813b5/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 61c326d..1c28bf3 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,8 @@ All Issues:
** Bug
* [MESOS-5911] - webUI redirection to leader in browser does not work.
* [MESOS-5913] - Stale socket FD usage when using libevent + SSL.
+ * [MESOS-5923] - Ubuntu 14.04 LTS GPU Isolator "/run" directory is noexec.
+ * [MESOS-5959] - All non-root tests fail on GPU machine.
Release Notes - Mesos - Version 1.0.0
[2/5] mesos git commit: Added check for root permissions to
'NvidiaVolume::create()'.
Posted by ji...@apache.org.
Added check for root permissions to 'NvidiaVolume::create()'.
Review: https://reviews.apache.org/r/50644/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7d66e4c5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7d66e4c5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7d66e4c5
Branch: refs/heads/1.0.x
Commit: 7d66e4c562ccb77f84bf744137fc0b973267f157
Parents: 094c897
Author: Kevin Klues <kl...@gmail.com>
Authored: Mon Aug 1 09:06:04 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Aug 1 21:47:21 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/mesos/isolators/gpu/volume.cpp | 11 +++++++++++
src/tests/containerizer/nvidia_gpu_isolator_tests.cpp | 4 ++--
2 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7d66e4c5/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/gpu/volume.cpp b/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
index 4b3651a..4d87c11 100644
--- a/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
+++ b/src/slave/containerizer/mesos/isolators/gpu/volume.cpp
@@ -191,6 +191,17 @@ const string& NvidiaVolume::CONTAINER_PATH() const
Try<NvidiaVolume> NvidiaVolume::create()
{
+ // We need root permissions in order to create the volume.
+ Result<string> user = os::user();
+ if (!user.isSome()) {
+ return Error("Failed to determine user: " +
+ (user.isError() ? user.error() : "username not found"));
+ }
+
+ if (user.get() != "root") {
+ return Error("NvidiaVolume::create() requires root privileges");
+ }
+
// Append the Nvidia driver version to the name of the volume.
Try<Nothing> initialized = nvml::initialize();
if (initialized.isError()) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/7d66e4c5/src/tests/containerizer/nvidia_gpu_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/nvidia_gpu_isolator_tests.cpp b/src/tests/containerizer/nvidia_gpu_isolator_tests.cpp
index 7944c0d..1473287 100644
--- a/src/tests/containerizer/nvidia_gpu_isolator_tests.cpp
+++ b/src/tests/containerizer/nvidia_gpu_isolator_tests.cpp
@@ -584,7 +584,7 @@ TEST_F(NvidiaGpuTest, NVIDIA_GPU_Allocator)
// Tests that we can create the volume that consolidates
// the Nvidia libraries and binaries.
-TEST_F(NvidiaGpuTest, NVIDIA_GPU_VolumeCreation)
+TEST_F(NvidiaGpuTest, ROOT_NVIDIA_GPU_VolumeCreation)
{
Try<NvidiaVolume> volume = NvidiaVolume::create();
ASSERT_SOME(volume);
@@ -603,7 +603,7 @@ TEST_F(NvidiaGpuTest, NVIDIA_GPU_VolumeCreation)
// Tests that we can properly detect when an Nvidia volume should be
// injected into a Docker container given its ImageManifest.
-TEST_F(NvidiaGpuTest, NVIDIA_GPU_VolumeShouldInject)
+TEST_F(NvidiaGpuTest, ROOT_NVIDIA_GPU_VolumeShouldInject)
{
Try<JSON::Object> json = JSON::parse<JSON::Object>(
R"~(
[4/5] mesos git commit: Added extra conditions for deciding when to
create "NvidiaComponents".
Posted by ji...@apache.org.
Added extra conditions for deciding when to create "NvidiaComponents".
A recent addition to ensure that 'NvidiaVolume::create()' ran as root
broke all non-root tests on GPU machines. The reason is that we
unconditionally create this volume so long as we detect
'nvml.isAvailable()', which will fail now that we are only allowed to
create this volume if we have root permissions.
We fix this by adding the proper conditions to determine when / if we
should create this volume based on some combination of --containerizer
and --isolation flags.
Review: https://reviews.apache.org/r/50672/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b9d52a98
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b9d52a98
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b9d52a98
Branch: refs/heads/1.0.x
Commit: b9d52a98b4d412bf42d1e8a5de5b84fb9a227d10
Parents: 62c7c70
Author: Kevin Klues <kl...@gmail.com>
Authored: Mon Aug 1 15:52:43 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Aug 1 21:47:45 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/containerizer.cpp | 52 ++++++++++++++++++--------
1 file changed, 36 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b9d52a98/src/slave/containerizer/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/containerizer.cpp b/src/slave/containerizer/containerizer.cpp
index f10a08a..ba3b3f6 100644
--- a/src/slave/containerizer/containerizer.cpp
+++ b/src/slave/containerizer/containerizer.cpp
@@ -225,28 +225,48 @@ Try<Containerizer*> Containerizer::create(
#ifdef __linux__
if (nvml::isAvailable()) {
- Try<Resources> gpus = NvidiaGpuAllocator::resources(flags);
-
- if (gpus.isError()) {
- return Error("Failed call to NvidiaGpuAllocator::resources: " +
- gpus.error());
+ // If we are using the docker containerizer (either alone or in
+ // conjunction with the mesos containerizer), unconditionally
+ // create the Nvidia components and pass them through. If we are
+ // using the mesos containerizer alone, make sure we also have the
+ // `gpu/nvidia` isolator flag set before creating these components.
+ bool shouldCreate = false;
+
+ if (containerizerTypes.count("docker") > 0) {
+ shouldCreate = true;
+ } else if (containerizerTypes.count("mesos") > 0) {
+ const vector<string> _isolators = strings::tokenize(flags.isolation, ",");
+ const set<string> isolators(_isolators.begin(), _isolators.end());
+
+ if (isolators.count("gpu/nvidia") > 0) {
+ shouldCreate = true;
+ }
}
- Try<NvidiaGpuAllocator> allocator =
- NvidiaGpuAllocator::create(flags, gpus.get());
+ if (shouldCreate) {
+ Try<Resources> gpus = NvidiaGpuAllocator::resources(flags);
- if (allocator.isError()) {
- return Error("Failed to NvidiaGpuAllocator::create: " +
- allocator.error());
- }
+ if (gpus.isError()) {
+ return Error("Failed call to NvidiaGpuAllocator::resources: " +
+ gpus.error());
+ }
- Try<NvidiaVolume> volume = NvidiaVolume::create();
+ Try<NvidiaGpuAllocator> allocator =
+ NvidiaGpuAllocator::create(flags, gpus.get());
- if (volume.isError()) {
- return Error("Failed to NvidiaVolume::create: " + volume.error());
- }
+ if (allocator.isError()) {
+ return Error("Failed to NvidiaGpuAllocator::create: " +
+ allocator.error());
+ }
+
+ Try<NvidiaVolume> volume = NvidiaVolume::create();
- nvidia = NvidiaComponents(allocator.get(), volume.get());
+ if (volume.isError()) {
+ return Error("Failed to NvidiaVolume::create: " + volume.error());
+ }
+
+ nvidia = NvidiaComponents(allocator.get(), volume.get());
+ }
}
#endif