You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/05/03 06:55:37 UTC

mesos git commit: Skipped the bind mounting of the agent's work_dir if possible.

Repository: mesos
Updated Branches:
  refs/heads/master 602c2b5cd -> 6b28aab05


Skipped the bind mounting of the agent's work_dir if possible.

In filesystem/linux isolator, we will need to make sure the mount that
contains the agent's working directory is a shared mount in its own
peer group so that cloning the host mount table (e.g., when launching
a container with a new mount namespace) does not create an extra
reference to the mounts under the agent's work_dir.

However, if the mount containing the agent's working directory is
already a shared mount in its own peer group, we don't have to create
another bind mount for the agent's working directory. This patch
allows the filesystem/linux isolator to skip the bind mount if
possible.

Review: https://reviews.apache.org/r/46858


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6b28aab0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6b28aab0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6b28aab0

Branch: refs/heads/master
Commit: 6b28aab052333e47bc53f66e13e4d78cc1bab3dd
Parents: 602c2b5
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 29 15:12:47 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon May 2 21:55:19 2016 -0700

----------------------------------------------------------------------
 .../mesos/isolators/filesystem/linux.cpp        | 157 +++++++++++--------
 .../containerizer/filesystem_isolator_tests.cpp |  66 ++++----
 2 files changed, 131 insertions(+), 92 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/6b28aab0/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp b/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
index 8805143..5edbce9 100644
--- a/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
+++ b/src/slave/containerizer/mesos/isolators/filesystem/linux.cpp
@@ -72,93 +72,120 @@ Try<Isolator*> LinuxFilesystemIsolatorProcess::create(const Flags& flags)
     return Error("LinuxFilesystemIsolator requires root privileges");
   }
 
-  // Make slave's work_dir a shared mount so that when forking a child
-  // process (with a new mount namespace), the child process does not
-  // hold extra references to container's work directory mounts and
-  // provisioner mounts (e.g., when using the bind backend) because
-  // cleanup operations within work_dir can be propagted to all
-  // container namespaces. See MESOS-3483 for more details.
-  LOG(INFO) << "Making '" << flags.work_dir << "' a shared mount";
+  // Make sure that slave's working directory is in a shared mount so
+  // that when forking a child process (with a new mount namespace),
+  // the child process does not hold extra references to container's
+  // persistent volume mounts and provisioner mounts (e.g., when using
+  // the bind/overlayfs backend). This ensures that cleanup operations
+  // within slave's working directory can be propagated to all
+  // containers. See MESOS-3483 for more details.
+
+  // Mount table entries use realpaths. Therefore, we first get the
+  // realpath of the slave's working directory.
+  Result<string> workDir = os::realpath(flags.work_dir);
+  if (!workDir.isSome()) {
+    return Error(
+        "Failed to get the realpath of slave's working directory: " +
+        (workDir.isError() ? workDir.error() : "Not found"));
+  }
 
   Try<fs::MountInfoTable> table = fs::MountInfoTable::read();
   if (table.isError()) {
     return Error("Failed to get mount table: " + table.error());
   }
 
+  // Trying to find the mount entry that contains the slave's working
+  // directory. We achieve that by doing a reverse traverse of the
+  // mount table to find the first entry whose target is a prefix of
+  // slave's working directory.
   Option<fs::MountInfoTable::Entry> workDirMount;
-  foreach (const fs::MountInfoTable::Entry& entry, table.get().entries) {
-    // TODO(jieyu): Make sure 'flags.work_dir' is a canonical path.
-    if (entry.target == flags.work_dir) {
+  foreach (const fs::MountInfoTable::Entry& entry,
+           adaptor::reverse(table->entries)) {
+    if (strings::startsWith(workDir.get(), entry.target)) {
       workDirMount = entry;
       break;
     }
   }
 
-  // Do a self bind mount if needed. If the mount already exists, make
-  // sure it is a shared mount of its own peer group.
+  // It's unlikely that we cannot find 'workDirMount' because '/' is
+  // always mounted and will be the 'workDirMount' if no other mounts
+  // found in between.
   if (workDirMount.isNone()) {
-    // NOTE: Instead of using fs::mount to perform the bind mount, we
-    // use the shell command here because the syscall 'mount' does not
-    // update the mount table (i.e., /etc/mtab). In other words, the
-    // mount will not be visible if the operator types command
-    // 'mount'. Since this mount will still be presented after all
-    // containers and the slave are stopped, it's better to make it
-    // visible. It's OK to use the blocking os::shell here because
-    // 'create' will only be invoked during initialization.
-    Try<string> mount = os::shell(
-        "mount --bind %s %s && "
-        "mount --make-slave %s && "
-        "mount --make-shared %s",
-        flags.work_dir.c_str(),
-        flags.work_dir.c_str(),
-        flags.work_dir.c_str(),
-        flags.work_dir.c_str());
-
-    if (mount.isError()) {
-      return Error(
-          "Failed to self bind mount '" + flags.work_dir +
-          "' and make it a shared mount: " + mount.error());
-    }
+    return Error("Cannot find the mount containing slave's working directory");
+  }
+
+  // If 'workDirMount' is a shared mount in its own peer group, then
+  // we don't need to do anything. Otherwise, we need to do a self
+  // bind mount of slave's working directory to make sure it's a
+  // shared mount in its own peer group.
+  bool bindMountNeeded = false;
+
+  if (workDirMount->shared().isNone()) {
+    bindMountNeeded = true;
   } else {
-    if (workDirMount.get().shared().isNone()) {
-      // This is the case where the work directory mount is not a
-      // shared mount yet (possibly due to slave crash while preparing
-      // the work directory mount). It's safe to re-do the following.
+    foreach (const fs::MountInfoTable::Entry& entry, table->entries) {
+      // Skip 'workDirMount' and any mount underneath it. Also, we
+      // skip those mounts whose targets are not the parent of the
+      // working directory because even if they are in the same peer
+      // group as the working directory mount, it won't affect it.
+      if (entry.id != workDirMount->id &&
+          !strings::startsWith(entry.target, workDir.get()) &&
+          entry.shared() == workDirMount->shared() &&
+          strings::startsWith(workDir.get(), entry.target)) {
+        bindMountNeeded = true;
+        break;
+      }
+    }
+  }
+
+  if (bindMountNeeded) {
+    if (workDirMount->target != workDir.get()) {
+      // This is the case where the working directory mount does not
+      // exist in the mount table (e.g., a new host running Mesos
+      // slave for the first time).
+      LOG(INFO) << "Bind mounting '" << workDir.get()
+                << "' and making it a shared mount";
+
+      // NOTE: Instead of using fs::mount to perform the bind mount,
+      // we use the shell command here because the syscall 'mount'
+      // does not update the mount table (i.e., /etc/mtab). In other
+      // words, the mount will not be visible if the operator types
+      // command 'mount'. Since this mount will still be presented
+      // after all containers and the slave are stopped, it's better
+      // to make it visible. It's OK to use the blocking os::shell
+      // here because 'create' will only be invoked during
+      // initialization.
       Try<string> mount = os::shell(
-          "mount --make-slave %s && "
+          "mount --bind %s %s && "
+          "mount --make-private %s && "
           "mount --make-shared %s",
-          flags.work_dir.c_str(),
-          flags.work_dir.c_str());
+          workDir->c_str(),
+          workDir->c_str(),
+          workDir->c_str(),
+          workDir->c_str());
 
       if (mount.isError()) {
         return Error(
-            "Failed to self bind mount '" + flags.work_dir +
+            "Failed to bind mount '" + workDir.get() +
             "' and make it a shared mount: " + mount.error());
       }
     } else {
-      // We need to make sure that the shared mount is in its own peer
-      // group. To check that, we need to get the parent mount.
-      foreach (const fs::MountInfoTable::Entry& entry, table.get().entries) {
-        if (entry.id == workDirMount.get().parent) {
-          // If the work directory mount and its parent mount are in
-          // the same peer group, we need to re-do the following
-          // commands so that they are in different peer groups.
-          if (entry.shared() == workDirMount.get().shared()) {
-            Try<string> mount = os::shell(
-                "mount --make-slave %s && "
-                "mount --make-shared %s",
-                flags.work_dir.c_str(),
-                flags.work_dir.c_str());
-
-            if (mount.isError()) {
-              return Error(
-                  "Failed to self bind mount '" + flags.work_dir +
-                  "' and make it a shared mount: " + mount.error());
-            }
-          }
-
-          break;
-        }
+      // This is the case where the working directory mount is in the
+      // mount table, but it's not a shared mount in its own peer
+      // group (possibly due to slave crash while preparing the
+      // working directory mount). It's safe to re-do the following.
+      LOG(INFO) << "Making '" << workDir.get() << "' a shared mount";
+
+      Try<string> mount = os::shell(
+          "mount --make-private %s && "
+          "mount --make-shared %s",
+          workDir->c_str(),
+          workDir->c_str());
+
+      if (mount.isError()) {
+        return Error(
+            "Failed to make '" + workDir.get() +
+            "' a shared mount: " + mount.error());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/mesos/blob/6b28aab0/src/tests/containerizer/filesystem_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/filesystem_isolator_tests.cpp b/src/tests/containerizer/filesystem_isolator_tests.cpp
index 29d3130..84ee944 100644
--- a/src/tests/containerizer/filesystem_isolator_tests.cpp
+++ b/src/tests/containerizer/filesystem_isolator_tests.cpp
@@ -96,15 +96,10 @@ protected:
   {
     // Try to remove any mounts under sandbox.
     if (::geteuid() == 0) {
-      Try<string> umount = os::shell(
-          "grep '%s' /proc/mounts | "
-          "cut -d' ' -f2 | "
-          "xargs --no-run-if-empty umount -l",
-          sandbox.get().c_str());
-
-      if (umount.isError()) {
+      Try<Nothing> unmount = fs::unmountAll(sandbox.get(), MNT_DETACH);
+      if (unmount.isError()) {
         LOG(ERROR) << "Failed to umount for sandbox '" << sandbox.get()
-                   << "': " << umount.error();
+                   << "': " << unmount.error();
       }
     }
 
@@ -1391,56 +1386,73 @@ TEST_F(LinuxFilesystemIsolatorTest, ROOT_SandboxEnvironmentVariable)
 }
 
 
-// This test verifies the slave's work directory mount preparation if
-// the mount does not exist initially.
-TEST_F(LinuxFilesystemIsolatorTest, ROOT_WorkDirMount)
+// This test verifies the case where we don't need a bind mount for
+// slave's working directory because the mount containing it is
+// already a shared mount in its own peer group.
+TEST_F(LinuxFilesystemIsolatorTest, ROOT_WorkDirMountNotNeeded)
 {
+  // Make 'sandbox' a shared mount in its own peer group.
+  ASSERT_SOME(os::shell(
+      "mount --bind %s %s && "
+      "mount --make-private %s &&"
+      "mount --make-shared %s",
+      sandbox->c_str(),
+      sandbox->c_str(),
+      sandbox->c_str(),
+      sandbox->c_str()));
+
+  // Slave's working directory is under 'sandbox'.
   slave::Flags flags = CreateSlaveFlags();
+  flags.work_dir = path::join(sandbox.get(), "slave");
 
-  Try<Isolator*> isolator = LinuxFilesystemIsolatorProcess::create(flags);
+  ASSERT_SOME(os::mkdir(flags.work_dir));
 
+  Try<Isolator*> isolator = LinuxFilesystemIsolatorProcess::create(flags);
   ASSERT_SOME(isolator);
 
   Try<fs::MountInfoTable> table = fs::MountInfoTable::read();
   ASSERT_SOME(table);
 
+  // Verifies that there's no mount for slave's working directory.
   bool mountFound = false;
-  foreach (const fs::MountInfoTable::Entry& entry, table.get().entries) {
+  foreach (const fs::MountInfoTable::Entry& entry, table->entries) {
     if (entry.target == flags.work_dir) {
-      EXPECT_SOME(entry.shared());
       mountFound = true;
     }
   }
 
-  EXPECT_TRUE(mountFound);
+  EXPECT_FALSE(mountFound);
 
   delete isolator.get();
 }
 
 
-// This test verifies the slave's work directory mount preparation if
-// the mount already exists (e.g., to simulate the case when the slave
-// crashes while preparing the work directory mount).
-TEST_F(LinuxFilesystemIsolatorTest, ROOT_WorkDirMountPreExists)
+// This test verifies the case where we do need a bind mount for
+// slave's working directory because the mount containing it is not a
+// shared mount in its own peer group.
+TEST_F(LinuxFilesystemIsolatorTest, ROOT_WorkDirMountNeeded)
 {
+  // Make 'sandbox' a private mount.
+  ASSERT_SOME(os::shell(
+      "mount --bind %s %s && "
+      "mount --make-private %s",
+      sandbox->c_str(),
+      sandbox->c_str(),
+      sandbox->c_str()));
+
   slave::Flags flags = CreateSlaveFlags();
+  flags.work_dir = path::join(sandbox.get(), "slave");
 
-  // Simulate the situation in which the slave crashes while preparing
-  // the work directory mount.
-  ASSERT_SOME(os::shell(
-      "mount --bind %s %s",
-      flags.work_dir.c_str(),
-      flags.work_dir.c_str()));
+  ASSERT_SOME(os::mkdir(flags.work_dir));
 
   Try<Isolator*> isolator = LinuxFilesystemIsolatorProcess::create(flags);
-
   ASSERT_SOME(isolator);
 
   Try<fs::MountInfoTable> table = fs::MountInfoTable::read();
   ASSERT_SOME(table);
 
   bool mountFound = false;
-  foreach (const fs::MountInfoTable::Entry& entry, table.get().entries) {
+  foreach (const fs::MountInfoTable::Entry& entry, table->entries) {
     if (entry.target == flags.work_dir) {
       EXPECT_SOME(entry.shared());
       mountFound = true;