You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by id...@apache.org on 2014/06/24 21:43:12 UTC

git commit: Wait on orphan cgroup destroy during launcher recovery.

Repository: mesos
Updated Branches:
  refs/heads/master 6b66a0714 -> f9506d6e5


Wait on orphan cgroup destroy during launcher recovery.

Review: https://reviews.apache.org/r/22893


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f9506d6e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f9506d6e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f9506d6e

Branch: refs/heads/master
Commit: f9506d6e51cd09e22bf4ff6ba8d9c4eb0882799c
Parents: 6b66a07
Author: Ian Downes <id...@twitter.com>
Authored: Mon Jun 23 14:56:34 2014 -0700
Committer: Ian Downes <id...@twitter.com>
Committed: Tue Jun 24 12:26:36 2014 -0700

----------------------------------------------------------------------
 src/slave/containerizer/launcher.cpp            | 12 +++----
 src/slave/containerizer/launcher.hpp            |  6 ++--
 src/slave/containerizer/linux_launcher.cpp      | 35 ++++++++++++++------
 src/slave/containerizer/linux_launcher.hpp      |  3 +-
 src/slave/containerizer/mesos_containerizer.cpp | 15 +++++----
 src/slave/containerizer/mesos_containerizer.hpp |  3 ++
 6 files changed, 48 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/launcher.cpp b/src/slave/containerizer/launcher.cpp
index c43396a..2b13d1f 100644
--- a/src/slave/containerizer/launcher.cpp
+++ b/src/slave/containerizer/launcher.cpp
@@ -47,18 +47,18 @@ Try<Launcher*> PosixLauncher::create(const Flags& flags)
 }
 
 
-Try<Nothing> PosixLauncher::recover(const list<RunState>& states)
+Future<Nothing> PosixLauncher::recover(const list<RunState>& states)
 {
   foreach (const RunState& state, states) {
     if (state.id.isNone()) {
-      return Error("ContainerID is required to recover");
+      return Failure("ContainerID is required to recover");
     }
 
     const ContainerID& containerId = state.id.get();
 
     if (state.forkedPid.isNone()) {
-      return Error("Executor pid is required to recover container " +
-                   stringify(containerId));
+      return Failure("Executor pid is required to recover container " +
+                     stringify(containerId));
     }
     pid_t pid = state.forkedPid.get();
 
@@ -69,8 +69,8 @@ Try<Nothing> PosixLauncher::recover(const list<RunState>& states)
       // before it hears about the termination of the earlier executor (also
       // unlikely). Regardless, the launcher can't do anything sensible so this
       // is considered an error.
-      return Error("Detected duplicate pid " + stringify(pid) +
-                   " for container " + stringify(containerId));
+      return Failure("Detected duplicate pid " + stringify(pid) +
+                     " for container " + stringify(containerId));
     }
 
     pids.put(containerId, pid);

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/launcher.hpp b/src/slave/containerizer/launcher.hpp
index 63c0cb6..835d7a9 100644
--- a/src/slave/containerizer/launcher.hpp
+++ b/src/slave/containerizer/launcher.hpp
@@ -39,7 +39,8 @@ public:
   virtual ~Launcher() {}
 
   // Recover the necessary state for each container listed in state.
-  virtual Try<Nothing> recover(const std::list<state::RunState>& states) = 0;
+  virtual process::Future<Nothing> recover(
+      const std::list<state::RunState>& states) = 0;
 
   // Fork a new process in the containerized context. The child will call the
   // specified function and the parent will return the child's pid.
@@ -65,7 +66,8 @@ public:
 
   virtual ~PosixLauncher() {}
 
-  virtual Try<Nothing> recover(const std::list<state::RunState>& states);
+  virtual process::Future<Nothing> recover(
+      const std::list<state::RunState>& states);
 
   virtual Try<pid_t> fork(
       const ContainerID& containerId,

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/linux_launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/linux_launcher.cpp b/src/slave/containerizer/linux_launcher.cpp
index 85c74f0..7ebccb4 100644
--- a/src/slave/containerizer/linux_launcher.cpp
+++ b/src/slave/containerizer/linux_launcher.cpp
@@ -23,6 +23,8 @@
 
 #include <vector>
 
+#include <process/collect.hpp>
+
 #include <stout/abort.hpp>
 #include <stout/hashset.hpp>
 #include <stout/path.hpp>
@@ -73,13 +75,19 @@ Try<Launcher*> LinuxLauncher::create(const Flags& flags)
 }
 
 
-Try<Nothing> LinuxLauncher::recover(const std::list<state::RunState>& states)
+Future<Nothing> _recover(const Future<list<Nothing> >& futures)
+{
+  return Nothing();
+}
+
+
+Future<Nothing> LinuxLauncher::recover(const std::list<state::RunState>& states)
 {
   hashset<string> cgroups;
 
   foreach (const RunState& state, states) {
     if (state.id.isNone()) {
-      return Error("ContainerID is required to recover");
+      return Failure("ContainerID is required to recover");
     }
     const ContainerID& containerId = state.id.get();
 
@@ -95,8 +103,8 @@ Try<Nothing> LinuxLauncher::recover(const std::list<state::RunState>& states)
     }
 
     if (state.forkedPid.isNone()) {
-      return Error("Executor pid is required to recover container " +
-                   stringify(containerId));
+      return Failure("Executor pid is required to recover container " +
+                     stringify(containerId));
     }
     pid_t pid = state.forkedPid.get();
 
@@ -107,8 +115,8 @@ Try<Nothing> LinuxLauncher::recover(const std::list<state::RunState>& states)
       // before it hears about the termination of the earlier executor (also
       // unlikely). Regardless, the launcher can't do anything sensible so this
       // is considered an error.
-      return Error("Detected duplicate pid " + stringify(pid) +
-                   " for container " + stringify(containerId));
+      return Failure("Detected duplicate pid " + stringify(pid) +
+                     " for container " + stringify(containerId));
     }
 
     pids.put(containerId, pid);
@@ -118,20 +126,25 @@ Try<Nothing> LinuxLauncher::recover(const std::list<state::RunState>& states)
 
   Try<vector<string> > orphans = cgroups::get(hierarchy, flags.cgroups_root);
   if (orphans.isError()) {
-    return Error(orphans.error());
+    return Failure(orphans.error());
   }
 
+  list<Future<Nothing> > futures;
+
   foreach (const string& orphan, orphans.get()) {
     if (!cgroups.contains(orphan)) {
       LOG(INFO) << "Removing orphaned cgroup"
                 << " '" << path::join("freezer", orphan) << "'";
-      // Do not wait on the destroy to complete so we don't block
-      // recovery.
-      cgroups::destroy(hierarchy, orphan, cgroups::DESTROY_TIMEOUT);
+      // We must wait for all cgroups to be destroyed; isolators
+      // assume all processes have been terminated for any orphaned
+      // containers before Isolator::recover() is called.
+      futures.push_back(
+          cgroups::destroy(hierarchy, orphan, cgroups::DESTROY_TIMEOUT));
     }
   }
 
-  return Nothing();
+  return collect(futures)
+    .then(lambda::bind(&_recover, lambda::_1));
 }
 
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/linux_launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/linux_launcher.hpp b/src/slave/containerizer/linux_launcher.hpp
index 60b401f..622810c 100644
--- a/src/slave/containerizer/linux_launcher.hpp
+++ b/src/slave/containerizer/linux_launcher.hpp
@@ -34,7 +34,8 @@ public:
 
   virtual ~LinuxLauncher() {}
 
-  virtual Try<Nothing> recover(const std::list<state::RunState>& states);
+  virtual process::Future<Nothing> recover(
+      const std::list<state::RunState>& states);
 
   virtual Try<pid_t> fork(
       const ContainerID& containerId,

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/mesos_containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos_containerizer.cpp b/src/slave/containerizer/mesos_containerizer.cpp
index 61c0a8d..76c101c 100644
--- a/src/slave/containerizer/mesos_containerizer.cpp
+++ b/src/slave/containerizer/mesos_containerizer.cpp
@@ -332,11 +332,14 @@ Future<Nothing> MesosContainerizerProcess::recover(
   }
 
   // Try to recover the launcher first.
-  Try<Nothing> recover = launcher->recover(recoverable);
-  if (recover.isError()) {
-    return Failure(recover.error());
-  }
+  return launcher->recover(recoverable)
+    .then(defer(self(), &Self::_recover, recoverable));
+}
+
 
+Future<Nothing> MesosContainerizerProcess::_recover(
+    const list<RunState>& recoverable)
+{
   // Then recover the isolators.
   list<Future<Nothing> > futures;
   foreach (const Owned<Isolator>& isolator, isolators) {
@@ -345,11 +348,11 @@ Future<Nothing> MesosContainerizerProcess::recover(
 
   // If all isolators recover then continue.
   return collect(futures)
-    .then(defer(self(), &Self::_recover, recoverable));
+    .then(defer(self(), &Self::__recover, recoverable));
 }
 
 
-Future<Nothing> MesosContainerizerProcess::_recover(
+Future<Nothing> MesosContainerizerProcess::__recover(
     const list<RunState>& recovered)
 {
   foreach (const RunState& run, recovered) {

http://git-wip-us.apache.org/repos/asf/mesos/blob/f9506d6e/src/slave/containerizer/mesos_containerizer.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos_containerizer.hpp b/src/slave/containerizer/mesos_containerizer.hpp
index 1f5908a..21affae 100644
--- a/src/slave/containerizer/mesos_containerizer.hpp
+++ b/src/slave/containerizer/mesos_containerizer.hpp
@@ -145,6 +145,9 @@ public:
 
 private:
   process::Future<Nothing> _recover(
+      const std::list<state::RunState>& recoverable);
+
+  process::Future<Nothing> __recover(
       const std::list<state::RunState>& recovered);
 
   process::Future<std::list<Option<CommandInfo> > > prepare(