You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by id...@apache.org on 2014/06/17 20:04:56 UTC

[3/3] git commit: Refactor cgroups::internal::Freezer.

Refactor cgroups::internal::Freezer.

The Freezer tries to converge to the "FROZEN" state by repeatedly (every
100 ms) writing "FROZEN" to the freezer.state control file. It assumes
there are two possible reasons why a process does not get frozen during
an attempt:

1. It was in the middle of being forked and did not receive the signal;
it will receive it at the next attempt.

2. It is in uninterruptable sleep ("D" state). Normally, this is from
device I/O or paging and is shortlived, in which case it'll be frozen on
retry. However, processes can get stuck in "D" state, either because of
a device issue, incorrect OOM handling, or kernel bugs. Under this
scenario the correct behavior is to fail after a timeout (defaults to 60
seconds).

Freezer functions have been namespaced under cgroups::freezer.

Review: https://reviews.apache.org/r/20817


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/07c3dc57
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/07c3dc57
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/07c3dc57

Branch: refs/heads/master
Commit: 07c3dc57fa272a9c25b5df2f7ac218fdcfb2f5d4
Parents: fb13868
Author: Ian Downes <id...@twitter.com>
Authored: Wed Apr 23 10:48:33 2014 -0700
Committer: Ian Downes <id...@twitter.com>
Committed: Tue Jun 17 11:00:41 2014 -0700

----------------------------------------------------------------------
 src/linux/cgroups.cpp       | 350 +++++++++++++++------------------------
 src/linux/cgroups.hpp       |  72 ++++----
 src/tests/cgroups_tests.cpp |  65 ++++----
 3 files changed, 195 insertions(+), 292 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/linux/cgroups.cpp
----------------------------------------------------------------------
diff --git a/src/linux/cgroups.cpp b/src/linux/cgroups.cpp
index 142ac43..723acc1 100644
--- a/src/linux/cgroups.cpp
+++ b/src/linux/cgroups.cpp
@@ -1296,270 +1296,144 @@ Future<uint64_t> listen(
   return future;
 }
 
-
 namespace internal {
 
+namespace freezer {
 
-// The process that freezes or thaws the cgroup.
-class Freezer : public Process<Freezer>
+Try<string> state(const string& hierarchy, const string& cgroup)
 {
-public:
-  Freezer(const string& _hierarchy,
-          const string& _cgroup,
-          const string& _action,
-          const Duration& _interval,
-          unsigned int _retries = FREEZE_RETRIES)
-    : hierarchy(_hierarchy),
-      cgroup(_cgroup),
-      action(_action),
-      interval(_interval),
-      retries(_retries) {}
-
-  virtual ~Freezer() {}
+  Try<string> state = cgroups::read(hierarchy, cgroup, "freezer.state");
 
-  // Return a future indicating the state of the freezer.
-  Future<bool> future() { return promise.future(); }
+  if (state.isError()) {
+    return Error("Failed to read freezer state: " + state.error());
+  }
 
-protected:
-  virtual void initialize()
-  {
-    // Stop the process if no one cares.
-    promise.future().onDiscard(lambda::bind(
-        static_cast<void (*)(const UPID&, bool)>(terminate), self(), true));
+  return strings::trim(state.get());
+}
 
-    CHECK(interval >= Seconds(0));
 
-    // Start the action.
-    CHECK(action == "FREEZE" || action == "THAW");
-    if (action == "FREEZE") {
-      freeze();
-    } else if (action == "THAW") {
-      thaw();
-    }
+Try<Nothing> state(
+    const string& hierarchy,
+    const string& cgroup,
+    const string& state)
+{
+  if (state != "FROZEN" && state != "THAWED") {
+    return Error("Invalid freezer state requested: " + state);
   }
 
-  virtual void finalize()
-  {
-    promise.discard();
+  Try<Nothing> write = cgroups::write(
+      hierarchy, cgroup, "freezer.state", state);
+  if (write.isError()) {
+    return Error("Failed to write '" + state +
+                 "' to control 'freezer.state': " + write.error());
+  } else {
+    return Nothing();
   }
+}
 
-private:
-  void freeze()
-  {
-    LOG(INFO) << "Trying to freeze cgroup " << path::join(hierarchy, cgroup);
+} // namespace freezer {
 
-    Try<Nothing> write = internal::write(
-        hierarchy, cgroup, "freezer.state", "FROZEN");
+class Freezer : public Process<Freezer>
+{
+public:
+  Freezer(
+      const string& _hierarchy,
+      const string& _cgroup)
+    : hierarchy(_hierarchy),
+      cgroup(_cgroup),
+      start(Clock::now()) {}
 
-    if (write.isError()) {
-      promise.fail("Failed to write control 'freezer.state': " + write.error());
-      terminate(self());
-    } else {
-      watchFrozen();
-    }
-  }
+  virtual ~Freezer() {}
 
-  void thaw()
+  void freeze()
   {
-    LOG(INFO) << "Trying to thaw cgroup " << path::join(hierarchy, cgroup);
-
-    Try<Nothing> write = internal::write(
-        hierarchy, cgroup, "freezer.state", "THAWED");
-
-    if (write.isError()) {
-      promise.fail("Failed to write control 'freezer.state': " + write.error());
+    Try<Nothing> freeze =
+      internal::freezer::state(hierarchy, cgroup, "FROZEN");
+    if (freeze.isError()) {
+      promise.fail(freeze.error());
       terminate(self());
-    } else {
-      watchThawed();
+      return;
     }
-  }
-
-  void watchFrozen(unsigned int attempt = 0)
-  {
-    Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
 
+    Try<string> state = internal::freezer::state(hierarchy, cgroup);
     if (state.isError()) {
-      promise.fail("Failed to read control 'freezer.state': " + state.error());
+      promise.fail(state.error());
       terminate(self());
       return;
     }
 
-    if (strings::trim(state.get()) == "FROZEN") {
-      LOG(INFO) << "Successfully froze cgroup " << path::join(hierarchy, cgroup)
-                << " after " << attempt + 1 << " attempts";
-      promise.set(true);
+    if (state.get() == "FROZEN") {
+      LOG(INFO) << "Successfullly froze cgroup "
+                << path::join(hierarchy, cgroup)
+                << " after " << (Clock::now() - start);
+      promise.set(Nothing());
       terminate(self());
       return;
-    } else if (strings::trim(state.get()) == "FREEZING") {
-      // The freezer.state is in FREEZING state. This is because not all the
-      // processes in the given cgroup can be frozen at the moment. The main
-      // cause is that some of the processes are in stopped/traced state ('T'
-      // state shown in ps command). It is likely that the freezer.state keeps
-      // in FREEZING state if these stopped/traced processes are not resumed.
-      // Therefore, here we send SIGCONT to those stopped/traced processes to
-      // make sure that the freezer can finish.
-      // TODO(jieyu): This code can be removed in the future as the newer
-      // version of the kernel solves this problem (e.g. Linux-3.2.0).
-      Try<set<pid_t> > pids = processes(hierarchy, cgroup);
-      if (pids.isError()) {
-        promise.fail("Failed to get processes of cgroup: " + pids.error());
-        terminate(self());
-        return;
-      }
-
-      // It appears possible for processes to go away while the cgroup
-      // is in the FREEZING state. We ignore such processes.
-      // See: https://issues.apache.org/jira/browse/MESOS-461
-      foreach (pid_t pid, pids.get()) {
-        Result<proc::ProcessStatus> status = proc::status(pid);
-
-        if (!status.isSome()) {
-          LOG(WARNING)
-            << "Failed to get process status for pid " << pid << ": "
-            << (status.isError() ? status.error() : "pid does not exist");
-          continue;
-        }
-
-        // Check whether the process is in stopped/traced state.
-        if (status.get().state == 'T') {
-          // Send a SIGCONT signal to the process.
-          if (::kill(pid, SIGCONT) == -1) {
-            promise.fail(
-                "Failed to send SIGCONT to process " + stringify(pid) +
-                ": " + string(strerror(errno)));
-            terminate(self());
-            return;
-          }
-        }
-      }
-
-      if (attempt > retries) {
-        LOG(WARNING) << "Unable to freeze " << path::join(hierarchy, cgroup)
-                     << " within " << retries + 1 << " attempts";
-        promise.set(false);
-        terminate(self());
-        return;
-      }
-
-      // Retry the freezing operation.
-      Try<Nothing> write = internal::write(
-          hierarchy, cgroup, "freezer.state", "FROZEN");
-
-      if (write.isError()) {
-        promise.fail(
-            "Failed to write control 'freezer.state': " + write.error());
-        terminate(self());
-        return;
-      }
-
-      // Not done yet, keep watching (and possibly retrying).
-      delay(interval, self(), &Freezer::watchFrozen, attempt + 1);
-    } else {
-      LOG(FATAL) << "Unexpected state: " << strings::trim(state.get())
-                 << " of cgroup " << path::join(hierarchy, cgroup);
     }
+
+    // Attempt to freeze the freezer cgroup again.
+    delay(Milliseconds(100), self(), &Self::freeze);
   }
 
-  void watchThawed()
+  void thaw()
   {
-    Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
+    Try<Nothing> thaw = internal::freezer::state(hierarchy, cgroup, "THAWED");
+    if (thaw.isError()) {
+      promise.fail(thaw.error());
+      terminate(self());
+      return;
+    }
 
+    Try<string> state = internal::freezer::state(hierarchy, cgroup);
     if (state.isError()) {
-      promise.fail("Failed to read control 'freezer.state': " + state.error());
+      promise.fail(state.error());
       terminate(self());
       return;
     }
 
-    if (strings::trim(state.get()) == "THAWED") {
-      LOG(INFO) << "Successfully thawed " << path::join(hierarchy, cgroup);
-      promise.set(true);
+    if (state.get() == "THAWED") {
+      LOG(INFO) << "Successfullly thawed cgroup "
+                << path::join(hierarchy, cgroup)
+                << " after " << (Clock::now() - start);
+      promise.set(Nothing());
       terminate(self());
-    } else if (strings::trim(state.get()) == "FROZEN") {
-      // Not done yet, keep watching.
-      delay(interval, self(), &Freezer::watchThawed);
-    } else {
-      LOG(FATAL) << "Unexpected state: " << strings::trim(state.get())
-                 << " of cgroup " << path::join(hierarchy, cgroup);
+      return;
     }
-  }
 
-  const string hierarchy;
-  const string cgroup;
-  const string action;
-  const Duration interval;
-  const unsigned int retries;
-  Promise<bool> promise;
-};
-
-} // namespace internal {
-
-
-Future<bool> freeze(
-    const string& hierarchy,
-    const string& cgroup,
-    const Duration& interval,
-    unsigned int retries)
-{
-  Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
-  if (error.isSome()) {
-    return Failure(error.get());
+    // Attempt to thaw the freezer cgroup again.
+    delay(Milliseconds(100), self(), &Self::thaw);
   }
 
-  if (interval < Seconds(0)) {
-    return Failure("Interval should be non-negative");
-  }
-
-  // Check the current freezer state.
-  Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
-  if (state.isError()) {
-    return Failure(
-        "Failed to read control 'freezer.state': " + state.error());
-  } else if (strings::trim(state.get()) == "FROZEN") {
-    // Immediately return success.
-    return true;
-  }
+  Future<Nothing> future() { return promise.future(); }
 
-  internal::Freezer* freezer =
-    new internal::Freezer(hierarchy, cgroup, "FREEZE", interval, retries);
-  Future<bool> future = freezer->future();
-  spawn(freezer, true);
-  return future;
-}
-
-
-Future<bool> thaw(
-    const string& hierarchy,
-    const string& cgroup,
-    const Duration& interval)
-{
-  Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
-  if (error.isSome()) {
-    return Failure(error.get());
-  }
+protected:
+  virtual void initialize()
+  {
+    Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
+    if (error.isSome()) {
+      promise.fail("Invalid freezer cgroup: " + error.get().message);
+      terminate(self());
+      return;
+    }
 
-  if (interval < Seconds(0)) {
-    return Failure("Interval should be non-negative");
+    // Stop attempting to freeze/thaw if nobody cares.
+    promise.future().onDiscard(lambda::bind(
+          static_cast<void(*)(const UPID&, bool)>(terminate), self(), true));
   }
 
-  // Check the current freezer state.
-  Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
-  if (state.isError()) {
-    return Failure(
-        "Failed to read control 'freezer.state': " + state.error());
-  } else if (strings::trim(state.get()) == "THAWED") {
-    // Immediately return success.
-    return true;
+  virtual void finalize()
+  {
+    promise.discard();
   }
 
-  internal::Freezer* freezer =
-    new internal::Freezer(hierarchy, cgroup, "THAW", interval);
-  Future<bool> future = freezer->future();
-  spawn(freezer, true);
-  return future;
-}
+private:
 
+  const string hierarchy;
+  const string cgroup;
+  const Time start;
+  Promise<Nothing> promise;
+};
 
-namespace internal {
 
 // The process used to wait for a cgroup to become empty (no task in it).
 class EmptyWatcher: public Process<EmptyWatcher>
@@ -1701,9 +1575,9 @@ private:
     chain.onAny(defer(self(), &Self::finished, lambda::_1));
   }
 
-  Future<bool> freeze()
+  Future<Nothing> freeze()
   {
-    return cgroups::freeze(hierarchy, cgroup, interval);
+    return cgroups::freezer::freeze(hierarchy, cgroup);
   }
 
   Future<Nothing> kill(const int signal)
@@ -1715,9 +1589,9 @@ private:
     return Nothing();
   }
 
-  Future<bool> thaw()
+  Future<Nothing> thaw()
   {
-    return cgroups::thaw(hierarchy, cgroup, interval);
+    return cgroups::freezer::thaw(hierarchy, cgroup);
   }
 
   Future<bool> empty()
@@ -2114,4 +1988,42 @@ Try<Bytes> max_usage_in_bytes(const string& hierarchy, const string& cgroup)
 
 } // namespace memory {
 
+
+namespace freezer {
+
+Future<Nothing> freeze(
+    const string& hierarchy,
+    const string& cgroup)
+{
+  LOG(INFO) << "Freezing cgroup " << path::join(hierarchy, cgroup);
+
+  internal::Freezer* freezer = new internal::Freezer(hierarchy, cgroup);
+
+  Future<Nothing> future = freezer->future();
+  spawn(freezer, true);
+
+  dispatch(freezer, &internal::Freezer::freeze);
+
+  return future;
+}
+
+
+Future<Nothing> thaw(
+    const string& hierarchy,
+    const string& cgroup)
+{
+  LOG(INFO) << "Thawing cgroup " << path::join(hierarchy, cgroup);
+
+  internal::Freezer* freezer = new internal::Freezer(hierarchy, cgroup);
+
+  Future<Nothing> future = freezer->future();
+  spawn(freezer, true);
+
+  dispatch(freezer, &internal::Freezer::thaw);
+
+  return future;
+}
+
+} // namespace freezer {
+
 } // namespace cgroups {

http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/linux/cgroups.hpp
----------------------------------------------------------------------
diff --git a/src/linux/cgroups.hpp b/src/linux/cgroups.hpp
index 21d87a0..599badb 100644
--- a/src/linux/cgroups.hpp
+++ b/src/linux/cgroups.hpp
@@ -29,6 +29,7 @@
 #include <sys/types.h>
 
 #include <process/future.hpp>
+#include <process/timeout.hpp>
 
 #include <stout/bytes.hpp>
 #include <stout/duration.hpp>
@@ -39,8 +40,7 @@
 
 namespace cgroups {
 
-// Default number of retry attempts when trying to freeze a cgroup.
-const unsigned int FREEZE_RETRIES = 50;
+const Duration DESTROY_TIMEOUT = Seconds(60);
 const unsigned int EMPTY_WATCHER_RETRIES = 50;
 
 // Default number of assign attempts when moving threads to a cgroup.
@@ -321,48 +321,6 @@ process::Future<uint64_t> listen(
     const Option<std::string>& args = Option<std::string>::none());
 
 
-// Freeze all the processes in a given cgroup. We try to use the freezer
-// subsystem implemented in cgroups. More detail can be found in
-// <kernel-source>/Documentation/cgroups/freezer-subsystem.txt. This function
-// will return a future which will become ready when all the processes have been
-// frozen (FROZEN). The future can be discarded to cancel the operation. The
-// freezer state after the cancellation is not defined. So the users need to
-// read the control file if they need to know the freezer state after the
-// cancellation. This function will return future failure if the freezer
-// subsystem is not available or it is not attached to the given hierarchy, or
-// the given cgroup is not valid, or the given cgroup has already been frozen.
-// @param   hierarchy   Path to the hierarchy root.
-// @param   cgroup      Path to the cgroup relative to the hierarchy root.
-// @param   interval    The time interval between two state check
-//                      requests (default: 0.1 seconds).
-// @param   retries     Number of retry attempts before giving up. None
-//                      indicates infinite retries. (default: 50 attempts).
-// @return  A future which will become true when all processes are frozen, or
-//          false when all retries have occurred unsuccessfully.
-//          Error if something unexpected happens.
-process::Future<bool> freeze(
-    const std::string& hierarchy,
-    const std::string& cgroup,
-    const Duration& interval = Milliseconds(100),
-    const unsigned int retries = FREEZE_RETRIES);
-
-
-// Thaw the given cgroup. This is a revert operation of freezeCgroup. It will
-// return error if the given cgroup is already thawed. Same as
-// freezeCgroup, this function will return a future which can be discarded to
-// allow users to cancel the operation.
-// @param   hierarchy   Path to the hierarchy root.
-// @param   cgroup      Path to the cgroup relative to the hierarchy root.
-// @param   interval    The time interval between two state check
-//                      requests (default: 0.1 seconds).
-// @return  A future which will become ready when all processes are thawed.
-//          Error if something unexpected happens.
-process::Future<bool> thaw(
-    const std::string& hierarchy,
-    const std::string& cgroup,
-    const Duration& interval = Milliseconds(100));
-
-
 // Destroy a cgroup under a given hierarchy. It will also recursively
 // destroy any sub-cgroups. If the freezer subsystem is attached to
 // the hierarchy, we attempt to kill all tasks in a given cgroup,
@@ -482,6 +440,32 @@ Try<Bytes> max_usage_in_bytes(
 
 } // namespace memory {
 
+
+// Freezer controls.
+// The freezer can be in one of three states:
+// 1. THAWED   : No process in the cgroup is frozen.
+// 2. FREEZING : Freezing is in progress but not all processes are frozen.
+// 3. FROZEN   : All processes are frozen.
+namespace freezer {
+
+// Freeze all processes in the given cgroup. The cgroup must be in a freezer
+// hierarchy. This function will return a future which will become ready when
+// all processes have been frozen (cgroup is in the FROZEN state).
+process::Future<Nothing> freeze(
+    const std::string& hierarchy,
+    const std::string& cgroup);
+
+
+// Thaw all processes in the given cgroup. The cgroup must be in a freezer
+// hierarchy. This is a revert operation of freezer::freeze. This function will
+// return a future which will become ready when all processes have been thawed
+// (cgroup is in the THAWED state).
+process::Future<Nothing> thaw(
+    const std::string& hierarchy,
+    const std::string& cgroup);
+
+} // namespace freezer {
+
 } // namespace cgroups {
 
 #endif // __CGROUPS_HPP__

http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/tests/cgroups_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/cgroups_tests.cpp b/src/tests/cgroups_tests.cpp
index adcf46b..c00c30a 100644
--- a/src/tests/cgroups_tests.cpp
+++ b/src/tests/cgroups_tests.cpp
@@ -602,35 +602,7 @@ TEST_F(CgroupsAnyHierarchyWithFreezerTest, ROOT_CGROUPS_Freeze)
   pid_t pid = ::fork();
   ASSERT_NE(-1, pid);
 
-  if (pid > 0) {
-    // In parent process.
-    ::close(pipes[1]);
-
-    // Wait until child has assigned the cgroup.
-    ASSERT_LT(0, ::read(pipes[0], &dummy, sizeof(dummy)));
-    ::close(pipes[0]);
-
-    // Freeze the test cgroup.
-    Future<bool> freeze = cgroups::freeze(hierarchy, TEST_CGROUPS_ROOT);
-    freeze.await(Seconds(5));
-    ASSERT_TRUE(freeze.isReady());
-    EXPECT_EQ(true, freeze.get());
-
-    // Thaw the test cgroup.
-    Future<bool> thaw = cgroups::thaw(hierarchy, TEST_CGROUPS_ROOT);
-    thaw.await(Seconds(5));
-    ASSERT_TRUE(thaw.isReady());
-    EXPECT_EQ(true, thaw.get());
-
-    // Kill the child process.
-    ASSERT_NE(-1, ::kill(pid, SIGKILL));
-
-    // Wait for the child process.
-    int status;
-    EXPECT_NE(-1, ::waitpid((pid_t) -1, &status, 0));
-    ASSERT_TRUE(WIFSIGNALED(status));
-    EXPECT_EQ(SIGKILL, WTERMSIG(status));
-  } else {
+  if (pid == 0) {
     // In child process.
     ::close(pipes[0]);
 
@@ -657,6 +629,41 @@ TEST_F(CgroupsAnyHierarchyWithFreezerTest, ROOT_CGROUPS_Freeze)
     std::cerr << "Reach an unreachable statement!" << std::endl;
     abort();
   }
+
+  // In parent process.
+  ::close(pipes[1]);
+
+  // Wait until child has assigned the cgroup.
+  ASSERT_LT(0, ::read(pipes[0], &dummy, sizeof(dummy)));
+  ::close(pipes[0]);
+
+  // Freeze the test cgroup.
+  AWAIT_EXPECT_READY(cgroups::freezer::freeze(hierarchy, TEST_CGROUPS_ROOT));
+
+  // Thaw the test cgroup.
+  AWAIT_EXPECT_READY(cgroups::freezer::thaw(hierarchy, TEST_CGROUPS_ROOT));
+
+  // Kill the child process.
+  ASSERT_NE(-1, ::kill(pid, SIGKILL));
+
+  // Wait for the child process.
+  int status;
+  EXPECT_NE(-1, ::waitpid((pid_t) -1, &status, 0));
+  ASSERT_TRUE(WIFSIGNALED(status));
+  EXPECT_EQ(SIGKILL, WTERMSIG(status));
+}
+
+
+TEST_F(CgroupsAnyHierarchyWithCpuMemoryTest, ROOT_CGROUPS_FreezeNonFreezer)
+{
+  std::string hierarchy = path::join(baseHierarchy, "cpu");
+  ASSERT_SOME(cgroups::create(hierarchy, TEST_CGROUPS_ROOT));
+
+  AWAIT_EXPECT_FAILED(cgroups::freezer::freeze(hierarchy, TEST_CGROUPS_ROOT));
+  AWAIT_EXPECT_FAILED(cgroups::freezer::thaw(hierarchy, TEST_CGROUPS_ROOT));
+
+  // The cgroup is empty so we should still be able to destroy it.
+  AWAIT_READY(cgroups::destroy(hierarchy, TEST_CGROUPS_ROOT));
 }