You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by id...@apache.org on 2014/06/17 20:04:56 UTC
[3/3] git commit: Refactor cgroups::internal::Freezer.
Refactor cgroups::internal::Freezer.
The Freezer tries to converge to the "FROZEN" state by repeatedly (every
100 ms) writing "FROZEN" to the freezer.state control file. It assumes
there are two possible reasons why a process does not get frozen during
an attempt:
1. It was in the middle of being forked and did not receive the signal;
it will receive it at the next attempt.
2. It is in uninterruptable sleep ("D" state). Normally, this is from
device I/O or paging and is shortlived, in which case it'll be frozen on
retry. However, processes can get stuck in "D" state, either because of
a device issue, incorrect OOM handling, or kernel bugs. Under this
scenario the correct behavior is to fail after a timeout (defaults to 60
seconds).
Freezer functions have been namespaced under cgroups::freezer.
Review: https://reviews.apache.org/r/20817
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/07c3dc57
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/07c3dc57
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/07c3dc57
Branch: refs/heads/master
Commit: 07c3dc57fa272a9c25b5df2f7ac218fdcfb2f5d4
Parents: fb13868
Author: Ian Downes <id...@twitter.com>
Authored: Wed Apr 23 10:48:33 2014 -0700
Committer: Ian Downes <id...@twitter.com>
Committed: Tue Jun 17 11:00:41 2014 -0700
----------------------------------------------------------------------
src/linux/cgroups.cpp | 350 +++++++++++++++------------------------
src/linux/cgroups.hpp | 72 ++++----
src/tests/cgroups_tests.cpp | 65 ++++----
3 files changed, 195 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/linux/cgroups.cpp
----------------------------------------------------------------------
diff --git a/src/linux/cgroups.cpp b/src/linux/cgroups.cpp
index 142ac43..723acc1 100644
--- a/src/linux/cgroups.cpp
+++ b/src/linux/cgroups.cpp
@@ -1296,270 +1296,144 @@ Future<uint64_t> listen(
return future;
}
-
namespace internal {
+namespace freezer {
-// The process that freezes or thaws the cgroup.
-class Freezer : public Process<Freezer>
+Try<string> state(const string& hierarchy, const string& cgroup)
{
-public:
- Freezer(const string& _hierarchy,
- const string& _cgroup,
- const string& _action,
- const Duration& _interval,
- unsigned int _retries = FREEZE_RETRIES)
- : hierarchy(_hierarchy),
- cgroup(_cgroup),
- action(_action),
- interval(_interval),
- retries(_retries) {}
-
- virtual ~Freezer() {}
+ Try<string> state = cgroups::read(hierarchy, cgroup, "freezer.state");
- // Return a future indicating the state of the freezer.
- Future<bool> future() { return promise.future(); }
+ if (state.isError()) {
+ return Error("Failed to read freezer state: " + state.error());
+ }
-protected:
- virtual void initialize()
- {
- // Stop the process if no one cares.
- promise.future().onDiscard(lambda::bind(
- static_cast<void (*)(const UPID&, bool)>(terminate), self(), true));
+ return strings::trim(state.get());
+}
- CHECK(interval >= Seconds(0));
- // Start the action.
- CHECK(action == "FREEZE" || action == "THAW");
- if (action == "FREEZE") {
- freeze();
- } else if (action == "THAW") {
- thaw();
- }
+Try<Nothing> state(
+ const string& hierarchy,
+ const string& cgroup,
+ const string& state)
+{
+ if (state != "FROZEN" && state != "THAWED") {
+ return Error("Invalid freezer state requested: " + state);
}
- virtual void finalize()
- {
- promise.discard();
+ Try<Nothing> write = cgroups::write(
+ hierarchy, cgroup, "freezer.state", state);
+ if (write.isError()) {
+ return Error("Failed to write '" + state +
+ "' to control 'freezer.state': " + write.error());
+ } else {
+ return Nothing();
}
+}
-private:
- void freeze()
- {
- LOG(INFO) << "Trying to freeze cgroup " << path::join(hierarchy, cgroup);
+} // namespace freezer {
- Try<Nothing> write = internal::write(
- hierarchy, cgroup, "freezer.state", "FROZEN");
+class Freezer : public Process<Freezer>
+{
+public:
+ Freezer(
+ const string& _hierarchy,
+ const string& _cgroup)
+ : hierarchy(_hierarchy),
+ cgroup(_cgroup),
+ start(Clock::now()) {}
- if (write.isError()) {
- promise.fail("Failed to write control 'freezer.state': " + write.error());
- terminate(self());
- } else {
- watchFrozen();
- }
- }
+ virtual ~Freezer() {}
- void thaw()
+ void freeze()
{
- LOG(INFO) << "Trying to thaw cgroup " << path::join(hierarchy, cgroup);
-
- Try<Nothing> write = internal::write(
- hierarchy, cgroup, "freezer.state", "THAWED");
-
- if (write.isError()) {
- promise.fail("Failed to write control 'freezer.state': " + write.error());
+ Try<Nothing> freeze =
+ internal::freezer::state(hierarchy, cgroup, "FROZEN");
+ if (freeze.isError()) {
+ promise.fail(freeze.error());
terminate(self());
- } else {
- watchThawed();
+ return;
}
- }
-
- void watchFrozen(unsigned int attempt = 0)
- {
- Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
+ Try<string> state = internal::freezer::state(hierarchy, cgroup);
if (state.isError()) {
- promise.fail("Failed to read control 'freezer.state': " + state.error());
+ promise.fail(state.error());
terminate(self());
return;
}
- if (strings::trim(state.get()) == "FROZEN") {
- LOG(INFO) << "Successfully froze cgroup " << path::join(hierarchy, cgroup)
- << " after " << attempt + 1 << " attempts";
- promise.set(true);
+ if (state.get() == "FROZEN") {
+ LOG(INFO) << "Successfullly froze cgroup "
+ << path::join(hierarchy, cgroup)
+ << " after " << (Clock::now() - start);
+ promise.set(Nothing());
terminate(self());
return;
- } else if (strings::trim(state.get()) == "FREEZING") {
- // The freezer.state is in FREEZING state. This is because not all the
- // processes in the given cgroup can be frozen at the moment. The main
- // cause is that some of the processes are in stopped/traced state ('T'
- // state shown in ps command). It is likely that the freezer.state keeps
- // in FREEZING state if these stopped/traced processes are not resumed.
- // Therefore, here we send SIGCONT to those stopped/traced processes to
- // make sure that the freezer can finish.
- // TODO(jieyu): This code can be removed in the future as the newer
- // version of the kernel solves this problem (e.g. Linux-3.2.0).
- Try<set<pid_t> > pids = processes(hierarchy, cgroup);
- if (pids.isError()) {
- promise.fail("Failed to get processes of cgroup: " + pids.error());
- terminate(self());
- return;
- }
-
- // It appears possible for processes to go away while the cgroup
- // is in the FREEZING state. We ignore such processes.
- // See: https://issues.apache.org/jira/browse/MESOS-461
- foreach (pid_t pid, pids.get()) {
- Result<proc::ProcessStatus> status = proc::status(pid);
-
- if (!status.isSome()) {
- LOG(WARNING)
- << "Failed to get process status for pid " << pid << ": "
- << (status.isError() ? status.error() : "pid does not exist");
- continue;
- }
-
- // Check whether the process is in stopped/traced state.
- if (status.get().state == 'T') {
- // Send a SIGCONT signal to the process.
- if (::kill(pid, SIGCONT) == -1) {
- promise.fail(
- "Failed to send SIGCONT to process " + stringify(pid) +
- ": " + string(strerror(errno)));
- terminate(self());
- return;
- }
- }
- }
-
- if (attempt > retries) {
- LOG(WARNING) << "Unable to freeze " << path::join(hierarchy, cgroup)
- << " within " << retries + 1 << " attempts";
- promise.set(false);
- terminate(self());
- return;
- }
-
- // Retry the freezing operation.
- Try<Nothing> write = internal::write(
- hierarchy, cgroup, "freezer.state", "FROZEN");
-
- if (write.isError()) {
- promise.fail(
- "Failed to write control 'freezer.state': " + write.error());
- terminate(self());
- return;
- }
-
- // Not done yet, keep watching (and possibly retrying).
- delay(interval, self(), &Freezer::watchFrozen, attempt + 1);
- } else {
- LOG(FATAL) << "Unexpected state: " << strings::trim(state.get())
- << " of cgroup " << path::join(hierarchy, cgroup);
}
+
+ // Attempt to freeze the freezer cgroup again.
+ delay(Milliseconds(100), self(), &Self::freeze);
}
- void watchThawed()
+ void thaw()
{
- Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
+ Try<Nothing> thaw = internal::freezer::state(hierarchy, cgroup, "THAWED");
+ if (thaw.isError()) {
+ promise.fail(thaw.error());
+ terminate(self());
+ return;
+ }
+ Try<string> state = internal::freezer::state(hierarchy, cgroup);
if (state.isError()) {
- promise.fail("Failed to read control 'freezer.state': " + state.error());
+ promise.fail(state.error());
terminate(self());
return;
}
- if (strings::trim(state.get()) == "THAWED") {
- LOG(INFO) << "Successfully thawed " << path::join(hierarchy, cgroup);
- promise.set(true);
+ if (state.get() == "THAWED") {
+ LOG(INFO) << "Successfullly thawed cgroup "
+ << path::join(hierarchy, cgroup)
+ << " after " << (Clock::now() - start);
+ promise.set(Nothing());
terminate(self());
- } else if (strings::trim(state.get()) == "FROZEN") {
- // Not done yet, keep watching.
- delay(interval, self(), &Freezer::watchThawed);
- } else {
- LOG(FATAL) << "Unexpected state: " << strings::trim(state.get())
- << " of cgroup " << path::join(hierarchy, cgroup);
+ return;
}
- }
- const string hierarchy;
- const string cgroup;
- const string action;
- const Duration interval;
- const unsigned int retries;
- Promise<bool> promise;
-};
-
-} // namespace internal {
-
-
-Future<bool> freeze(
- const string& hierarchy,
- const string& cgroup,
- const Duration& interval,
- unsigned int retries)
-{
- Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
- if (error.isSome()) {
- return Failure(error.get());
+ // Attempt to thaw the freezer cgroup again.
+ delay(Milliseconds(100), self(), &Self::thaw);
}
- if (interval < Seconds(0)) {
- return Failure("Interval should be non-negative");
- }
-
- // Check the current freezer state.
- Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
- if (state.isError()) {
- return Failure(
- "Failed to read control 'freezer.state': " + state.error());
- } else if (strings::trim(state.get()) == "FROZEN") {
- // Immediately return success.
- return true;
- }
+ Future<Nothing> future() { return promise.future(); }
- internal::Freezer* freezer =
- new internal::Freezer(hierarchy, cgroup, "FREEZE", interval, retries);
- Future<bool> future = freezer->future();
- spawn(freezer, true);
- return future;
-}
-
-
-Future<bool> thaw(
- const string& hierarchy,
- const string& cgroup,
- const Duration& interval)
-{
- Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
- if (error.isSome()) {
- return Failure(error.get());
- }
+protected:
+ virtual void initialize()
+ {
+ Option<Error> error = verify(hierarchy, cgroup, "freezer.state");
+ if (error.isSome()) {
+ promise.fail("Invalid freezer cgroup: " + error.get().message);
+ terminate(self());
+ return;
+ }
- if (interval < Seconds(0)) {
- return Failure("Interval should be non-negative");
+ // Stop attempting to freeze/thaw if nobody cares.
+ promise.future().onDiscard(lambda::bind(
+ static_cast<void(*)(const UPID&, bool)>(terminate), self(), true));
}
- // Check the current freezer state.
- Try<string> state = internal::read(hierarchy, cgroup, "freezer.state");
- if (state.isError()) {
- return Failure(
- "Failed to read control 'freezer.state': " + state.error());
- } else if (strings::trim(state.get()) == "THAWED") {
- // Immediately return success.
- return true;
+ virtual void finalize()
+ {
+ promise.discard();
}
- internal::Freezer* freezer =
- new internal::Freezer(hierarchy, cgroup, "THAW", interval);
- Future<bool> future = freezer->future();
- spawn(freezer, true);
- return future;
-}
+private:
+ const string hierarchy;
+ const string cgroup;
+ const Time start;
+ Promise<Nothing> promise;
+};
-namespace internal {
// The process used to wait for a cgroup to become empty (no task in it).
class EmptyWatcher: public Process<EmptyWatcher>
@@ -1701,9 +1575,9 @@ private:
chain.onAny(defer(self(), &Self::finished, lambda::_1));
}
- Future<bool> freeze()
+ Future<Nothing> freeze()
{
- return cgroups::freeze(hierarchy, cgroup, interval);
+ return cgroups::freezer::freeze(hierarchy, cgroup);
}
Future<Nothing> kill(const int signal)
@@ -1715,9 +1589,9 @@ private:
return Nothing();
}
- Future<bool> thaw()
+ Future<Nothing> thaw()
{
- return cgroups::thaw(hierarchy, cgroup, interval);
+ return cgroups::freezer::thaw(hierarchy, cgroup);
}
Future<bool> empty()
@@ -2114,4 +1988,42 @@ Try<Bytes> max_usage_in_bytes(const string& hierarchy, const string& cgroup)
} // namespace memory {
+
+namespace freezer {
+
+Future<Nothing> freeze(
+ const string& hierarchy,
+ const string& cgroup)
+{
+ LOG(INFO) << "Freezing cgroup " << path::join(hierarchy, cgroup);
+
+ internal::Freezer* freezer = new internal::Freezer(hierarchy, cgroup);
+
+ Future<Nothing> future = freezer->future();
+ spawn(freezer, true);
+
+ dispatch(freezer, &internal::Freezer::freeze);
+
+ return future;
+}
+
+
+Future<Nothing> thaw(
+ const string& hierarchy,
+ const string& cgroup)
+{
+ LOG(INFO) << "Thawing cgroup " << path::join(hierarchy, cgroup);
+
+ internal::Freezer* freezer = new internal::Freezer(hierarchy, cgroup);
+
+ Future<Nothing> future = freezer->future();
+ spawn(freezer, true);
+
+ dispatch(freezer, &internal::Freezer::thaw);
+
+ return future;
+}
+
+} // namespace freezer {
+
} // namespace cgroups {
http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/linux/cgroups.hpp
----------------------------------------------------------------------
diff --git a/src/linux/cgroups.hpp b/src/linux/cgroups.hpp
index 21d87a0..599badb 100644
--- a/src/linux/cgroups.hpp
+++ b/src/linux/cgroups.hpp
@@ -29,6 +29,7 @@
#include <sys/types.h>
#include <process/future.hpp>
+#include <process/timeout.hpp>
#include <stout/bytes.hpp>
#include <stout/duration.hpp>
@@ -39,8 +40,7 @@
namespace cgroups {
-// Default number of retry attempts when trying to freeze a cgroup.
-const unsigned int FREEZE_RETRIES = 50;
+const Duration DESTROY_TIMEOUT = Seconds(60);
const unsigned int EMPTY_WATCHER_RETRIES = 50;
// Default number of assign attempts when moving threads to a cgroup.
@@ -321,48 +321,6 @@ process::Future<uint64_t> listen(
const Option<std::string>& args = Option<std::string>::none());
-// Freeze all the processes in a given cgroup. We try to use the freezer
-// subsystem implemented in cgroups. More detail can be found in
-// <kernel-source>/Documentation/cgroups/freezer-subsystem.txt. This function
-// will return a future which will become ready when all the processes have been
-// frozen (FROZEN). The future can be discarded to cancel the operation. The
-// freezer state after the cancellation is not defined. So the users need to
-// read the control file if they need to know the freezer state after the
-// cancellation. This function will return future failure if the freezer
-// subsystem is not available or it is not attached to the given hierarchy, or
-// the given cgroup is not valid, or the given cgroup has already been frozen.
-// @param hierarchy Path to the hierarchy root.
-// @param cgroup Path to the cgroup relative to the hierarchy root.
-// @param interval The time interval between two state check
-// requests (default: 0.1 seconds).
-// @param retries Number of retry attempts before giving up. None
-// indicates infinite retries. (default: 50 attempts).
-// @return A future which will become true when all processes are frozen, or
-// false when all retries have occurred unsuccessfully.
-// Error if something unexpected happens.
-process::Future<bool> freeze(
- const std::string& hierarchy,
- const std::string& cgroup,
- const Duration& interval = Milliseconds(100),
- const unsigned int retries = FREEZE_RETRIES);
-
-
-// Thaw the given cgroup. This is a revert operation of freezeCgroup. It will
-// return error if the given cgroup is already thawed. Same as
-// freezeCgroup, this function will return a future which can be discarded to
-// allow users to cancel the operation.
-// @param hierarchy Path to the hierarchy root.
-// @param cgroup Path to the cgroup relative to the hierarchy root.
-// @param interval The time interval between two state check
-// requests (default: 0.1 seconds).
-// @return A future which will become ready when all processes are thawed.
-// Error if something unexpected happens.
-process::Future<bool> thaw(
- const std::string& hierarchy,
- const std::string& cgroup,
- const Duration& interval = Milliseconds(100));
-
-
// Destroy a cgroup under a given hierarchy. It will also recursively
// destroy any sub-cgroups. If the freezer subsystem is attached to
// the hierarchy, we attempt to kill all tasks in a given cgroup,
@@ -482,6 +440,32 @@ Try<Bytes> max_usage_in_bytes(
} // namespace memory {
+
+// Freezer controls.
+// The freezer can be in one of three states:
+// 1. THAWED : No process in the cgroup is frozen.
+// 2. FREEZING : Freezing is in progress but not all processes are frozen.
+// 3. FROZEN : All processes are frozen.
+namespace freezer {
+
+// Freeze all processes in the given cgroup. The cgroup must be in a freezer
+// hierarchy. This function will return a future which will become ready when
+// all processes have been frozen (cgroup is in the FROZEN state).
+process::Future<Nothing> freeze(
+ const std::string& hierarchy,
+ const std::string& cgroup);
+
+
+// Thaw all processes in the given cgroup. The cgroup must be in a freezer
+// hierarchy. This is a revert operation of freezer::freeze. This function will
+// return a future which will become ready when all processes have been thawed
+// (cgroup is in the THAWED state).
+process::Future<Nothing> thaw(
+ const std::string& hierarchy,
+ const std::string& cgroup);
+
+} // namespace freezer {
+
} // namespace cgroups {
#endif // __CGROUPS_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/07c3dc57/src/tests/cgroups_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/cgroups_tests.cpp b/src/tests/cgroups_tests.cpp
index adcf46b..c00c30a 100644
--- a/src/tests/cgroups_tests.cpp
+++ b/src/tests/cgroups_tests.cpp
@@ -602,35 +602,7 @@ TEST_F(CgroupsAnyHierarchyWithFreezerTest, ROOT_CGROUPS_Freeze)
pid_t pid = ::fork();
ASSERT_NE(-1, pid);
- if (pid > 0) {
- // In parent process.
- ::close(pipes[1]);
-
- // Wait until child has assigned the cgroup.
- ASSERT_LT(0, ::read(pipes[0], &dummy, sizeof(dummy)));
- ::close(pipes[0]);
-
- // Freeze the test cgroup.
- Future<bool> freeze = cgroups::freeze(hierarchy, TEST_CGROUPS_ROOT);
- freeze.await(Seconds(5));
- ASSERT_TRUE(freeze.isReady());
- EXPECT_EQ(true, freeze.get());
-
- // Thaw the test cgroup.
- Future<bool> thaw = cgroups::thaw(hierarchy, TEST_CGROUPS_ROOT);
- thaw.await(Seconds(5));
- ASSERT_TRUE(thaw.isReady());
- EXPECT_EQ(true, thaw.get());
-
- // Kill the child process.
- ASSERT_NE(-1, ::kill(pid, SIGKILL));
-
- // Wait for the child process.
- int status;
- EXPECT_NE(-1, ::waitpid((pid_t) -1, &status, 0));
- ASSERT_TRUE(WIFSIGNALED(status));
- EXPECT_EQ(SIGKILL, WTERMSIG(status));
- } else {
+ if (pid == 0) {
// In child process.
::close(pipes[0]);
@@ -657,6 +629,41 @@ TEST_F(CgroupsAnyHierarchyWithFreezerTest, ROOT_CGROUPS_Freeze)
std::cerr << "Reach an unreachable statement!" << std::endl;
abort();
}
+
+ // In parent process.
+ ::close(pipes[1]);
+
+ // Wait until child has assigned the cgroup.
+ ASSERT_LT(0, ::read(pipes[0], &dummy, sizeof(dummy)));
+ ::close(pipes[0]);
+
+ // Freeze the test cgroup.
+ AWAIT_EXPECT_READY(cgroups::freezer::freeze(hierarchy, TEST_CGROUPS_ROOT));
+
+ // Thaw the test cgroup.
+ AWAIT_EXPECT_READY(cgroups::freezer::thaw(hierarchy, TEST_CGROUPS_ROOT));
+
+ // Kill the child process.
+ ASSERT_NE(-1, ::kill(pid, SIGKILL));
+
+ // Wait for the child process.
+ int status;
+ EXPECT_NE(-1, ::waitpid((pid_t) -1, &status, 0));
+ ASSERT_TRUE(WIFSIGNALED(status));
+ EXPECT_EQ(SIGKILL, WTERMSIG(status));
+}
+
+
+TEST_F(CgroupsAnyHierarchyWithCpuMemoryTest, ROOT_CGROUPS_FreezeNonFreezer)
+{
+ std::string hierarchy = path::join(baseHierarchy, "cpu");
+ ASSERT_SOME(cgroups::create(hierarchy, TEST_CGROUPS_ROOT));
+
+ AWAIT_EXPECT_FAILED(cgroups::freezer::freeze(hierarchy, TEST_CGROUPS_ROOT));
+ AWAIT_EXPECT_FAILED(cgroups::freezer::thaw(hierarchy, TEST_CGROUPS_ROOT));
+
+ // The cgroup is empty so we should still be able to destroy it.
+ AWAIT_READY(cgroups::destroy(hierarchy, TEST_CGROUPS_ROOT));
}