You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2017/11/15 08:13:28 UTC
[04/15] mesos git commit: Added recovery logic for standalone
containers.
Added recovery logic for standalone containers.
Although there is no way to launch standalone containers yet,
this commit outlines the expected layout of container metadata
which should be populated when launching standalone containers.
The layout is fairly simple, as standalone containers have no
framework, executor, or tasks to worry about. The sandbox directory
will live under a new top-level directory `containers` and there is
no metadata to checkpoint at the moment.
The containerizer will checkpoint a marker file (in the runtime
directory) so that it knows to recover all standalone containers.
Review: https://reviews.apache.org/r/60888
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/14145780
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/14145780
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/14145780
Branch: refs/heads/master
Commit: 14145780bd056e973bee674a4fa6a519336fcf6a
Parents: 69a7551
Author: Joseph Wu <jo...@apache.org>
Authored: Mon Jun 5 18:21:22 2017 -0700
Committer: Joseph Wu <jo...@apache.org>
Committed: Tue Nov 14 16:58:40 2017 -0800
----------------------------------------------------------------------
src/slave/containerizer/mesos/containerizer.cpp | 59 +++++++++++++++-----
src/slave/containerizer/mesos/paths.cpp | 20 +++++++
src/slave/containerizer/mesos/paths.hpp | 16 ++++++
src/slave/paths.cpp | 16 ++++++
src/slave/paths.hpp | 11 ++++
5 files changed, 109 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/14145780/src/slave/containerizer/mesos/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp
index 100e3bb..c5ca76e 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -661,9 +661,10 @@ Future<Nothing> MesosContainerizerProcess::recover(
{
LOG(INFO) << "Recovering containerizer";
- // Gather the executor run states that we will attempt to recover.
+ // Gather the container states that we will attempt to recover.
list<ContainerState> recoverable;
if (state.isSome()) {
+ // Gather the latest run of checkpointed executors.
foreachvalue (const FrameworkState& framework, state.get().frameworks) {
foreachvalue (const ExecutorState& executor, framework.executors) {
if (executor.info.isNone()) {
@@ -743,11 +744,9 @@ Future<Nothing> MesosContainerizerProcess::recover(
}
}
- // Recover the executor containers from 'SlaveState'.
- hashset<ContainerID> alive;
+ // Recover the containers from 'SlaveState'.
foreach (const ContainerState& state, recoverable) {
const ContainerID& containerId = state.container_id();
- alive.insert(containerId);
// Contruct the structure for containers from the 'SlaveState'
// first, to maintain the children list in the container.
@@ -767,6 +766,13 @@ Future<Nothing> MesosContainerizerProcess::recover(
hashset<ContainerID> orphans;
// Recover the containers from the runtime directory.
+ //
+ // NOTE: The returned vector guarantees that parent containers
+ // will always appear before their child containers (if any).
+ // This is particularly important for containers nested underneath
+ // standalone containers, because standalone containers are only
+ // added to the list of recoverable containers in the following loop,
+ // whereas normal parent containers are added in the prior loop.
Try<vector<ContainerID>> containerIds =
containerizer::paths::getContainerIds(flags.runtime_dir);
@@ -781,7 +787,7 @@ Future<Nothing> MesosContainerizerProcess::recover(
// that we aggregate with any orphans that get returned from
// calling `launcher->recover`.
foreach (const ContainerID& containerId, containerIds.get()) {
- if (alive.contains(containerId)) {
+ if (containers_.contains(containerId)) {
continue;
}
@@ -806,7 +812,11 @@ Future<Nothing> MesosContainerizerProcess::recover(
return Failure("Failed to get container pid: " + pid.error());
}
- // Determine the sandbox if this is a nested container.
+ // Determine the sandbox if this is a nested or standalone container.
+ const bool isStandaloneContainer =
+ containerizer::paths::isStandaloneContainer(
+ flags.runtime_dir, containerId);
+
Option<string> directory;
if (containerId.has_parent()) {
const ContainerID& rootContainerId =
@@ -818,6 +828,8 @@ Future<Nothing> MesosContainerizerProcess::recover(
containers_[rootContainerId]->directory.get(),
containerId);
}
+ } else if (isStandaloneContainer) {
+ directory = slave::paths::getContainerPath(flags.work_dir, containerId);
}
Owned<Container> container(new Container());
@@ -838,19 +850,22 @@ Future<Nothing> MesosContainerizerProcess::recover(
containers_[containerId] = container;
- // Add recoverable nested containers to the list of 'ContainerState'.
- //
// TODO(klueska): The final check in the if statement makes sure
// that this container was not marked for forcible destruction on
// recover. We currently only support 'destroy-on-recovery'
// semantics for nested `DEBUG` containers. If we ever support it
// on other types of containers, we may need duplicate this logic
// elsewhere.
- if (containerId.has_parent() &&
- alive.contains(protobuf::getRootContainerId(containerId)) &&
- pid.isSome() &&
- !containerizer::paths::getContainerForceDestroyOnRecovery(
- flags.runtime_dir, containerId)) {
+ const bool isRecoverableNestedContainer =
+ containerId.has_parent() &&
+ (containers_.contains(protobuf::getRootContainerId(containerId))) &&
+ pid.isSome() &&
+ !containerizer::paths::getContainerForceDestroyOnRecovery(
+ flags.runtime_dir, containerId);
+
+ // Add recoverable nested containers or standalone containers
+ // to the list of 'ContainerState'.
+ if (isRecoverableNestedContainer || isStandaloneContainer) {
CHECK_SOME(directory);
ContainerState state =
protobuf::slave::createContainerState(
@@ -1168,6 +1183,24 @@ Future<bool> MesosContainerizerProcess::launch(
}
}
+ // If we are launching a standalone container, checkpoint a file to
+ // mark it as a standalone container. Nested containers launched
+ // under a standalone container are treated as nested containers
+ // (_not_ as both standalone and nested containers).
+ if (!containerId.has_parent() &&
+ !containerConfig.has_task_info() &&
+ !containerConfig.has_executor_info()) {
+ const string path =
+ containerizer::paths::getStandaloneContainerMarkerPath(
+ flags.runtime_dir, containerId);
+
+ Try<Nothing> checkpointed = slave::state::checkpoint(path, "");
+ if (checkpointed.isError()) {
+ return Failure(
+ "Failed to checkpoint file to mark container as standalone");
+ }
+ }
+
Owned<Container> container(new Container());
container->state = PROVISIONING;
container->config = containerConfig;
http://git-wip-us.apache.org/repos/asf/mesos/blob/14145780/src/slave/containerizer/mesos/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/paths.cpp b/src/slave/containerizer/mesos/paths.cpp
index 0c61c20..23f1fee 100644
--- a/src/slave/containerizer/mesos/paths.cpp
+++ b/src/slave/containerizer/mesos/paths.cpp
@@ -284,6 +284,26 @@ Result<ContainerTermination> getContainerTermination(
}
+string getStandaloneContainerMarkerPath(
+ const string& runtimeDir,
+ const ContainerID& containerId)
+{
+ return path::join(
+ getRuntimePath(runtimeDir, containerId),
+ STANDALONE_MARKER_FILE);
+}
+
+
+bool isStandaloneContainer(
+ const string& runtimeDir,
+ const ContainerID& containerId)
+{
+ const string path = getStandaloneContainerMarkerPath(runtimeDir, containerId);
+
+ return os::exists(path);
+}
+
+
Try<vector<ContainerID>> getContainerIds(const string& runtimeDir)
{
lambda::function<Try<vector<ContainerID>>(const Option<ContainerID>&)> helper;
http://git-wip-us.apache.org/repos/asf/mesos/blob/14145780/src/slave/containerizer/mesos/paths.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/paths.hpp b/src/slave/containerizer/mesos/paths.hpp
index a03f15e..7b67ccf 100644
--- a/src/slave/containerizer/mesos/paths.hpp
+++ b/src/slave/containerizer/mesos/paths.hpp
@@ -59,6 +59,7 @@ namespace paths {
// | |-- socket
// |-- launch_info
// |-- pid
+// |-- standalone.marker
// |-- status
// |-- termination
@@ -71,6 +72,7 @@ constexpr char FORCE_DESTROY_ON_RECOVERY_FILE[] = "force_destroy_on_recovery";
constexpr char IO_SWITCHBOARD_DIRECTORY[] = "io_switchboard";
constexpr char CONTAINER_DIRECTORY[] = "containers";
constexpr char CONTAINER_LAUNCH_INFO_FILE[] = "launch_info";
+constexpr char STANDALONE_MARKER_FILE[] = "standalone.marker";
enum Mode
@@ -166,6 +168,20 @@ Result<mesos::slave::ContainerTermination> getContainerTermination(
const ContainerID& containerId);
+// The helper method to get the standalone container marker path.
+std::string getStandaloneContainerMarkerPath(
+ const std::string& runtimeDir,
+ const ContainerID& containerId);
+
+
+// The helper method to check if the given container is a standalone
+// container or not. This is determined by the existence (or not) of
+// a marker file in the container's runtime metadata directory.
+bool isStandaloneContainer(
+ const std::string& runtimeDir,
+ const ContainerID& containerId);
+
+
// The helper method to list all container IDs (including nested
// containers) from the container runtime directory. The order of
// returned vector is a result of pre-ordering walk (i.e., parent
http://git-wip-us.apache.org/repos/asf/mesos/blob/14145780/src/slave/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/paths.cpp b/src/slave/paths.cpp
index fd54652..b03ffee 100644
--- a/src/slave/paths.cpp
+++ b/src/slave/paths.cpp
@@ -66,6 +66,7 @@ const char RESOURCES_INFO_FILE[] = "resources.info";
const char RESOURCES_TARGET_FILE[] = "resources.target";
+const char CONTAINERS_DIR[] = "containers";
const char SLAVES_DIR[] = "slaves";
const char FRAMEWORKS_DIR[] = "frameworks";
const char EXECUTORS_DIR[] = "executors";
@@ -157,6 +158,21 @@ string getSlavePath(
}
+Try<list<string>> getContainerPaths(
+ const string& rootDir)
+{
+ return fs::list(path::join(rootDir, CONTAINERS_DIR, "*"));
+}
+
+
+string getContainerPath(
+ const string& rootDir,
+ const ContainerID& containerId)
+{
+ return path::join(rootDir, CONTAINERS_DIR, stringify(containerId));
+}
+
+
string getSlaveInfoPath(
const string& rootDir,
const SlaveID& slaveId)
http://git-wip-us.apache.org/repos/asf/mesos/blob/14145780/src/slave/paths.hpp
----------------------------------------------------------------------
diff --git a/src/slave/paths.hpp b/src/slave/paths.hpp
index f000508..66dfa45 100644
--- a/src/slave/paths.hpp
+++ b/src/slave/paths.hpp
@@ -50,6 +50,8 @@ namespace paths {
// The file system layout is as follows:
//
// root ('--work_dir' flag)
+// |-- containers
+// | |-- <container_id> (sandbox)
// |-- slaves
// | |-- latest (symlink)
// | |-- <slave_id>
@@ -137,6 +139,15 @@ std::string getSlavePath(
const SlaveID& slaveId);
+Try<std::list<std::string>> getContainerPaths(
+ const std::string& rootDir);
+
+
+std::string getContainerPath(
+ const std::string& rootDir,
+ const ContainerID& containerId);
+
+
Try<std::list<std::string>> getFrameworkPaths(
const std::string& rootDir,
const SlaveID& slaveId);