You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2015/11/12 00:16:28 UTC
mesos git commit: Relocated launcher and linux_launcher as
MesosContainerizer specific.
Repository: mesos
Updated Branches:
refs/heads/master c23cfaa61 -> a6fbf34b9
Relocated launcher and linux_launcher as MesosContainerizer specific.
Review: https://reviews.apache.org/r/39695
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/a6fbf34b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/a6fbf34b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/a6fbf34b
Branch: refs/heads/master
Commit: a6fbf34b997251a48a7d8e79b9310d7d7d28a1cb
Parents: c23cfaa
Author: Gilbert Song <so...@gmail.com>
Authored: Wed Nov 11 15:15:56 2015 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Nov 11 15:15:58 2015 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 6 +-
src/Makefile.am | 8 +-
src/slave/containerizer/containerizer.cpp | 8 +-
.../containerizer/external_containerizer.hpp | 3 +-
src/slave/containerizer/launcher.cpp | 180 -------
src/slave/containerizer/launcher.hpp | 123 -----
src/slave/containerizer/linux_launcher.cpp | 501 -------------------
src/slave/containerizer/linux_launcher.hpp | 84 ----
src/slave/containerizer/mesos/containerizer.cpp | 5 +-
src/slave/containerizer/mesos/containerizer.hpp | 3 +-
src/slave/containerizer/mesos/launcher.cpp | 180 +++++++
src/slave/containerizer/mesos/launcher.hpp | 123 +++++
.../containerizer/mesos/linux_launcher.cpp | 501 +++++++++++++++++++
.../containerizer/mesos/linux_launcher.hpp | 84 ++++
.../containerizer/filesystem_isolator_tests.cpp | 2 +-
src/tests/containerizer/isolator_tests.cpp | 5 +-
src/tests/containerizer/launcher.hpp | 2 +-
.../containerizer/mesos_containerizer_tests.cpp | 2 +-
src/tests/containerizer/port_mapping_tests.cpp | 8 +-
19 files changed, 915 insertions(+), 913 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 7be464e..a145188 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -192,12 +192,12 @@ if (NOT WIN32)
slave/containerizer/external_containerizer.cpp
slave/containerizer/fetcher.cpp
slave/containerizer/isolator.cpp
- slave/containerizer/mesos/isolators/filesystem/posix.cpp
- slave/containerizer/mesos/isolators/posix/disk.cpp
- slave/containerizer/launcher.cpp
slave/containerizer/mesos/containerizer.cpp
slave/containerizer/mesos/launch.cpp
+ slave/containerizer/mesos/launcher.cpp
slave/containerizer/mesos/mount.cpp
+ slave/containerizer/mesos/isolators/filesystem/posix.cpp
+ slave/containerizer/mesos/isolators/posix/disk.cpp
slave/containerizer/mesos/provisioner/paths.cpp
slave/containerizer/mesos/provisioner/provisioner.cpp
slave/containerizer/mesos/provisioner/store.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index ae2740a..d40378f 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -548,9 +548,9 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/external_containerizer.cpp \
slave/containerizer/fetcher.cpp \
slave/containerizer/isolator.cpp \
- slave/containerizer/launcher.cpp \
slave/containerizer/mesos/containerizer.cpp \
slave/containerizer/mesos/launch.cpp \
+ slave/containerizer/mesos/launcher.cpp \
slave/containerizer/mesos/mount.cpp \
slave/containerizer/mesos/isolators/filesystem/posix.cpp \
slave/containerizer/mesos/isolators/posix/disk.cpp \
@@ -744,7 +744,7 @@ if OS_LINUX
libmesos_no_3rdparty_la_SOURCES += linux/fs.cpp
libmesos_no_3rdparty_la_SOURCES += linux/perf.cpp
libmesos_no_3rdparty_la_SOURCES += linux/systemd.cpp
- libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp
+ libmesos_no_3rdparty_la_SOURCES += slave/containerizer/mesos/linux_launcher.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/mesos/isolators/cgroups/cpushare.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/mesos/isolators/cgroups/mem.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/mesos/isolators/cgroups/perf_event.cpp
@@ -865,10 +865,10 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/external_containerizer.hpp \
slave/containerizer/fetcher.hpp \
slave/containerizer/isolator.hpp \
- slave/containerizer/launcher.hpp \
- slave/containerizer/linux_launcher.hpp \
slave/containerizer/mesos/containerizer.hpp \
slave/containerizer/mesos/launch.hpp \
+ slave/containerizer/mesos/launcher.hpp \
+ slave/containerizer/mesos/linux_launcher.hpp \
slave/containerizer/mesos/mount.hpp \
slave/containerizer/mesos/isolators/posix.hpp \
slave/containerizer/mesos/isolators/cgroups/constants.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/containerizer.cpp b/src/slave/containerizer/containerizer.cpp
index 0675336..0ed2631 100644
--- a/src/slave/containerizer/containerizer.cpp
+++ b/src/slave/containerizer/containerizer.cpp
@@ -34,16 +34,16 @@
#include "slave/flags.hpp"
#include "slave/slave.hpp"
-#ifdef __linux__
-#include "slave/containerizer/linux_launcher.hpp"
-#endif // __linux__
#include "slave/containerizer/composing.hpp"
#include "slave/containerizer/containerizer.hpp"
#include "slave/containerizer/docker.hpp"
-#include "slave/containerizer/launcher.hpp"
#include "slave/containerizer/external_containerizer.hpp"
#include "slave/containerizer/mesos/containerizer.hpp"
+#include "slave/containerizer/mesos/launcher.hpp"
+#ifdef __linux__
+#include "slave/containerizer/mesos/linux_launcher.hpp"
+#endif // __linux__
using std::map;
using std::string;
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/external_containerizer.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/external_containerizer.hpp b/src/slave/containerizer/external_containerizer.hpp
index c00cebb..cdb87c1 100644
--- a/src/slave/containerizer/external_containerizer.hpp
+++ b/src/slave/containerizer/external_containerizer.hpp
@@ -34,7 +34,8 @@
#include "slave/state.hpp"
#include "slave/containerizer/containerizer.hpp"
-#include "slave/containerizer/launcher.hpp"
+
+#include "slave/containerizer/mesos/launcher.hpp"
namespace mesos {
namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/launcher.cpp b/src/slave/containerizer/launcher.cpp
deleted file mode 100644
index 668ae80..0000000
--- a/src/slave/containerizer/launcher.cpp
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <mesos/type_utils.hpp>
-
-#include <process/collect.hpp>
-#include <process/delay.hpp>
-#include <process/process.hpp>
-#include <process/reap.hpp>
-
-#include <stout/unreachable.hpp>
-
-#include <stout/os/killtree.hpp>
-
-#include "mesos/resources.hpp"
-
-#include "slave/containerizer/launcher.hpp"
-
-using namespace process;
-
-using std::list;
-using std::map;
-using std::string;
-using std::vector;
-
-using mesos::slave::ContainerState;
-
-namespace mesos {
-namespace internal {
-namespace slave {
-
-
-Try<Launcher*> PosixLauncher::create(const Flags& flags)
-{
- return new PosixLauncher();
-}
-
-
-Future<hashset<ContainerID>> PosixLauncher::recover(
- const list<ContainerState>& states)
-{
- foreach (const ContainerState& state, states) {
- const ContainerID& containerId = state.container_id();
- pid_t pid = state.pid();
-
- if (pids.containsValue(pid)) {
- // This should (almost) never occur. There is the possibility
- // that a new executor is launched with the same pid as one that
- // just exited (highly unlikely) and the slave dies after the
- // new executor is launched but before it hears about the
- // termination of the earlier executor (also unlikely).
- // Regardless, the launcher can't do anything sensible so this
- // is considered an error.
- return Failure("Detected duplicate pid " + stringify(pid) +
- " for container " + stringify(containerId));
- }
-
- pids.put(containerId, pid);
- }
-
- return hashset<ContainerID>();
-}
-
-
-// The setup function in child before the exec.
-static int childSetup(const Option<lambda::function<int()>>& setup)
-{
- // POSIX guarantees a forked child's pid does not match any existing
- // process group id so only a single setsid() is required and the
- // session id will be the pid.
- // TODO(idownes): perror is not listed as async-signal-safe and
- // should be reimplemented safely.
- // TODO(jieyu): Move this logic to the subprocess (i.e.,
- // mesos-containerizer launch).
- if (::setsid() == -1) {
- perror("Failed to put child in a new session");
- _exit(1);
- }
-
- if (setup.isSome()) {
- return setup.get()();
- }
-
- return 0;
-}
-
-
-Try<pid_t> PosixLauncher::fork(
- const ContainerID& containerId,
- const string& path,
- const vector<string>& argv,
- const Subprocess::IO& in,
- const Subprocess::IO& out,
- const Subprocess::IO& err,
- const Option<flags::FlagsBase>& flags,
- const Option<map<string, string>>& environment,
- const Option<lambda::function<int()>>& setup,
- const Option<int>& namespaces)
-{
- if (pids.contains(containerId)) {
- return Error("Process has already been forked for container " +
- stringify(containerId));
- }
-
- Try<Subprocess> child = subprocess(
- path,
- argv,
- in,
- out,
- err,
- flags,
- environment,
- lambda::bind(&childSetup, setup));
-
- if (child.isError()) {
- return Error("Failed to fork a child process: " + child.error());
- }
-
- LOG(INFO) << "Forked child with pid '" << child.get().pid()
- << "' for container '" << containerId << "'";
-
- // Store the pid (session id and process group id).
- pids.put(containerId, child.get().pid());
-
- return child.get().pid();
-}
-
-
-// Forward declaration.
-Future<Nothing> _destroy(const Future<Option<int>>& future);
-
-
-Future<Nothing> PosixLauncher::destroy(const ContainerID& containerId)
-{
- if (!pids.contains(containerId)) {
- return Failure("Unknown container " + containerId.value());
- }
-
- pid_t pid = pids.get(containerId).get();
-
- // Kill all processes in the session and process group.
- Try<list<os::ProcessTree>> trees = os::killtree(pid, SIGKILL, true, true);
-
- pids.erase(containerId);
-
- // The child process may not have been waited on yet so we'll delay
- // completing destroy until we're sure it has been reaped.
- return process::reap(pid)
- .then(lambda::bind(&_destroy, lambda::_1));
-}
-
-
-Future<Nothing> _destroy(const Future<Option<int>>& future)
-{
- if (future.isReady()) {
- return Nothing();
- } else {
- return Failure("Failed to kill all processes: " +
- (future.isFailed() ? future.failure() : "unknown error"));
- }
-}
-
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/launcher.hpp b/src/slave/containerizer/launcher.hpp
deleted file mode 100644
index 8945776..0000000
--- a/src/slave/containerizer/launcher.hpp
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __LAUNCHER_HPP__
-#define __LAUNCHER_HPP__
-
-#include <sys/types.h>
-
-#include <list>
-#include <map>
-#include <string>
-
-#include <mesos/mesos.hpp>
-
-#include <mesos/slave/isolator.hpp>
-
-#include <process/future.hpp>
-#include <process/subprocess.hpp>
-
-#include <stout/flags.hpp>
-#include <stout/hashmap.hpp>
-#include <stout/hashset.hpp>
-#include <stout/lambda.hpp>
-#include <stout/option.hpp>
-#include <stout/try.hpp>
-
-#include "slave/flags.hpp"
-
-namespace mesos {
-namespace internal {
-namespace slave {
-
-class Launcher
-{
-public:
- virtual ~Launcher() {}
-
- // Recover the necessary state for each container listed in state.
- // Return the set of containers that are known to the launcher but
- // not known to the slave (a.k.a. orphans).
- virtual process::Future<hashset<ContainerID>> recover(
- const std::list<mesos::slave::ContainerState>& states) = 0;
-
- // Fork a new process in the containerized context. The child will
- // exec the binary at the given path with the given argv, flags and
- // environment. The I/O of the child will be redirected according to
- // the specified I/O descriptors. The user can provide a 'setup'
- // function which will be invoked in the child process right before
- // the exec. The 'setup' function has to be async signal safe. The
- // parent will return the child's pid if the fork is successful.
- virtual Try<pid_t> fork(
- const ContainerID& containerId,
- const std::string& path,
- const std::vector<std::string>& argv,
- const process::Subprocess::IO& in,
- const process::Subprocess::IO& out,
- const process::Subprocess::IO& err,
- const Option<flags::FlagsBase>& flags,
- const Option<std::map<std::string, std::string>>& environment,
- const Option<lambda::function<int()>>& setup,
- const Option<int>& namespaces) = 0;
-
- // Kill all processes in the containerized context.
- virtual process::Future<Nothing> destroy(const ContainerID& containerId) = 0;
-};
-
-
-// Launcher suitable for any POSIX compliant system. Uses process
-// groups and sessions to track processes in a container. POSIX states
-// that process groups cannot migrate between sessions so all
-// processes for a container will be contained in a session.
-class PosixLauncher : public Launcher
-{
-public:
- static Try<Launcher*> create(const Flags& flags);
-
- virtual ~PosixLauncher() {}
-
- virtual process::Future<hashset<ContainerID>> recover(
- const std::list<mesos::slave::ContainerState>& states);
-
- virtual Try<pid_t> fork(
- const ContainerID& containerId,
- const std::string& path,
- const std::vector<std::string>& argv,
- const process::Subprocess::IO& in,
- const process::Subprocess::IO& out,
- const process::Subprocess::IO& err,
- const Option<flags::FlagsBase>& flags,
- const Option<std::map<std::string, std::string>>& environment,
- const Option<lambda::function<int()>>& setup,
- const Option<int>& namespaces);
-
- virtual process::Future<Nothing> destroy(const ContainerID& containerId);
-
-private:
- PosixLauncher() {}
-
- // The 'pid' is the process id of the first process and also the
- // process group id and session id.
- hashmap<ContainerID, pid_t> pids;
-};
-
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
-
-#endif // __LAUNCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/linux_launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/linux_launcher.cpp b/src/slave/containerizer/linux_launcher.cpp
deleted file mode 100644
index ea76894..0000000
--- a/src/slave/containerizer/linux_launcher.cpp
+++ /dev/null
@@ -1,501 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <sched.h>
-#include <unistd.h>
-
-#include <linux/sched.h>
-
-#include <vector>
-
-#include <process/collect.hpp>
-
-#include <stout/abort.hpp>
-#include <stout/check.hpp>
-#include <stout/hashset.hpp>
-#include <stout/os.hpp>
-#include <stout/path.hpp>
-#include <stout/strings.hpp>
-
-#include "linux/cgroups.hpp"
-#include "linux/ns.hpp"
-#include "linux/systemd.hpp"
-
-#include "mesos/resources.hpp"
-
-#include "slave/containerizer/linux_launcher.hpp"
-
-#include "slave/containerizer/mesos/isolators/namespaces/pid.hpp"
-
-using namespace process;
-
-using std::list;
-using std::map;
-using std::set;
-using std::string;
-using std::vector;
-
-using mesos::slave::ContainerState;
-
-namespace mesos {
-namespace internal {
-namespace slave {
-
-static ContainerID container(const string& cgroup)
-{
- string basename = Path(cgroup).basename();
-
- ContainerID containerId;
- containerId.set_value(basename);
- return containerId;
-}
-
-
-// `_systemdHierarchy` is only set if running on a systemd environment.
-LinuxLauncher::LinuxLauncher(
- const Flags& _flags,
- const string& _freezerHierarchy,
- const Option<string>& _systemdHierarchy)
- : flags(_flags),
- freezerHierarchy(_freezerHierarchy),
- systemdHierarchy(_systemdHierarchy) {}
-
-
-Try<Launcher*> LinuxLauncher::create(const Flags& flags)
-{
- Try<string> freezerHierarchy = cgroups::prepare(
- flags.cgroups_hierarchy,
- "freezer",
- flags.cgroups_root);
-
- if (freezerHierarchy.isError()) {
- return Error(
- "Failed to create Linux launcher: " + freezerHierarchy.error());
- }
-
- // Ensure that no other subsystem is attached to the freezer hierarchy.
- Try<set<string>> subsystems = cgroups::subsystems(freezerHierarchy.get());
- if (subsystems.isError()) {
- return Error(
- "Failed to get the list of attached subsystems for hierarchy " +
- freezerHierarchy.get());
- } else if (subsystems.get().size() != 1) {
- return Error(
- "Unexpected subsystems found attached to the hierarchy " +
- freezerHierarchy.get());
- }
-
- LOG(INFO) << "Using " << freezerHierarchy.get()
- << " as the freezer hierarchy for the Linux launcher";
-
- // On systemd environments we currently migrate executor pids into a separate
- // executor slice. This allows the life-time of the executor to be extended
- // past the life-time of the slave. See MESOS-3352.
- // The LinuxLauncher takes responsibility for creating and starting this
- // slice. It then migrates executor pids into this slice before it "unpauses"
- // the executor. This is the same pattern as the freezer.
-
- // If this is a systemd environment, ensure that the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE` exists and is running.
- // TODO(jmlvanre): Prevent racing between multiple agents for this creation
- // logic.
- if (systemd::exists()) {
- systemd::Flags systemdFlags;
- systemdFlags.runtime_directory = flags.systemd_runtime_directory;
- systemdFlags.cgroups_hierarchy = flags.cgroups_hierarchy;
- Try<Nothing> initialize = systemd::initialize(systemdFlags);
- if (initialize.isError()) {
- return Error("Failed to initialize systemd: " + initialize.error());
- }
-
- // Check whether the `SYSTEMD_MESOS_EXECUTORS_SLICE` already exists. Create
- // it if it does not exist.
- // We explicitly don't modify the file if it exists in case operators want
- // to over-ride the settings for the slice that we provide when we create
- // the `Unit` below.
- const Path path(path::join(
- systemd::runtimeDirectory(),
- SYSTEMD_MESOS_EXECUTORS_SLICE));
-
- if (!systemd::slices::exists(path)) {
- // A simple systemd file to allow us to start a new slice.
- string unit = "[Unit]\nDescription=Mesos Executors Slice\n";
-
- Try<Nothing> create = systemd::slices::create(path, unit);
-
- if (create.isError()) {
- return Error("Failed to create systemd slice '" +
- stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) + "': " +
- create.error());
- }
- }
-
- // Regardless of whether we created the file or it existed already, we
- // `start` the executor slice. It is safe (a no-op) to `start` an already
- // running slice.
- Try<Nothing> start = systemd::slices::start(SYSTEMD_MESOS_EXECUTORS_SLICE);
-
- if (start.isError()) {
- return Error("Failed to start '" +
- stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) +
- "': " + start.error());
- }
-
- // Now the `SYSTEMD_MESOS_EXECUTORS_SLICE` is ready for us to assign any
- // executors. We can verify that our cgroups assignments will work by
- // testing the hierarchy.
- Try<bool> exists = cgroups::exists(
- systemd::hierarchy(),
- SYSTEMD_MESOS_EXECUTORS_SLICE);
-
- if (exists.isError() || !exists.get()) {
- return Error("Failed to locate systemd cgroups hierarchy: " +
- (exists.isError() ? exists.error() : "does not exist"));
- }
- }
-
- return new LinuxLauncher(
- flags,
- freezerHierarchy.get(),
- systemd::exists() ?
- Some(systemd::hierarchy()) :
- Option<std::string>::none());
-}
-
-
-bool LinuxLauncher::available()
-{
- // Make sure:
- // - we run as root
- // - "freezer" subsytem is enabled.
-
- Try<bool> freezer = cgroups::enabled("freezer");
- return ::geteuid() == 0 &&
- freezer.isSome() &&
- freezer.get();
-}
-
-
-Future<hashset<ContainerID>> LinuxLauncher::recover(
- const std::list<ContainerState>& states)
-{
- hashset<string> recovered;
-
- // On systemd environments, capture the pids under the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE` for validation during recovery.
- Result<std::set<pid_t>> mesosExecutorSlicePids = None();
- if (systemdHierarchy.isSome()) {
- mesosExecutorSlicePids =
- cgroups::processes(systemdHierarchy.get(), SYSTEMD_MESOS_EXECUTORS_SLICE);
-
- // If we error out trying to read the pids from the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE` we fail. This is a programmer error as we
- // did not set up the slice correctly.
- if (mesosExecutorSlicePids.isError()) {
- return Failure("Failed to read pids from systemd '" +
- stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) + "'");
- }
- }
-
- foreach (const ContainerState& state, states) {
- const ContainerID& containerId = state.container_id();
- pid_t pid = state.pid();
-
- if (pids.containsValue(pid)) {
- // This should (almost) never occur. There is the possibility
- // that a new executor is launched with the same pid as one that
- // just exited (highly unlikely) and the slave dies after the
- // new executor is launched but before it hears about the
- // termination of the earlier executor (also unlikely).
- // Regardless, the launcher can't do anything sensible so this
- // is considered an error.
- return Failure("Detected duplicate pid " + stringify(pid) +
- " for container " + stringify(containerId));
- }
-
- // Store the pid now because if the freezer cgroup is absent
- // (slave terminated after the cgroup is destroyed but before it
- // was notified) then we'll still need it for the check in
- // destroy() when we clean up.
- pids.put(containerId, pid);
-
- Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
-
- if (!exists.get()) {
- // This may occur if the freezer cgroup was destroyed but the
- // slave dies before noticing this. The containerizer will
- // monitor the container's pid and notice that it has exited,
- // triggering destruction of the container.
- LOG(INFO) << "Couldn't find freezer cgroup for container "
- << containerId << ", assuming already destroyed";
- continue;
- }
-
- // If we are on a systemd environment, check that the pid is still in the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE`. If it is not, warn the operator that
- // resource isolation may be invalidated.
- // TODO(jmlvanre): Add a flag that enforces this matching (i.e. exits if a
- // pid was found in the freezer but not in the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE`. We need to flag to support the upgrade
- // path.
- if (systemdHierarchy.isSome() && mesosExecutorSlicePids.isSome()) {
- if (mesosExecutorSlicePids.get().count(pid) <= 0) {
- LOG(WARNING)
- << "Couldn't find pid '" << pid << "' in '"
- << SYSTEMD_MESOS_EXECUTORS_SLICE << "'. This can lead to lack of"
- << " proper resource isolation";
- }
- }
-
- recovered.insert(cgroup(containerId));
- }
-
- // Return the set of orphan containers.
- Try<vector<string>> cgroups =
- cgroups::get(freezerHierarchy, flags.cgroups_root);
-
- if (cgroups.isError()) {
- return Failure(cgroups.error());
- }
-
- foreach (const string& cgroup, cgroups.get()) {
- if (!recovered.contains(cgroup)) {
- orphans.insert(container(cgroup));
- }
- }
-
- return orphans;
-}
-
-
-static int childSetup(
- int pipes[2],
- const Option<lambda::function<int()>>& setup)
-{
- // In child.
- ::close(pipes[1]);
-
- // Do a blocking read on the pipe until the parent signals us to
- // continue.
- char dummy;
- ssize_t length;
- while ((length = ::read(pipes[0], &dummy, sizeof(dummy))) == -1 &&
- errno == EINTR);
-
- if (length != sizeof(dummy)) {
- ABORT("Failed to synchronize with parent");
- }
-
- ::close(pipes[0]);
-
- // Move to a different session (and new process group) so we're
- // independent from the slave's session (otherwise children will
- // receive SIGHUP if the slave exits).
- // TODO(idownes): perror is not listed as async-signal-safe and
- // should be reimplemented safely.
- // TODO(jieyu): Move this logic to the subprocess (i.e.,
- // mesos-containerizer launch).
- if (::setsid() == -1) {
- perror("Failed to put child in a new session");
- return 1;
- }
-
- if (setup.isSome()) {
- return setup.get()();
- }
-
- return 0;
-}
-
-
-Try<pid_t> LinuxLauncher::fork(
- const ContainerID& containerId,
- const string& path,
- const vector<string>& argv,
- const process::Subprocess::IO& in,
- const process::Subprocess::IO& out,
- const process::Subprocess::IO& err,
- const Option<flags::FlagsBase>& flags,
- const Option<map<string, string>>& environment,
- const Option<lambda::function<int()>>& setup,
- const Option<int>& namespaces)
-{
- // Create a freezer cgroup for this container if necessary.
- Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
- if (exists.isError()) {
- return Error("Failed to check existence of freezer cgroup: " +
- exists.error());
- }
-
- if (!exists.get()) {
- Try<Nothing> created =
- cgroups::create(freezerHierarchy, cgroup(containerId));
-
- if (created.isError()) {
- return Error("Failed to create freezer cgroup: " + created.error());
- }
- }
-
- // Use a pipe to block the child until it's been moved into the
- // freezer cgroup.
- int pipes[2];
-
- // We assume this should not fail under reasonable conditions so we
- // use CHECK.
- CHECK_EQ(0, ::pipe(pipes));
-
- int cloneFlags = namespaces.isSome() ? namespaces.get() : 0;
- cloneFlags |= SIGCHLD; // Specify SIGCHLD as child termination signal.
-
- LOG(INFO) << "Cloning child process with flags = "
- << ns::stringify(cloneFlags);
-
- Try<Subprocess> child = subprocess(
- path,
- argv,
- in,
- out,
- err,
- flags,
- environment,
- lambda::bind(&childSetup, pipes, setup),
- lambda::bind(&os::clone, lambda::_1, cloneFlags));
-
- if (child.isError()) {
- return Error("Failed to clone child process: " + child.error());
- }
-
- // Parent.
- os::close(pipes[0]);
-
- // Move the child into the freezer cgroup. Any grandchildren will
- // also be contained in the cgroup.
- // TODO(jieyu): Move this logic to the subprocess (i.e.,
- // mesos-containerizer launch).
- Try<Nothing> assign = cgroups::assign(
- freezerHierarchy,
- cgroup(containerId),
- child.get().pid());
-
- if (assign.isError()) {
- LOG(ERROR) << "Failed to assign process " << child.get().pid()
- << " of container '" << containerId << "'"
- << " to its freezer cgroup: " << assign.error();
-
- ::kill(child.get().pid(), SIGKILL);
- return Error("Failed to contain process");
- }
-
- // If we are on systemd, then move the child into the
- // `SYSTEMD_MESOS_EXECUTORS_SLICE`. As with the freezer, any grandchildren
- // will also be contained in the slice.
- if (systemdHierarchy.isSome()) {
- Try<Nothing> assign = cgroups::assign(
- systemdHierarchy.get(),
- SYSTEMD_MESOS_EXECUTORS_SLICE,
- child.get().pid());
-
- if (assign.isError()) {
- LOG(ERROR) << "Failed to assign process " << child.get().pid()
- << " of container '" << containerId << "'"
- << " to its systemd executor slice: " << assign.error();
-
- ::kill(child.get().pid(), SIGKILL);
- return Error("Failed to contain process on systemd");
- }
-
- LOG(INFO) << "Assigned child process '" << child.get().pid() << "' to '"
- << SYSTEMD_MESOS_EXECUTORS_SLICE << "'";
- }
-
- // Now that we've contained the child we can signal it to continue
- // by writing to the pipe.
- char dummy;
- ssize_t length;
- while ((length = ::write(pipes[1], &dummy, sizeof(dummy))) == -1 &&
- errno == EINTR);
-
- os::close(pipes[1]);
-
- if (length != sizeof(dummy)) {
- // Ensure the child is killed.
- ::kill(child.get().pid(), SIGKILL);
- return Error("Failed to synchronize child process");
- }
-
- if (!pids.contains(containerId)) {
- pids.put(containerId, child.get().pid());
- }
-
- return child.get().pid();
-}
-
-
-Future<Nothing> LinuxLauncher::destroy(const ContainerID& containerId)
-{
- if (!pids.contains(containerId) && !orphans.contains(containerId)) {
- return Failure("Unknown container");
- }
-
- pids.erase(containerId);
- orphans.erase(containerId);
-
- // Just return if the cgroup was destroyed and the slave didn't receive the
- // notification. See comment in recover().
- Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
- if (exists.isError()) {
- return Failure("Failed to check existence of freezer cgroup: " +
- exists.error());
- }
-
- if (!exists.get()) {
- return Nothing();
- }
-
- Result<ino_t> containerPidNs =
- NamespacesPidIsolatorProcess::getNamespace(containerId);
-
- if (containerPidNs.isSome()) {
- LOG(INFO) << "Using pid namespace to destroy container " << containerId;
-
- return ns::pid::destroy(containerPidNs.get())
- .then(lambda::bind(
- (Future<Nothing>(*)(const string&,
- const string&,
- const Duration&))(&cgroups::destroy),
- freezerHierarchy,
- cgroup(containerId),
- cgroups::DESTROY_TIMEOUT));
- }
-
- // Try to clean up using just the freezer cgroup.
- return cgroups::destroy(
- freezerHierarchy,
- cgroup(containerId),
- cgroups::DESTROY_TIMEOUT);
-}
-
-
-string LinuxLauncher::cgroup(const ContainerID& containerId)
-{
- return path::join(flags.cgroups_root, containerId.value());
-}
-
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/linux_launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/linux_launcher.hpp b/src/slave/containerizer/linux_launcher.hpp
deleted file mode 100644
index 627df89..0000000
--- a/src/slave/containerizer/linux_launcher.hpp
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __LINUX_LAUNCHER_HPP__
-#define __LINUX_LAUNCHER_HPP__
-
-#include "slave/containerizer/launcher.hpp"
-
-namespace mesos {
-namespace internal {
-namespace slave {
-
-// TODO(jmlvanre): We may want to allow this to be configured.
-static const char SYSTEMD_MESOS_EXECUTORS_SLICE[] = "mesos_executors.slice";
-
-// Launcher for Linux systems with cgroups. Uses a freezer cgroup to
-// track pids.
-class LinuxLauncher : public Launcher
-{
-public:
- static Try<Launcher*> create(const Flags& flags);
-
- // Returns 'true' if prerequisites for using LinuxLauncher are available.
- static bool available();
-
- virtual ~LinuxLauncher() {}
-
- virtual process::Future<hashset<ContainerID>> recover(
- const std::list<mesos::slave::ContainerState>& states);
-
- virtual Try<pid_t> fork(
- const ContainerID& containerId,
- const std::string& path,
- const std::vector<std::string>& argv,
- const process::Subprocess::IO& in,
- const process::Subprocess::IO& out,
- const process::Subprocess::IO& err,
- const Option<flags::FlagsBase>& flags,
- const Option<std::map<std::string, std::string>>& environment,
- const Option<lambda::function<int()>>& setup,
- const Option<int>& namespaces);
-
- virtual process::Future<Nothing> destroy(const ContainerID& containerId);
-
-private:
- LinuxLauncher(
- const Flags& flags,
- const std::string& freezerHierarchy,
- const Option<std::string>& systemdHierarchy);
-
- static const std::string subsystem;
- const Flags flags;
- const std::string freezerHierarchy;
- const Option<std::string> systemdHierarchy;
-
- std::string cgroup(const ContainerID& containerId);
-
- // The 'pid' is the process id of the child process and also the
- // process group id and session id.
- hashmap<ContainerID, pid_t> pids;
-
- hashset<ContainerID> orphans;
-};
-
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
-
-#endif // __LINUX_LAUNCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp
index 08243b6..38fd868 100644
--- a/src/slave/containerizer/mesos/containerizer.cpp
+++ b/src/slave/containerizer/mesos/containerizer.cpp
@@ -42,9 +42,10 @@
#include "slave/containerizer/containerizer.hpp"
#include "slave/containerizer/fetcher.hpp"
-#include "slave/containerizer/launcher.hpp"
+
+#include "slave/containerizer/mesos/launcher.hpp"
#ifdef __linux__
-#include "slave/containerizer/linux_launcher.hpp"
+#include "slave/containerizer/mesos/linux_launcher.hpp"
#endif
#include "slave/containerizer/mesos/isolators/posix.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/containerizer.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/containerizer.hpp b/src/slave/containerizer/mesos/containerizer.hpp
index 3d134ca..e33f527 100644
--- a/src/slave/containerizer/mesos/containerizer.hpp
+++ b/src/slave/containerizer/mesos/containerizer.hpp
@@ -32,7 +32,8 @@
#include "slave/state.hpp"
#include "slave/containerizer/containerizer.hpp"
-#include "slave/containerizer/launcher.hpp"
+
+#include "slave/containerizer/mesos/launcher.hpp"
namespace mesos {
namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/launcher.cpp b/src/slave/containerizer/mesos/launcher.cpp
new file mode 100644
index 0000000..085c3e3
--- /dev/null
+++ b/src/slave/containerizer/mesos/launcher.cpp
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <mesos/type_utils.hpp>
+
+#include <process/collect.hpp>
+#include <process/delay.hpp>
+#include <process/process.hpp>
+#include <process/reap.hpp>
+
+#include <stout/unreachable.hpp>
+
+#include <stout/os/killtree.hpp>
+
+#include "mesos/resources.hpp"
+
+#include "slave/containerizer/mesos/launcher.hpp"
+
+using namespace process;
+
+using std::list;
+using std::map;
+using std::string;
+using std::vector;
+
+using mesos::slave::ContainerState;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+
+Try<Launcher*> PosixLauncher::create(const Flags& flags)
+{
+ return new PosixLauncher();
+}
+
+
+Future<hashset<ContainerID>> PosixLauncher::recover(
+ const list<ContainerState>& states)
+{
+ foreach (const ContainerState& state, states) {
+ const ContainerID& containerId = state.container_id();
+ pid_t pid = state.pid();
+
+ if (pids.containsValue(pid)) {
+ // This should (almost) never occur. There is the possibility
+ // that a new executor is launched with the same pid as one that
+ // just exited (highly unlikely) and the slave dies after the
+ // new executor is launched but before it hears about the
+ // termination of the earlier executor (also unlikely).
+ // Regardless, the launcher can't do anything sensible so this
+ // is considered an error.
+ return Failure("Detected duplicate pid " + stringify(pid) +
+ " for container " + stringify(containerId));
+ }
+
+ pids.put(containerId, pid);
+ }
+
+ return hashset<ContainerID>();
+}
+
+
+// The setup function in child before the exec.
+static int childSetup(const Option<lambda::function<int()>>& setup)
+{
+ // POSIX guarantees a forked child's pid does not match any existing
+ // process group id so only a single setsid() is required and the
+ // session id will be the pid.
+ // TODO(idownes): perror is not listed as async-signal-safe and
+ // should be reimplemented safely.
+ // TODO(jieyu): Move this logic to the subprocess (i.e.,
+ // mesos-containerizer launch).
+ if (::setsid() == -1) {
+ perror("Failed to put child in a new session");
+ _exit(1);
+ }
+
+ if (setup.isSome()) {
+ return setup.get()();
+ }
+
+ return 0;
+}
+
+
+Try<pid_t> PosixLauncher::fork(
+ const ContainerID& containerId,
+ const string& path,
+ const vector<string>& argv,
+ const Subprocess::IO& in,
+ const Subprocess::IO& out,
+ const Subprocess::IO& err,
+ const Option<flags::FlagsBase>& flags,
+ const Option<map<string, string>>& environment,
+ const Option<lambda::function<int()>>& setup,
+ const Option<int>& namespaces)
+{
+ if (pids.contains(containerId)) {
+ return Error("Process has already been forked for container " +
+ stringify(containerId));
+ }
+
+ Try<Subprocess> child = subprocess(
+ path,
+ argv,
+ in,
+ out,
+ err,
+ flags,
+ environment,
+ lambda::bind(&childSetup, setup));
+
+ if (child.isError()) {
+ return Error("Failed to fork a child process: " + child.error());
+ }
+
+ LOG(INFO) << "Forked child with pid '" << child.get().pid()
+ << "' for container '" << containerId << "'";
+
+ // Store the pid (session id and process group id).
+ pids.put(containerId, child.get().pid());
+
+ return child.get().pid();
+}
+
+
+// Forward declaration.
+Future<Nothing> _destroy(const Future<Option<int>>& future);
+
+
+Future<Nothing> PosixLauncher::destroy(const ContainerID& containerId)
+{
+ if (!pids.contains(containerId)) {
+ return Failure("Unknown container " + containerId.value());
+ }
+
+ pid_t pid = pids.get(containerId).get();
+
+ // Kill all processes in the session and process group.
+ Try<list<os::ProcessTree>> trees = os::killtree(pid, SIGKILL, true, true);
+
+ pids.erase(containerId);
+
+ // The child process may not have been waited on yet so we'll delay
+ // completing destroy until we're sure it has been reaped.
+ return process::reap(pid)
+ .then(lambda::bind(&_destroy, lambda::_1));
+}
+
+
+Future<Nothing> _destroy(const Future<Option<int>>& future)
+{
+ if (future.isReady()) {
+ return Nothing();
+ } else {
+ return Failure("Failed to kill all processes: " +
+ (future.isFailed() ? future.failure() : "unknown error"));
+ }
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/launcher.hpp b/src/slave/containerizer/mesos/launcher.hpp
new file mode 100644
index 0000000..8945776
--- /dev/null
+++ b/src/slave/containerizer/mesos/launcher.hpp
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LAUNCHER_HPP__
+#define __LAUNCHER_HPP__
+
+#include <sys/types.h>
+
+#include <list>
+#include <map>
+#include <string>
+
+#include <mesos/mesos.hpp>
+
+#include <mesos/slave/isolator.hpp>
+
+#include <process/future.hpp>
+#include <process/subprocess.hpp>
+
+#include <stout/flags.hpp>
+#include <stout/hashmap.hpp>
+#include <stout/hashset.hpp>
+#include <stout/lambda.hpp>
+#include <stout/option.hpp>
+#include <stout/try.hpp>
+
+#include "slave/flags.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+class Launcher
+{
+public:
+ virtual ~Launcher() {}
+
+ // Recover the necessary state for each container listed in state.
+ // Return the set of containers that are known to the launcher but
+ // not known to the slave (a.k.a. orphans).
+ virtual process::Future<hashset<ContainerID>> recover(
+ const std::list<mesos::slave::ContainerState>& states) = 0;
+
+ // Fork a new process in the containerized context. The child will
+ // exec the binary at the given path with the given argv, flags and
+ // environment. The I/O of the child will be redirected according to
+ // the specified I/O descriptors. The user can provide a 'setup'
+ // function which will be invoked in the child process right before
+ // the exec. The 'setup' function has to be async signal safe. The
+ // parent will return the child's pid if the fork is successful.
+ virtual Try<pid_t> fork(
+ const ContainerID& containerId,
+ const std::string& path,
+ const std::vector<std::string>& argv,
+ const process::Subprocess::IO& in,
+ const process::Subprocess::IO& out,
+ const process::Subprocess::IO& err,
+ const Option<flags::FlagsBase>& flags,
+ const Option<std::map<std::string, std::string>>& environment,
+ const Option<lambda::function<int()>>& setup,
+ const Option<int>& namespaces) = 0;
+
+ // Kill all processes in the containerized context.
+ virtual process::Future<Nothing> destroy(const ContainerID& containerId) = 0;
+};
+
+
+// Launcher suitable for any POSIX compliant system. Uses process
+// groups and sessions to track processes in a container. POSIX states
+// that process groups cannot migrate between sessions so all
+// processes for a container will be contained in a session.
+class PosixLauncher : public Launcher
+{
+public:
+ static Try<Launcher*> create(const Flags& flags);
+
+ virtual ~PosixLauncher() {}
+
+ virtual process::Future<hashset<ContainerID>> recover(
+ const std::list<mesos::slave::ContainerState>& states);
+
+ virtual Try<pid_t> fork(
+ const ContainerID& containerId,
+ const std::string& path,
+ const std::vector<std::string>& argv,
+ const process::Subprocess::IO& in,
+ const process::Subprocess::IO& out,
+ const process::Subprocess::IO& err,
+ const Option<flags::FlagsBase>& flags,
+ const Option<std::map<std::string, std::string>>& environment,
+ const Option<lambda::function<int()>>& setup,
+ const Option<int>& namespaces);
+
+ virtual process::Future<Nothing> destroy(const ContainerID& containerId);
+
+private:
+ PosixLauncher() {}
+
+ // The 'pid' is the process id of the first process and also the
+ // process group id and session id.
+ hashmap<ContainerID, pid_t> pids;
+};
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __LAUNCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/linux_launcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/linux_launcher.cpp b/src/slave/containerizer/mesos/linux_launcher.cpp
new file mode 100644
index 0000000..b6f76e2
--- /dev/null
+++ b/src/slave/containerizer/mesos/linux_launcher.cpp
@@ -0,0 +1,501 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <sched.h>
+#include <unistd.h>
+
+#include <linux/sched.h>
+
+#include <vector>
+
+#include <process/collect.hpp>
+
+#include <stout/abort.hpp>
+#include <stout/check.hpp>
+#include <stout/hashset.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+#include <stout/strings.hpp>
+
+#include "linux/cgroups.hpp"
+#include "linux/ns.hpp"
+#include "linux/systemd.hpp"
+
+#include "mesos/resources.hpp"
+
+#include "slave/containerizer/mesos/linux_launcher.hpp"
+
+#include "slave/containerizer/mesos/isolators/namespaces/pid.hpp"
+
+using namespace process;
+
+using std::list;
+using std::map;
+using std::set;
+using std::string;
+using std::vector;
+
+using mesos::slave::ContainerState;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+static ContainerID container(const string& cgroup)
+{
+ string basename = Path(cgroup).basename();
+
+ ContainerID containerId;
+ containerId.set_value(basename);
+ return containerId;
+}
+
+
+// `_systemdHierarchy` is only set if running on a systemd environment.
+LinuxLauncher::LinuxLauncher(
+ const Flags& _flags,
+ const string& _freezerHierarchy,
+ const Option<string>& _systemdHierarchy)
+ : flags(_flags),
+ freezerHierarchy(_freezerHierarchy),
+ systemdHierarchy(_systemdHierarchy) {}
+
+
+Try<Launcher*> LinuxLauncher::create(const Flags& flags)
+{
+ Try<string> freezerHierarchy = cgroups::prepare(
+ flags.cgroups_hierarchy,
+ "freezer",
+ flags.cgroups_root);
+
+ if (freezerHierarchy.isError()) {
+ return Error(
+ "Failed to create Linux launcher: " + freezerHierarchy.error());
+ }
+
+ // Ensure that no other subsystem is attached to the freezer hierarchy.
+ Try<set<string>> subsystems = cgroups::subsystems(freezerHierarchy.get());
+ if (subsystems.isError()) {
+ return Error(
+ "Failed to get the list of attached subsystems for hierarchy " +
+ freezerHierarchy.get());
+ } else if (subsystems.get().size() != 1) {
+ return Error(
+ "Unexpected subsystems found attached to the hierarchy " +
+ freezerHierarchy.get());
+ }
+
+ LOG(INFO) << "Using " << freezerHierarchy.get()
+ << " as the freezer hierarchy for the Linux launcher";
+
+ // On systemd environments we currently migrate executor pids into a separate
+ // executor slice. This allows the life-time of the executor to be extended
+ // past the life-time of the slave. See MESOS-3352.
+ // The LinuxLauncher takes responsibility for creating and starting this
+ // slice. It then migrates executor pids into this slice before it "unpauses"
+ // the executor. This is the same pattern as the freezer.
+
+ // If this is a systemd environment, ensure that the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE` exists and is running.
+ // TODO(jmlvanre): Prevent racing between multiple agents for this creation
+ // logic.
+ if (systemd::exists()) {
+ systemd::Flags systemdFlags;
+ systemdFlags.runtime_directory = flags.systemd_runtime_directory;
+ systemdFlags.cgroups_hierarchy = flags.cgroups_hierarchy;
+ Try<Nothing> initialize = systemd::initialize(systemdFlags);
+ if (initialize.isError()) {
+ return Error("Failed to initialize systemd: " + initialize.error());
+ }
+
+ // Check whether the `SYSTEMD_MESOS_EXECUTORS_SLICE` already exists. Create
+ // it if it does not exist.
+ // We explicitly don't modify the file if it exists in case operators want
+ // to over-ride the settings for the slice that we provide when we create
+ // the `Unit` below.
+ const Path path(path::join(
+ systemd::runtimeDirectory(),
+ SYSTEMD_MESOS_EXECUTORS_SLICE));
+
+ if (!systemd::slices::exists(path)) {
+ // A simple systemd file to allow us to start a new slice.
+ string unit = "[Unit]\nDescription=Mesos Executors Slice\n";
+
+ Try<Nothing> create = systemd::slices::create(path, unit);
+
+ if (create.isError()) {
+ return Error("Failed to create systemd slice '" +
+ stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) + "': " +
+ create.error());
+ }
+ }
+
+ // Regardless of whether we created the file or it existed already, we
+ // `start` the executor slice. It is safe (a no-op) to `start` an already
+ // running slice.
+ Try<Nothing> start = systemd::slices::start(SYSTEMD_MESOS_EXECUTORS_SLICE);
+
+ if (start.isError()) {
+ return Error("Failed to start '" +
+ stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) +
+ "': " + start.error());
+ }
+
+ // Now the `SYSTEMD_MESOS_EXECUTORS_SLICE` is ready for us to assign any
+ // executors. We can verify that our cgroups assignments will work by
+ // testing the hierarchy.
+ Try<bool> exists = cgroups::exists(
+ systemd::hierarchy(),
+ SYSTEMD_MESOS_EXECUTORS_SLICE);
+
+ if (exists.isError() || !exists.get()) {
+ return Error("Failed to locate systemd cgroups hierarchy: " +
+ (exists.isError() ? exists.error() : "does not exist"));
+ }
+ }
+
+ return new LinuxLauncher(
+ flags,
+ freezerHierarchy.get(),
+ systemd::exists() ?
+ Some(systemd::hierarchy()) :
+ Option<std::string>::none());
+}
+
+
+bool LinuxLauncher::available()
+{
+ // Make sure:
+ // - we run as root
+ // - "freezer" subsytem is enabled.
+
+ Try<bool> freezer = cgroups::enabled("freezer");
+ return ::geteuid() == 0 &&
+ freezer.isSome() &&
+ freezer.get();
+}
+
+
+Future<hashset<ContainerID>> LinuxLauncher::recover(
+ const std::list<ContainerState>& states)
+{
+ hashset<string> recovered;
+
+ // On systemd environments, capture the pids under the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE` for validation during recovery.
+ Result<std::set<pid_t>> mesosExecutorSlicePids = None();
+ if (systemdHierarchy.isSome()) {
+ mesosExecutorSlicePids =
+ cgroups::processes(systemdHierarchy.get(), SYSTEMD_MESOS_EXECUTORS_SLICE);
+
+ // If we error out trying to read the pids from the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE` we fail. This is a programmer error as we
+ // did not set up the slice correctly.
+ if (mesosExecutorSlicePids.isError()) {
+ return Failure("Failed to read pids from systemd '" +
+ stringify(SYSTEMD_MESOS_EXECUTORS_SLICE) + "'");
+ }
+ }
+
+ foreach (const ContainerState& state, states) {
+ const ContainerID& containerId = state.container_id();
+ pid_t pid = state.pid();
+
+ if (pids.containsValue(pid)) {
+ // This should (almost) never occur. There is the possibility
+ // that a new executor is launched with the same pid as one that
+ // just exited (highly unlikely) and the slave dies after the
+ // new executor is launched but before it hears about the
+ // termination of the earlier executor (also unlikely).
+ // Regardless, the launcher can't do anything sensible so this
+ // is considered an error.
+ return Failure("Detected duplicate pid " + stringify(pid) +
+ " for container " + stringify(containerId));
+ }
+
+ // Store the pid now because if the freezer cgroup is absent
+ // (slave terminated after the cgroup is destroyed but before it
+ // was notified) then we'll still need it for the check in
+ // destroy() when we clean up.
+ pids.put(containerId, pid);
+
+ Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
+
+ if (!exists.get()) {
+ // This may occur if the freezer cgroup was destroyed but the
+ // slave dies before noticing this. The containerizer will
+ // monitor the container's pid and notice that it has exited,
+ // triggering destruction of the container.
+ LOG(INFO) << "Couldn't find freezer cgroup for container "
+ << containerId << ", assuming already destroyed";
+ continue;
+ }
+
+ // If we are on a systemd environment, check that the pid is still in the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE`. If it is not, warn the operator that
+ // resource isolation may be invalidated.
+ // TODO(jmlvanre): Add a flag that enforces this matching (i.e. exits if a
+ // pid was found in the freezer but not in the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE`. We need to flag to support the upgrade
+ // path.
+ if (systemdHierarchy.isSome() && mesosExecutorSlicePids.isSome()) {
+ if (mesosExecutorSlicePids.get().count(pid) <= 0) {
+ LOG(WARNING)
+ << "Couldn't find pid '" << pid << "' in '"
+ << SYSTEMD_MESOS_EXECUTORS_SLICE << "'. This can lead to lack of"
+ << " proper resource isolation";
+ }
+ }
+
+ recovered.insert(cgroup(containerId));
+ }
+
+ // Return the set of orphan containers.
+ Try<vector<string>> cgroups =
+ cgroups::get(freezerHierarchy, flags.cgroups_root);
+
+ if (cgroups.isError()) {
+ return Failure(cgroups.error());
+ }
+
+ foreach (const string& cgroup, cgroups.get()) {
+ if (!recovered.contains(cgroup)) {
+ orphans.insert(container(cgroup));
+ }
+ }
+
+ return orphans;
+}
+
+
+static int childSetup(
+ int pipes[2],
+ const Option<lambda::function<int()>>& setup)
+{
+ // In child.
+ ::close(pipes[1]);
+
+ // Do a blocking read on the pipe until the parent signals us to
+ // continue.
+ char dummy;
+ ssize_t length;
+ while ((length = ::read(pipes[0], &dummy, sizeof(dummy))) == -1 &&
+ errno == EINTR);
+
+ if (length != sizeof(dummy)) {
+ ABORT("Failed to synchronize with parent");
+ }
+
+ ::close(pipes[0]);
+
+ // Move to a different session (and new process group) so we're
+ // independent from the slave's session (otherwise children will
+ // receive SIGHUP if the slave exits).
+ // TODO(idownes): perror is not listed as async-signal-safe and
+ // should be reimplemented safely.
+ // TODO(jieyu): Move this logic to the subprocess (i.e.,
+ // mesos-containerizer launch).
+ if (::setsid() == -1) {
+ perror("Failed to put child in a new session");
+ return 1;
+ }
+
+ if (setup.isSome()) {
+ return setup.get()();
+ }
+
+ return 0;
+}
+
+
+Try<pid_t> LinuxLauncher::fork(
+ const ContainerID& containerId,
+ const string& path,
+ const vector<string>& argv,
+ const process::Subprocess::IO& in,
+ const process::Subprocess::IO& out,
+ const process::Subprocess::IO& err,
+ const Option<flags::FlagsBase>& flags,
+ const Option<map<string, string>>& environment,
+ const Option<lambda::function<int()>>& setup,
+ const Option<int>& namespaces)
+{
+ // Create a freezer cgroup for this container if necessary.
+ Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
+ if (exists.isError()) {
+ return Error("Failed to check existence of freezer cgroup: " +
+ exists.error());
+ }
+
+ if (!exists.get()) {
+ Try<Nothing> created =
+ cgroups::create(freezerHierarchy, cgroup(containerId));
+
+ if (created.isError()) {
+ return Error("Failed to create freezer cgroup: " + created.error());
+ }
+ }
+
+ // Use a pipe to block the child until it's been moved into the
+ // freezer cgroup.
+ int pipes[2];
+
+ // We assume this should not fail under reasonable conditions so we
+ // use CHECK.
+ CHECK_EQ(0, ::pipe(pipes));
+
+ int cloneFlags = namespaces.isSome() ? namespaces.get() : 0;
+ cloneFlags |= SIGCHLD; // Specify SIGCHLD as child termination signal.
+
+ LOG(INFO) << "Cloning child process with flags = "
+ << ns::stringify(cloneFlags);
+
+ Try<Subprocess> child = subprocess(
+ path,
+ argv,
+ in,
+ out,
+ err,
+ flags,
+ environment,
+ lambda::bind(&childSetup, pipes, setup),
+ lambda::bind(&os::clone, lambda::_1, cloneFlags));
+
+ if (child.isError()) {
+ return Error("Failed to clone child process: " + child.error());
+ }
+
+ // Parent.
+ os::close(pipes[0]);
+
+ // Move the child into the freezer cgroup. Any grandchildren will
+ // also be contained in the cgroup.
+ // TODO(jieyu): Move this logic to the subprocess (i.e.,
+ // mesos-containerizer launch).
+ Try<Nothing> assign = cgroups::assign(
+ freezerHierarchy,
+ cgroup(containerId),
+ child.get().pid());
+
+ if (assign.isError()) {
+ LOG(ERROR) << "Failed to assign process " << child.get().pid()
+ << " of container '" << containerId << "'"
+ << " to its freezer cgroup: " << assign.error();
+
+ ::kill(child.get().pid(), SIGKILL);
+ return Error("Failed to contain process");
+ }
+
+ // If we are on systemd, then move the child into the
+ // `SYSTEMD_MESOS_EXECUTORS_SLICE`. As with the freezer, any grandchildren
+ // will also be contained in the slice.
+ if (systemdHierarchy.isSome()) {
+ Try<Nothing> assign = cgroups::assign(
+ systemdHierarchy.get(),
+ SYSTEMD_MESOS_EXECUTORS_SLICE,
+ child.get().pid());
+
+ if (assign.isError()) {
+ LOG(ERROR) << "Failed to assign process " << child.get().pid()
+ << " of container '" << containerId << "'"
+ << " to its systemd executor slice: " << assign.error();
+
+ ::kill(child.get().pid(), SIGKILL);
+ return Error("Failed to contain process on systemd");
+ }
+
+ LOG(INFO) << "Assigned child process '" << child.get().pid() << "' to '"
+ << SYSTEMD_MESOS_EXECUTORS_SLICE << "'";
+ }
+
+ // Now that we've contained the child we can signal it to continue
+ // by writing to the pipe.
+ char dummy;
+ ssize_t length;
+ while ((length = ::write(pipes[1], &dummy, sizeof(dummy))) == -1 &&
+ errno == EINTR);
+
+ os::close(pipes[1]);
+
+ if (length != sizeof(dummy)) {
+ // Ensure the child is killed.
+ ::kill(child.get().pid(), SIGKILL);
+ return Error("Failed to synchronize child process");
+ }
+
+ if (!pids.contains(containerId)) {
+ pids.put(containerId, child.get().pid());
+ }
+
+ return child.get().pid();
+}
+
+
+Future<Nothing> LinuxLauncher::destroy(const ContainerID& containerId)
+{
+ if (!pids.contains(containerId) && !orphans.contains(containerId)) {
+ return Failure("Unknown container");
+ }
+
+ pids.erase(containerId);
+ orphans.erase(containerId);
+
+ // Just return if the cgroup was destroyed and the slave didn't receive the
+ // notification. See comment in recover().
+ Try<bool> exists = cgroups::exists(freezerHierarchy, cgroup(containerId));
+ if (exists.isError()) {
+ return Failure("Failed to check existence of freezer cgroup: " +
+ exists.error());
+ }
+
+ if (!exists.get()) {
+ return Nothing();
+ }
+
+ Result<ino_t> containerPidNs =
+ NamespacesPidIsolatorProcess::getNamespace(containerId);
+
+ if (containerPidNs.isSome()) {
+ LOG(INFO) << "Using pid namespace to destroy container " << containerId;
+
+ return ns::pid::destroy(containerPidNs.get())
+ .then(lambda::bind(
+ (Future<Nothing>(*)(const string&,
+ const string&,
+ const Duration&))(&cgroups::destroy),
+ freezerHierarchy,
+ cgroup(containerId),
+ cgroups::DESTROY_TIMEOUT));
+ }
+
+ // Try to clean up using just the freezer cgroup.
+ return cgroups::destroy(
+ freezerHierarchy,
+ cgroup(containerId),
+ cgroups::DESTROY_TIMEOUT);
+}
+
+
+string LinuxLauncher::cgroup(const ContainerID& containerId)
+{
+ return path::join(flags.cgroups_root, containerId.value());
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/slave/containerizer/mesos/linux_launcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/linux_launcher.hpp b/src/slave/containerizer/mesos/linux_launcher.hpp
new file mode 100644
index 0000000..cbf12c5
--- /dev/null
+++ b/src/slave/containerizer/mesos/linux_launcher.hpp
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LINUX_LAUNCHER_HPP__
+#define __LINUX_LAUNCHER_HPP__
+
+#include "slave/containerizer/mesos/launcher.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+// TODO(jmlvanre): We may want to allow this to be configured.
+static const char SYSTEMD_MESOS_EXECUTORS_SLICE[] = "mesos_executors.slice";
+
+// Launcher for Linux systems with cgroups. Uses a freezer cgroup to
+// track pids.
+class LinuxLauncher : public Launcher
+{
+public:
+ static Try<Launcher*> create(const Flags& flags);
+
+ // Returns 'true' if prerequisites for using LinuxLauncher are available.
+ static bool available();
+
+ virtual ~LinuxLauncher() {}
+
+ virtual process::Future<hashset<ContainerID>> recover(
+ const std::list<mesos::slave::ContainerState>& states);
+
+ virtual Try<pid_t> fork(
+ const ContainerID& containerId,
+ const std::string& path,
+ const std::vector<std::string>& argv,
+ const process::Subprocess::IO& in,
+ const process::Subprocess::IO& out,
+ const process::Subprocess::IO& err,
+ const Option<flags::FlagsBase>& flags,
+ const Option<std::map<std::string, std::string>>& environment,
+ const Option<lambda::function<int()>>& setup,
+ const Option<int>& namespaces);
+
+ virtual process::Future<Nothing> destroy(const ContainerID& containerId);
+
+private:
+ LinuxLauncher(
+ const Flags& flags,
+ const std::string& freezerHierarchy,
+ const Option<std::string>& systemdHierarchy);
+
+ static const std::string subsystem;
+ const Flags flags;
+ const std::string freezerHierarchy;
+ const Option<std::string> systemdHierarchy;
+
+ std::string cgroup(const ContainerID& containerId);
+
+ // The 'pid' is the process id of the child process and also the
+ // process group id and session id.
+ hashmap<ContainerID, pid_t> pids;
+
+ hashset<ContainerID> orphans;
+};
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __LINUX_LAUNCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/tests/containerizer/filesystem_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/filesystem_isolator_tests.cpp b/src/tests/containerizer/filesystem_isolator_tests.cpp
index 7032095..b3e4fc8 100644
--- a/src/tests/containerizer/filesystem_isolator_tests.cpp
+++ b/src/tests/containerizer/filesystem_isolator_tests.cpp
@@ -40,7 +40,7 @@
#include "slave/paths.hpp"
#ifdef __linux__
-#include "slave/containerizer/linux_launcher.hpp"
+#include "slave/containerizer/mesos/linux_launcher.hpp"
#include "slave/containerizer/mesos/isolators/filesystem/linux.hpp"
#endif
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/tests/containerizer/isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/isolator_tests.cpp b/src/tests/containerizer/isolator_tests.cpp
index 25dbd54..01d22b4 100644
--- a/src/tests/containerizer/isolator_tests.cpp
+++ b/src/tests/containerizer/isolator_tests.cpp
@@ -58,13 +58,12 @@
#endif // __linux__
#include "slave/containerizer/mesos/isolators/posix.hpp"
-#include "slave/containerizer/launcher.hpp"
+#include "slave/containerizer/mesos/launcher.hpp"
#ifdef __linux__
#include "slave/containerizer/fetcher.hpp"
-#include "slave/containerizer/linux_launcher.hpp"
-
#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/launch.hpp"
+#include "slave/containerizer/mesos/linux_launcher.hpp"
#endif // __linux__
#include "tests/flags.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/tests/containerizer/launcher.hpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/launcher.hpp b/src/tests/containerizer/launcher.hpp
index 5d34bab..f587dda 100644
--- a/src/tests/containerizer/launcher.hpp
+++ b/src/tests/containerizer/launcher.hpp
@@ -39,7 +39,7 @@
#include <stout/nothing.hpp>
#include <stout/option.hpp>
-#include "slave/containerizer/launcher.hpp"
+#include "slave/containerizer/mesos/launcher.hpp"
namespace mesos {
namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/tests/containerizer/mesos_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/mesos_containerizer_tests.cpp b/src/tests/containerizer/mesos_containerizer_tests.cpp
index b48133c..d1ff139 100644
--- a/src/tests/containerizer/mesos_containerizer_tests.cpp
+++ b/src/tests/containerizer/mesos_containerizer_tests.cpp
@@ -36,9 +36,9 @@
#include "slave/flags.hpp"
#include "slave/containerizer/fetcher.hpp"
-#include "slave/containerizer/launcher.hpp"
#include "slave/containerizer/mesos/containerizer.hpp"
+#include "slave/containerizer/mesos/launcher.hpp"
#include "tests/flags.hpp"
#include "tests/mesos.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/a6fbf34b/src/tests/containerizer/port_mapping_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/port_mapping_tests.cpp b/src/tests/containerizer/port_mapping_tests.cpp
index 520e6b9..9195338 100644
--- a/src/tests/containerizer/port_mapping_tests.cpp
+++ b/src/tests/containerizer/port_mapping_tests.cpp
@@ -56,14 +56,14 @@
#include "slave/flags.hpp"
#include "slave/slave.hpp"
-#include "slave/containerizer/mesos/isolators/network/port_mapping.hpp"
-
#include "slave/containerizer/fetcher.hpp"
-#include "slave/containerizer/launcher.hpp"
-#include "slave/containerizer/linux_launcher.hpp"
#include "slave/containerizer/mesos/containerizer.hpp"
#include "slave/containerizer/mesos/launch.hpp"
+#include "slave/containerizer/mesos/launcher.hpp"
+#include "slave/containerizer/mesos/linux_launcher.hpp"
+
+#include "slave/containerizer/mesos/isolators/network/port_mapping.hpp"
#include "tests/flags.hpp"
#include "tests/mesos.hpp"