You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bm...@apache.org on 2024/07/31 22:54:01 UTC
(mesos) branch master updated (b8936677f -> 5bed66c00)
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
from b8936677f [cgroups2] Pass device manager to controllers & cgroups2 isolator.
new 803868152 [cgroups2] Introduces the DeviceControllerProcess.
new 2f545b112 [cgroups2] create device controller in Cgroups2Isolator.
new 5bed66c00 [cgroups2] Skip enabling of devices controller.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/CMakeLists.txt | 1 +
src/Makefile.am | 2 +
.../mesos/isolators/cgroups2/cgroups2.cpp | 34 ++---
.../mesos/isolators/cgroups2/constants.hpp | 1 +
.../controllers}/devices.cpp | 137 ++++++++-------------
.../controllers}/devices.hpp | 35 +++---
6 files changed, 92 insertions(+), 118 deletions(-)
copy src/slave/containerizer/mesos/isolators/{cgroups/subsystems => cgroups2/controllers}/devices.cpp (51%)
copy src/slave/containerizer/mesos/isolators/{cgroups/subsystems => cgroups2/controllers}/devices.hpp (68%)
(mesos) 01/03: [cgroups2] Introduces the DeviceControllerProcess.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8038681524f9b0efaacf77e09e7933ea7ca999ee
Author: Jason Zhou <ja...@gmail.com>
AuthorDate: Wed Jul 31 15:27:47 2024 -0700
[cgroups2] Introduces the DeviceControllerProcess.
Introduces a device controller that supports cgroups v2 and is available
in the Cgroups2IsolatorProcess. Device access control is made through
the DeviceManager.
Review: https://reviews.apache.org/r/75098/
---
src/CMakeLists.txt | 1 +
src/Makefile.am | 2 +
.../mesos/isolators/cgroups2/constants.hpp | 1 +
.../isolators/cgroups2/controllers/devices.cpp | 204 +++++++++++++++++++++
.../isolators/cgroups2/controllers/devices.hpp | 82 +++++++++
5 files changed, 290 insertions(+)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 9526ed17a..50684b615 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -349,6 +349,7 @@ set(LINUX_SRC
slave/containerizer/mesos/isolators/cgroups2/controllers/core.cpp
slave/containerizer/mesos/isolators/cgroups2/controllers/cpu.cpp
slave/containerizer/mesos/isolators/cgroups2/controllers/memory.cpp
+ slave/containerizer/mesos/isolators/cgroups2/controllers/devices.cpp
slave/containerizer/mesos/isolators/cgroups2/controllers/perf_event.cpp
slave/containerizer/device_manager/device_manager.cpp)
diff --git a/src/Makefile.am b/src/Makefile.am
index 456942389..5b91319d6 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1497,6 +1497,8 @@ MESOS_LINUX_FILES = \
slave/containerizer/mesos/isolators/cgroups2/controllers/memory.hpp \
slave/containerizer/mesos/isolators/cgroups2/controllers/perf_event.cpp \
slave/containerizer/mesos/isolators/cgroups2/controllers/perf_event.hpp \
+ slave/containerizer/mesos/isolators/cgroups2/controllers/devices.cpp \
+ slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp \
slave/containerizer/device_manager/device_manager.cpp \
slave/containerizer/device_manager/device_manager.hpp
diff --git a/src/slave/containerizer/mesos/isolators/cgroups2/constants.hpp b/src/slave/containerizer/mesos/isolators/cgroups2/constants.hpp
index bad79ad0c..5e6426694 100644
--- a/src/slave/containerizer/mesos/isolators/cgroups2/constants.hpp
+++ b/src/slave/containerizer/mesos/isolators/cgroups2/constants.hpp
@@ -39,6 +39,7 @@ const std::string CGROUPS2_CONTROLLER_CORE_NAME = "core";
const std::string CGROUPS2_CONTROLLER_CPU_NAME = "cpu";
const std::string CGROUPS2_CONTROLLER_MEMORY_NAME = "memory";
const std::string CGROUPS2_CONTROLLER_PERF_EVENT_NAME = "perf_event";
+const std::string CGROUPS2_CONTROLLER_DEVICES_NAME = "devices";
} // namespace slave {
} // namespace internal {
diff --git a/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.cpp b/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.cpp
new file mode 100644
index 000000000..702ed50b0
--- /dev/null
+++ b/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.cpp
@@ -0,0 +1,204 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <sys/stat.h>
+
+// This header include must be enclosed in an `extern "C"` block to
+// workaround a bug in glibc <= 2.12 (see MESOS-7378).
+//
+// TODO(neilc): Remove this when we no longer support glibc <= 2.12.
+extern "C" {
+#include <sys/sysmacros.h>
+}
+
+#include <process/defer.hpp>
+#include <process/id.hpp>
+
+#include <stout/nothing.hpp>
+#include <stout/try.hpp>
+#include <stout/os.hpp>
+
+#include "slave/containerizer/mesos/isolators/cgroups2/constants.hpp"
+#include "slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp"
+
+using cgroups::devices::Entry;
+
+using process::Failure;
+using process::Future;
+using process::Owned;
+
+using std::string;
+using std::vector;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+// The default list of devices to whitelist when device isolation is
+// turned on. The full list of devices can be found here:
+// https://www.kernel.org/doc/Documentation/devices.txt
+//
+// Device whitelisting is described here:
+// https://www.kernel.org/doc/Documentation/cgroup-v1/devices.txt
+static const char* DEFAULT_WHITELIST_ENTRIES[] = {
+ "c *:* m", // Make new character devices.
+ "b *:* m", // Make new block devices.
+ "c 5:1 rwm", // /dev/console
+ "c 4:0 rwm", // /dev/tty0
+ "c 4:1 rwm", // /dev/tty1
+ "c 136:* rwm", // /dev/pts/*
+ "c 5:2 rwm", // /dev/ptmx
+ "c 10:200 rwm", // /dev/net/tun
+ "c 1:3 rwm", // /dev/null
+ "c 1:5 rwm", // /dev/zero
+ "c 1:7 rwm", // /dev/full
+ "c 5:0 rwm", // /dev/tty
+ "c 1:9 rwm", // /dev/urandom
+ "c 1:8 rwm", // /dev/random
+};
+
+
+Try<Owned<ControllerProcess>> DeviceControllerProcess::create(
+ const Flags& flags,
+ const Owned<DeviceManager> deviceManager)
+{
+ vector<cgroups::devices::Entry> whitelistDeviceEntries;
+
+ foreach (const char* _entry, DEFAULT_WHITELIST_ENTRIES) {
+ whitelistDeviceEntries.push_back(
+ CHECK_NOTERROR(cgroups::devices::Entry::parse(_entry)));
+ }
+
+ if (flags.allowed_devices.isSome()) {
+ foreach (const DeviceAccess& device_access,
+ flags.allowed_devices->allowed_devices()) {
+ if (!device_access.device().has_path()) {
+ return Error("Whitelisted device has no device path provided");
+ }
+
+ const string& path = device_access.device().path();
+ const DeviceAccess_Access& access = device_access.access();
+
+ if (!(access.read() || access.write() || access.mknod())) {
+ return Error("Could not whitelist device '" + path + "'"
+ " without any access privileges");
+ }
+
+ Try<dev_t> device = os::stat::rdev(path);
+ if (device.isError()) {
+ return Error("Failed to obtain device ID for '" + path + "'"
+ ": " + device.error());
+ }
+
+ Try<mode_t> mode = os::stat::mode(path);
+ if (mode.isError()) {
+ return Error("Failed to obtain device mode for '" + path + "'"
+ ": " + mode.error());
+ }
+
+ Entry::Selector::Type type;
+ if (S_ISBLK(mode.get())) {
+ type = Entry::Selector::Type::BLOCK;
+ } else if (S_ISCHR(mode.get())) {
+ type = Entry::Selector::Type::CHARACTER;
+ } else {
+ return Error("Failed to determine device type for '" + path + "'");
+ }
+
+ cgroups::devices::Entry entry;
+ entry.selector.type = type;
+ entry.selector.major = major(device.get());
+ entry.selector.minor = minor(device.get());
+ entry.access.read = access.read();
+ entry.access.write = access.write();
+ entry.access.mknod = access.mknod();
+
+ whitelistDeviceEntries.push_back(entry);
+ }
+ }
+
+ return Owned<ControllerProcess>(new DeviceControllerProcess(
+ flags, whitelistDeviceEntries, deviceManager));
+}
+
+
+string DeviceControllerProcess::name() const
+{
+ return CGROUPS2_CONTROLLER_DEVICES_NAME;
+}
+
+
+DeviceControllerProcess::DeviceControllerProcess(
+ const Flags& _flags,
+ const std::vector<cgroups::devices::Entry>& _whitelistDeviceEntries,
+ const Owned<DeviceManager> _deviceManager)
+ : ProcessBase(process::ID::generate("cgroups-v2-devices-controller")),
+ ControllerProcess(_flags),
+ whitelistDeviceEntries(_whitelistDeviceEntries),
+ deviceManager(_deviceManager) {}
+
+
+Future<Nothing> DeviceControllerProcess::recover(
+ const ContainerID& containerId,
+ const string& cgroup)
+{
+ // TODO(jasonzhou): Work with device manager recovery.
+ if (containerIds.contains(containerId)) {
+ return Failure(
+ "The controller '" + name() + "' of container " +
+ stringify(containerId) + " has already been recovered");
+ }
+
+ containerIds.insert(containerId);
+
+ return Nothing();
+}
+
+
+Future<Nothing> DeviceControllerProcess::prepare(
+ const ContainerID& containerId,
+ const string& cgroup,
+ const mesos::slave::ContainerConfig& containerConfig)
+{
+ if (containerIds.contains(containerId)) {
+ return Failure("The controller '" + name() + "' has already been prepared");
+ }
+
+ return deviceManager->configure(cgroup, whitelistDeviceEntries, {})
+ .then(defer(self(), [=] {
+ containerIds.insert(containerId);
+ return Nothing();
+ }));
+}
+
+
+Future<Nothing> DeviceControllerProcess::cleanup(
+ const ContainerID& containerId,
+ const string& cgroup)
+{
+ if (!containerIds.contains(containerId)) {
+ VLOG(1) << "Ignoring cleanup controller '" << name() << "'"
+ << " for unknown container " << containerId;
+ return Nothing();
+ }
+
+ containerIds.erase(containerId);
+ return deviceManager->remove(cgroup);
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
diff --git a/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp b/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp
new file mode 100644
index 000000000..c8a2fa34f
--- /dev/null
+++ b/src/slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __DEVICES_HPP__
+#define __DEVICES_HPP__
+
+#include <string>
+#include <vector>
+
+#include <process/owned.hpp>
+
+#include <stout/hashset.hpp>
+#include <stout/try.hpp>
+
+#include "linux/cgroups.hpp"
+
+#include "slave/flags.hpp"
+
+#include "slave/containerizer/mesos/isolators/cgroups2/controller.hpp"
+#include "slave/containerizer/device_manager/device_manager.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+/**
+ * Represent cgroups v2 devices controller.
+ */
+class DeviceControllerProcess: public ControllerProcess
+{
+public:
+ static Try<process::Owned<ControllerProcess>> create(
+ const Flags& flags,
+ process::Owned<DeviceManager> deviceManager
+ );
+
+ ~DeviceControllerProcess() override = default;
+
+ std::string name() const override;
+
+ process::Future<Nothing> prepare(
+ const ContainerID& containerId,
+ const std::string& cgroup,
+ const mesos::slave::ContainerConfig& containerConfig) override;
+
+ process::Future<Nothing> recover(
+ const ContainerID& containerId,
+ const std::string& cgroup) override;
+
+ process::Future<Nothing> cleanup(
+ const ContainerID& containerId,
+ const std::string& cgroup) override;
+
+private:
+ DeviceControllerProcess(
+ const Flags& flags,
+ const std::vector<cgroups::devices::Entry>& whitelistDeviceEntries,
+ process::Owned<DeviceManager> deviceManager);
+
+ hashset<ContainerID> containerIds;
+ std::vector<cgroups::devices::Entry> whitelistDeviceEntries;
+ process::Owned<DeviceManager> deviceManager;
+};
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __DEVICES_HPP__
(mesos) 03/03: [cgroups2] Skip enabling of devices controller.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 5bed66c009a5b3c5670e3b7a5c0adae87b58417a
Author: Jason Zhou <ja...@gmail.com>
AuthorDate: Wed Jul 31 15:53:15 2024 -0700
[cgroups2] Skip enabling of devices controller.
Similar to the `perf_event` controller, the `devices` controller cannot
be written into cgroup.subtree_control file, so we skip the call to
cgroups2::controllers::enable for the device controller. Otherwise
we will run into an "Invalid argument" error from cgroups.
Review: https://reviews.apache.org/r/75130/
---
.../mesos/isolators/cgroups2/cgroups2.cpp | 24 ++++++++++++----------
1 file changed, 13 insertions(+), 11 deletions(-)
diff --git a/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp b/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
index 5e2856437..a027ee632 100644
--- a/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
+++ b/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
@@ -225,21 +225,23 @@ Future<Option<ContainerLaunchInfo>> Cgroups2IsolatorProcess::prepare(
// The "core" controller is always enabled because the "cgroup.*" control
// files exist for all cgroups.
//
- // Additionally, since "core" and "perf_event" aren't valid controller
- // names (i.e. they don't exist in "cgroup.controllers"), calling
- // `cgroups2::controllers::enable` with these cgroups will fail with
- // "Invalid argument".
+ // Additionally, since "core", "perf_event", and "devices" aren't valid
+ // controller names (i.e. they don't exist in "cgroup.controllers"),
+ // calling `cgroups2::controllers::enable` with these cgroups will fail
+ // with "Invalid argument".
//
- // Therefore, we skip enabling the "core" and "perf_event" controller here.
+ // Therefore, we skip enabling the "core", "perf_event", and "devices"
+ // controller here.
continue;
}
- // Similar to "core", "perf_event" does not exist in cgroup.controllers,
- // and therefore we cannot call cgroups2::controllers::enable with it
- // as it cannot be written into cgroup.subtree_control, but we still
- // need to push it into the controllers of the containers, so we will
- // only skip the call for cgroups2::controllers::enable
- if (controller->name() != "perf_event") {
+ // Similar to "core", "perf_event" and "devices" do not exist in
+ // cgroup.controllers, and therefore we cannot call
+ // cgroups2::controllers::enable with it as it cannot be written into
+ // cgroup.subtree_control, but we still need to push it into the controllers
+ // of the containers, so we will only skip the call for
+ // cgroups2::controllers::enable.
+ if (controller->name() != "perf_event" && controller->name() != "devices") {
Try<Nothing> enable =
cgroups2::controllers::enable(nonLeafCgroup, {controller->name()});
if (enable.isError()) {
(mesos) 02/03: [cgroups2] create device controller in Cgroups2Isolator.
Posted by bm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bmahler pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 2f545b11253800c33fa4d1fde073227deb772f89
Author: Jason Zhou <ja...@gmail.com>
AuthorDate: Wed Jul 31 15:51:32 2024 -0700
[cgroups2] create device controller in Cgroups2Isolator.
DeviceController needs to be created in Cgroups2Isolator with the
DeviceManager so that the default whitelist can be properly configured.
Review: https://reviews.apache.org/r/75121/
---
src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp b/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
index 3cf3645c4..5e2856437 100644
--- a/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
+++ b/src/slave/containerizer/mesos/isolators/cgroups2/cgroups2.cpp
@@ -22,6 +22,7 @@
#include "slave/containerizer/mesos/isolators/cgroups2/controllers/cpu.hpp"
#include "slave/containerizer/mesos/isolators/cgroups2/controllers/memory.hpp"
#include "slave/containerizer/mesos/isolators/cgroups2/controllers/perf_event.hpp"
+#include "slave/containerizer/mesos/isolators/cgroups2/controllers/devices.hpp"
#include <set>
#include <string>
@@ -84,9 +85,12 @@ Try<Isolator*> Cgroups2IsolatorProcess::create(
{"perf_event", &PerfEventControllerProcess::create}
};
- hashmap<string, Try<Owned<ControllerProcess>>(*)
- (const Flags&, const Owned<DeviceManager>)>
- creatorsWithDeviceManager = {};
+ hashmap<string, Try<Owned<ControllerProcess>>(*)(
+ const Flags&,
+ const Owned<DeviceManager>)>
+ creatorsWithDeviceManager = {
+ {"devices", &DeviceControllerProcess::create},
+ };
hashmap<string, Owned<Controller>> controllers;