You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2018/08/15 22:53:47 UTC
[mesos] branch 1.7.x updated (a554137 -> b178910)
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a change to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from a554137 Added MESOS-8814 to 1.7.0 CHANGELOG.
new a81a4f6 Call any function in a specified namespace.
new c55e86e Added networking statistics to cni isolator.
new 6c1f5c1 Added MESOS-5647 to 1.7.0 CHANGELOG.
new b178910 Highlighted MESOS-5647 in 1.7.0 CHANGELOG.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG | 4 +
docs/configuration/agent.md | 10 ++
src/linux/ns.hpp | 143 +++++++++++++++++++++
.../mesos/isolators/network/cni/cni.cpp | 91 +++++++++++++
.../mesos/isolators/network/cni/cni.hpp | 12 ++
src/slave/flags.cpp | 6 +
src/slave/flags.hpp | 1 +
src/tests/containerizer/ns_tests.cpp | 24 ++++
8 files changed, 291 insertions(+)
[mesos] 03/04: Added MESOS-5647 to 1.7.0 CHANGELOG.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 6c1f5c103b605227f5d4ba7b7d79283ed7d83cf8
Author: Jie Yu <yu...@gmail.com>
AuthorDate: Wed Aug 15 15:44:14 2018 -0700
Added MESOS-5647 to 1.7.0 CHANGELOG.
(cherry picked from commit 33b30849b92397cf186458b1e5a1043df9ef305a)
---
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG b/CHANGELOG
index b30f1fb..9c02962 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -163,6 +163,7 @@ All Resolved Issues:
* [MESOS-3442] - Port path_tests to Windows
* [MESOS-3444] - Port sendfile_tests
* [MESOS-4509] - Remove deprecated .json endpoints.
+ * [MESOS-5647] - Expose network statistics for containers on CNI network in the `network/cni` isolator.
* [MESOS-5814] - Port libprocess http_tests.cpp
* [MESOS-5817] - Port libprocess process_tests.cpp
* [MESOS-5941] - RemoteLink tests fail on Windows
[mesos] 04/04: Highlighted MESOS-5647 in 1.7.0 CHANGELOG.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit b1789105cb0948c0e3fe78d4a24d9c278685007b
Author: Jie Yu <yu...@gmail.com>
AuthorDate: Wed Aug 15 15:52:38 2018 -0700
Highlighted MESOS-5647 in 1.7.0 CHANGELOG.
(cherry picked from commit f8edb5086dfdc51e1016affddc287eb5a4dea66f)
---
CHANGELOG | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGELOG b/CHANGELOG
index 9c02962..974904d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -24,6 +24,9 @@ This release contains the following new features:
configuration. By utilizing I/O Completion Ports, this enables
non-blocking asynchronous I/O on Windows for sockets, pipes, and files.
+ * [MESOS-5647] - Expose network statistics for containers on CNI
+ network in the `network/cni` isolator.
+
Additional API Changes:
* [MESOS-9015] Allow resources to be removed when updating the sorter.
[mesos] 01/04: Call any function in a specified namespace.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit a81a4f69e2514a981bbe3653ee3ee4c666dfe68e
Author: Sergey Urbanovich <se...@gmail.com>
AuthorDate: Mon Aug 13 16:20:27 2018 -0700
Call any function in a specified namespace.
The NamespaceRunner runs any function in a specified namespace. To do
that it manages a separate thread which would be re-associated with
that namespace.
Review: https://reviews.apache.org/r/68053/
(cherry picked from commit fd762943c928be810ba4c2c0aab9e9a6ebc399e8)
---
src/linux/ns.hpp | 143 +++++++++++++++++++++++++++++++++++
src/tests/containerizer/ns_tests.cpp | 24 ++++++
2 files changed, 167 insertions(+)
diff --git a/src/linux/ns.hpp b/src/linux/ns.hpp
index 0b4136b..bb40038 100644
--- a/src/linux/ns.hpp
+++ b/src/linux/ns.hpp
@@ -26,11 +26,16 @@
#include <sys/syscall.h>
+#include <queue>
#include <set>
#include <string>
+#include <thread>
+
+#include <process/future.hpp>
#include <stout/lambda.hpp>
#include <stout/nothing.hpp>
+#include <stout/option.hpp>
#include <stout/result.hpp>
#include <stout/try.hpp>
@@ -178,6 +183,144 @@ Try<pid_t> clone(
// flags, e.g., CLONE_NEWNS | CLONE_NEWNET.
std::string stringify(int flags);
+
+// The NamespaceRunner runs any function in a specified namespace.
+// To do that it manages a separate thread which would be re-associated
+// with that namespace.
+class NamespaceRunner
+{
+public:
+ NamespaceRunner()
+ {
+ // Start the looper thread.
+ thread.reset(new std::thread(&NamespaceRunner::loop, this));
+ }
+
+ ~NamespaceRunner()
+ {
+ // Shutdown the queue.
+ queue.shutdown();
+ // Wait for the thread to complete.
+ thread->join();
+ thread.reset();
+ }
+
+ // Run any function in a specified namespace.
+ template <typename T>
+ process::Future<T> run(
+ const std::string& path,
+ const std::string& ns,
+ const lambda::function<Try<T>()>& func)
+ {
+ std::shared_ptr<process::Promise<T>> promise(
+ new process::Promise<T>);
+ process::Future<T> future = promise->future();
+
+ // Put a function to the queue, the function will be called
+ // in the thread. The thread will be re-associated with the
+ // specified namespace.
+ queue.put([=]{
+ Try<Nothing> setns = ::ns::setns(path, ns, false);
+ if (setns.isError()) {
+ promise->fail(setns.error());
+ } else {
+ promise->set(func());
+ }
+ });
+
+ return future;
+ }
+
+private:
+ typedef lambda::function<void()> Func;
+
+ // The thread loop.
+ void loop()
+ {
+ for (;;) {
+ // Get a function from the queue.
+ Option<Func> func = queue.get();
+
+ // Stop the thread if the queue is shutdowned.
+ if (func.isNone()) {
+ break;
+ }
+
+ // Call the function, it re-associates the thread with the
+ // specified namespace and calls the initial user function.
+ func.get()();
+ }
+ }
+
+ // It's not safe to use process::Queue when not all of its callers are
+ // managed by libprocess. Calling Future::await() in looper thread
+ // might cause the looper thread to be donated to a libprocess Process.
+ // If that Process is very busy (e.g., master or agent Process), it's
+ // possible that the looper thread will never re-gain control.
+ //
+ // ProcessingQueue uses mutex and condition variable to solve this
+ // problem. ProcessingQueue::get() can block the thread. The main
+ // use cases for the class are thread workers and thread pools.
+ template <typename T>
+ class ProcessingQueue
+ {
+ public:
+ ProcessingQueue() : finished(false) {}
+
+ ~ProcessingQueue() = default;
+
+ // Add an element to the queue and notify one client.
+ void put(T&& t)
+ {
+ synchronized (mutex) {
+ queue.push(std::forward<T>(t));
+ cond.notify_one();
+ }
+ }
+
+ // NOTE: This function blocks the thread. It returns the oldest
+ // element from the queue and returns None() if the queue is
+ // shutdowned.
+ Option<T> get()
+ {
+ synchronized (mutex) {
+ // Wait for either a new queue element or queue shutdown.
+ while (queue.empty() && !finished) {
+ synchronized_wait(&cond, &mutex);
+ }
+
+ if (finished) {
+ // The queue is shutdowned.
+ return None();
+ }
+
+ // Return the oldest element from the queue.
+ T t = std::move(queue.front());
+ queue.pop();
+ return Some(std::move(t));
+ }
+ }
+
+ // Shutdown the queue and notify all clients.
+ void shutdown() {
+ synchronized (mutex) {
+ finished = true;
+ std::queue<T>().swap(queue);
+ cond.notify_all();
+ }
+ }
+
+ private:
+ std::mutex mutex;
+ std::condition_variable cond;
+ std::queue<T> queue;
+ bool finished;
+ };
+
+ ProcessingQueue<Func> queue;
+ std::unique_ptr<std::thread> thread;
+};
+
} // namespace ns {
#endif // __LINUX_NS_HPP__
diff --git a/src/tests/containerizer/ns_tests.cpp b/src/tests/containerizer/ns_tests.cpp
index fa4349e..14aad89 100644
--- a/src/tests/containerizer/ns_tests.cpp
+++ b/src/tests/containerizer/ns_tests.cpp
@@ -266,6 +266,30 @@ TEST(NsTest, ROOT_clone)
EXPECT_NONE(status.get());
}
+
+// Test the ns::NamespaceRunner().
+TEST(NsTest, ROOT_NamespaceRunner)
+{
+ process::Future<int> r;
+
+ // Initialize the Runner.
+ ns::NamespaceRunner runner;
+
+ // Run a dummy function in a networking namespace.
+ lambda::function<Try<int>()> f = []() -> Try<int> {
+ return 42;
+ };
+
+ r = runner.run("/proc/self/ns/net", "net", f);
+ AWAIT_READY(r);
+ EXPECT_EQ(r.get(), 42);
+
+ // Run the function with an invalid namespace type.
+ r = runner.run("/proc/self/ns/net", "mnt", f);
+ AWAIT_FAILED(r);
+ EXPECT_EQ(r.failure(), "Invalid argument");
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {
[mesos] 02/04: Added networking statistics to cni isolator.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch 1.7.x
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c55e86e7b70066af7d353771e92463e0014c4901
Author: Sergey Urbanovich <se...@gmail.com>
AuthorDate: Wed Aug 15 14:53:01 2018 -0700
Added networking statistics to cni isolator.
On receiving a request for getting `usage` for a given container
the `network/cni` isolator uses getifaddrs(3) glibc function. The
function returns basic networking metrics for each networking
interface in the container networking namespace. It should work
right out of the box on all modern Linux-based systems.
To get more networking metrics please use Netlink Protocol Library.
However, you will have to open NETLINK sockets in each networking
namespace and manage them from the `network/cni` isolator.
JIRA: https://issues.apache.org/jira/browse/MESOS-5647
Review: https://reviews.apache.org/r/68054/
(cherry picked from commit 7708008a3c0c4b3a1b0384c2c739ea258306d671)
---
docs/configuration/agent.md | 10 +++
.../mesos/isolators/network/cni/cni.cpp | 91 ++++++++++++++++++++++
.../mesos/isolators/network/cni/cni.hpp | 12 +++
src/slave/flags.cpp | 6 ++
src/slave/flags.hpp | 1 +
5 files changed, 120 insertions(+)
diff --git a/docs/configuration/agent.md b/docs/configuration/agent.md
index 4e50b68..746a3e8 100644
--- a/docs/configuration/agent.md
+++ b/docs/configuration/agent.md
@@ -1125,6 +1125,16 @@ a network configuration file in JSON format in the specified directory.
</td>
</tr>
+<tr id="network_cni_metrics">
+ <td>
+ --[no-]network_cni_metrics
+ </td>
+ <td>
+This setting controls whether the networking metrics of the CNI isolator should
+be exposed.
+ </td>
+</tr>
+
<tr id="oversubscribed_resources_interval">
<td>
--oversubscribed_resources_interval=VALUE
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
index a743e65..6df52a5 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.cpp
@@ -46,6 +46,14 @@
#include "slave/state.hpp"
+// NOTE: The namespace wrapper prevents symbol collisions.
+// <linux/if_link.h> exports `struct net` that conflicts with
+// `namespace net` from stout library. This section must be
+// the last `#include` in the file.
+namespace netlink {
+#include <linux/if_link.h>
+} // namespace netlink {
+
namespace io = process::io;
namespace paths = mesos::internal::slave::cni::paths;
namespace spec = mesos::internal::slave::cni::spec;
@@ -1488,6 +1496,89 @@ Future<ContainerStatus> NetworkCniIsolatorProcess::status(
}
+Future<ResourceStatistics> NetworkCniIsolatorProcess::usage(
+ const ContainerID& containerId)
+{
+ // Don't expose networking metrics if this feature is disabled.
+ if (!flags.network_cni_metrics) {
+ return ResourceStatistics();
+ }
+
+ // NOTE: We don't keep an Info struct if the container is on the
+ // host network and has no image, or if during recovery, we found
+ // that the cleanup for this container is not required anymore
+ // (e.g., cleanup is done already, but the slave crashed and didn't
+ // realize that it's done).
+ if (!infos.contains(containerId)) {
+ return ResourceStatistics();
+ }
+
+ const string netns = paths::getNamespacePath(rootDir.get(), containerId);
+
+ // We collect networking statistics only for known interfaces.
+ hashmap<string, ContainerNetwork> containerNetworks =
+ infos[containerId]->containerNetworks;
+
+ hashset<string> ifNames;
+ foreachvalue (const ContainerNetwork& containerNetwork,
+ containerNetworks) {
+ ifNames.insert(containerNetwork.ifName);
+ }
+
+ // Call usageFunc with a custom network namespace in the separate thread.
+ lambda::function<Try<ResourceStatistics>()> usageFunc =
+ lambda::bind(&NetworkCniIsolatorProcess::_usage, ifNames);
+ return namespaceRunner.run(netns, "net", usageFunc);
+}
+
+
+Try<ResourceStatistics> NetworkCniIsolatorProcess::_usage(
+ const hashset<string> ifNames)
+{
+ ResourceStatistics result;
+
+ // TODO(urbanserj): To provide more stats insead of getifaddrs(3) we can add
+ // Netlink Protocol Library (libnl) as an optional external dependency.
+ struct ifaddrs* ifaddr = nullptr;
+ if (getifaddrs(&ifaddr) == -1) {
+ return ErrnoError();
+ }
+
+ for (struct ifaddrs* ifa = ifaddr; ifa != nullptr; ifa = ifa->ifa_next) {
+ // Skip unknown network interfaces.
+ if (ifa->ifa_name == nullptr || !ifNames.contains(ifa->ifa_name)) {
+ continue;
+ }
+ // Only interfaces with `AF_PACKET` family have link statistics.
+ if (ifa->ifa_addr->sa_family != AF_PACKET) {
+ continue;
+ }
+
+ struct netlink::rtnl_link_stats *stats =
+ (struct netlink::rtnl_link_stats *)ifa->ifa_data;
+
+ // TODO(urbanserj): Add support for multiple networking interfaces to
+ // ResourceStatistics and collect statistics per `ifa_name`.
+
+ // RX: Receive statistics.
+ result.set_net_rx_packets(result.net_rx_packets() + stats->rx_packets);
+ result.set_net_rx_bytes(result.net_rx_bytes() + stats->rx_bytes);
+ result.set_net_rx_errors(result.net_rx_errors() + stats->rx_errors);
+ result.set_net_rx_dropped(result.net_rx_dropped() + stats->rx_dropped);
+
+ // TX: Transmit statistics.
+ result.set_net_tx_packets(result.net_tx_packets() + stats->tx_packets);
+ result.set_net_tx_bytes(result.net_tx_bytes() + stats->tx_bytes);
+ result.set_net_tx_errors(result.net_tx_errors() + stats->tx_errors);
+ result.set_net_tx_dropped(result.net_tx_dropped() + stats->tx_dropped);
+ }
+
+ freeifaddrs(ifaddr);
+
+ return result;
+}
+
+
Future<Nothing> NetworkCniIsolatorProcess::cleanup(
const ContainerID& containerId)
{
diff --git a/src/slave/containerizer/mesos/isolators/network/cni/cni.hpp b/src/slave/containerizer/mesos/isolators/network/cni/cni.hpp
index 31ec4dd..4ac55c8 100644
--- a/src/slave/containerizer/mesos/isolators/network/cni/cni.hpp
+++ b/src/slave/containerizer/mesos/isolators/network/cni/cni.hpp
@@ -29,6 +29,8 @@
#include "slave/containerizer/mesos/isolators/network/cni/spec.hpp"
#include "slave/containerizer/mesos/isolators/network/cni/paths.hpp"
+#include "linux/ns.hpp"
+
namespace mesos {
namespace internal {
namespace slave {
@@ -68,6 +70,9 @@ public:
process::Future<ContainerStatus> status(
const ContainerID& containerId) override;
+ process::Future<ResourceStatistics> usage(
+ const ContainerID& containerId) override;
+
process::Future<Nothing> cleanup(
const ContainerID& containerId) override;
@@ -183,6 +188,9 @@ private:
const ContainerID& containerId,
const std::vector<process::Future<Nothing>>& detaches);
+ static Try<ResourceStatistics> _usage(
+ const hashset<std::string> ifNames);
+
// Searches the `networkConfigs` hashmap for a CNI network. If the
// hashmap doesn't contain the network, will try to load all the CNI
// configs from `flags.network_cni_config_dir`, and will then
@@ -218,6 +226,10 @@ private:
// Information of CNI networks that each container joins.
hashmap<ContainerID, process::Owned<Info>> infos;
+
+ // Runner manages a separate thread to call `usage` functions
+ // in the containers' namespaces.
+ ns::NamespaceRunner namespaceRunner;
};
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 54d9acc..f1727cd 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -1149,6 +1149,12 @@ mesos::internal::slave::Flags::Flags()
"the operator should install a network configuration file in JSON\n"
"format in the specified directory.");
+ add(&Flags::network_cni_metrics,
+ "network_cni_metrics",
+ "This setting controls whether the networking metrics of the CNI\n"
+ "isolator should be exposed.",
+ true);
+
add(&Flags::container_disk_watch_interval,
"container_disk_watch_interval",
"The interval between disk quota checks for containers. This flag is\n"
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 88c35da..0d1dac8 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -158,6 +158,7 @@ public:
Option<std::string> network_cni_plugins_dir;
Option<std::string> network_cni_config_dir;
+ bool network_cni_metrics;
Duration container_disk_watch_interval;
bool enforce_container_disk_quota;
Option<Modules> modules;