You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2015/01/20 22:56:20 UTC
mesos git commit: Added metrics in PortMappingIsolator for the number
of tcp connections in active or time wait status.
Repository: mesos
Updated Branches:
refs/heads/master 393e82cb0 -> b6b087e57
Added metrics in PortMappingIsolator for the number of tcp connections
in active or time wait status.
Review: https://reviews.apache.org/r/29033
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b6b087e5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b6b087e5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b6b087e5
Branch: refs/heads/master
Commit: b6b087e57ed67abe2c38173299615f90e1b0461e
Parents: 393e82c
Author: Chi Zhang <ch...@gmail.com>
Authored: Tue Jan 20 12:59:46 2015 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Tue Jan 20 13:56:19 2015 -0800
----------------------------------------------------------------------
include/mesos/mesos.proto | 5 +
.../isolators/network/port_mapping.cpp | 220 ++++++++++++++-----
.../isolators/network/port_mapping.hpp | 2 +
src/slave/flags.hpp | 31 ++-
src/tests/port_mapping_tests.cpp | 97 +++++++-
5 files changed, 288 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 859aa2f..26003fa 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -488,6 +488,11 @@ message ResourceStatistics {
optional double net_tcp_rtt_microsecs_p90 = 23;
optional double net_tcp_rtt_microsecs_p95 = 24;
optional double net_tcp_rtt_microsecs_p99 = 25;
+
+ // Total number of active tcp connections in a container.
+ optional double net_tcp_active_connections = 28;
+ // Total number of TIME_WAIT tcp connections in a container.
+ optional double net_tcp_time_wait_connections = 29;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/containerizer/isolators/network/port_mapping.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/isolators/network/port_mapping.cpp b/src/slave/containerizer/isolators/network/port_mapping.cpp
index 2d81336..b484bbf 100644
--- a/src/slave/containerizer/isolators/network/port_mapping.cpp
+++ b/src/slave/containerizer/isolators/network/port_mapping.cpp
@@ -38,11 +38,13 @@
#include <stout/hashset.hpp>
#include <stout/json.hpp>
#include <stout/lambda.hpp>
+#include <stout/numify.hpp>
#include <stout/os.hpp>
#include <stout/option.hpp>
#include <stout/protobuf.hpp>
#include <stout/result.hpp>
#include <stout/stringify.hpp>
+#include <stout/strings.hpp>
#include <stout/os/exists.hpp>
@@ -486,6 +488,17 @@ PortMappingStatistics::Flags::Flags()
add(&pid,
"pid",
"The pid of the process whose namespaces we will enter");
+
+ add(&enable_socket_statistics_summary,
+ "enable_socket_statistics_summary",
+ "Whether to collect socket statistics summary for this container\n",
+ false);
+
+ add(&enable_socket_statistics_details,
+ "enable_socket_statistics_details",
+ "Whether to collect socket statistics details (e.g., TCP RTT)\n"
+ "for this container.",
+ false);
}
@@ -511,55 +524,130 @@ int PortMappingStatistics::execute()
return 1;
}
- // NOTE: If the underlying library uses the older version of kernel
- // API, the family argument passed in may not be honored.
- Try<vector<diagnosis::socket::Info> > infos =
- diagnosis::socket::infos(AF_INET, diagnosis::socket::state::ALL);
+ JSON::Object object;
- if (infos.isError()) {
- cerr << "Failed to retrieve the socket information" << endl;
- }
+ if (flags.enable_socket_statistics_summary) {
+ // Collections for socket statistics summary are below.
- vector<uint32_t> RTTs;
- foreach (const diagnosis::socket::Info& info, infos.get()) {
- // We double check on family regardless.
- if (info.family != AF_INET) {
- continue;
+ // For TCP, get the number of ACTIVE and TIME_WAIT connections,
+ // from reading /proc/net/sockstat (/proc/net/sockstat6 for IPV6).
+ // This is not as expensive in the kernel because only counter
+ // values are accessed instead of a dump of all the sockets.
+ // Example output:
+
+ // $ cat /proc/net/sockstat
+ // sockets: used 1391
+ // TCP: inuse 33 orphan 0 tw 0 alloc 37 mem 6
+ // UDP: inuse 15 mem 7
+ // UDPLITE: inuse 0
+ // RAW: inuse 0
+ // FRAG: inuse 0 memory 0
+
+ Try<string> value = os::read("/proc/net/sockstat");
+ if (value.isError()) {
+ cerr << "Failed to read /proc/net/sockstat: " << value.error() << endl;
+ return 1;
}
- // We consider all sockets that have non-zero rtt value.
- if (info.tcpInfo.isSome() && info.tcpInfo.get().tcpi_rtt != 0) {
- RTTs.push_back(info.tcpInfo.get().tcpi_rtt);
+ foreach (const string& line, strings::tokenize(value.get(), "\n")) {
+ if (!strings::startsWith(line, "TCP")) {
+ continue;
+ }
+
+ vector<string> tokens = strings::tokenize(line, " ");
+ for (size_t i = 0; i < tokens.size(); i++) {
+ if (tokens[i] == "inuse") {
+ if (i + 1 >= tokens.size()) {
+ cerr << "Unexpected output from /proc/net/sockstat" << endl;
+ // Be a bit forgiving here here since the /proc file
+ // output format can change, though not very likely.
+ continue;
+ }
+
+ // Set number of active TCP connections.
+ Try<size_t> inuse = numify<size_t>(tokens[i+1]);
+ if (inuse.isError()) {
+ cerr << "Failed to parse the number of tcp connections in use: "
+ << inuse.error() << endl;
+ continue;
+ }
+
+ object.values["net_tcp_active_connections"] = inuse.get();
+ } else if (tokens[i] == "tw") {
+ if (i + 1 >= tokens.size()) {
+ cerr << "Unexpected output from /proc/net/sockstat" << endl;
+ // Be a bit forgiving here here since the /proc file
+ // output format can change, though not very likely.
+ continue;
+ }
+
+ // Set number of TIME_WAIT TCP connections.
+ Try<size_t> tw = numify<size_t>(tokens[i+1]);
+ if (tw.isError()) {
+ cerr << "Failed to parse the number of tcp connections in"
+ << " TIME_WAIT: " << tw.error() << endl;
+ continue;
+ }
+
+ object.values["net_tcp_time_wait_connections"] = tw.get();
+ }
+ }
}
}
- // Only print to stdout when we have results.
- if (RTTs.size() > 0) {
- std::sort(RTTs.begin(), RTTs.end());
+ if (flags.enable_socket_statistics_details) {
+ // Collections for socket statistics details are below.
+
+ // NOTE: If the underlying library uses the older version of
+ // kernel API, the family argument passed in may not be honored.
+ Try<vector<diagnosis::socket::Info> > infos =
+ diagnosis::socket::infos(AF_INET, diagnosis::socket::state::ALL);
- // NOTE: The size of RTTs is usually within 1 million so we don't
- // need to worry about overflow here.
- // TODO(jieyu): Right now, we choose to use "Nearest rank" for
- // simplicity. Consider directly using the Statistics abstraction
- // which computes "Linear interpolation between closest ranks".
- // http://en.wikipedia.org/wiki/Percentile
- size_t p50 = RTTs.size() * 50 / 100;
- size_t p90 = RTTs.size() * 90 / 100;
- size_t p95 = RTTs.size() * 95 / 100;
- size_t p99 = RTTs.size() * 99 / 100;
+ if (infos.isError()) {
+ cerr << "Failed to retrieve the socket information" << endl;
+ return 1;
+ }
- JSON::Object object;
- object.values["net_tcp_rtt_microsecs_p50"] = RTTs[p50];
- object.values["net_tcp_rtt_microsecs_p90"] = RTTs[p90];
- object.values["net_tcp_rtt_microsecs_p95"] = RTTs[p95];
- object.values["net_tcp_rtt_microsecs_p99"] = RTTs[p99];
+ vector<uint32_t> RTTs;
+ foreach (const diagnosis::socket::Info& info, infos.get()) {
+ // We double check on family regardless.
+ if (info.family != AF_INET) {
+ continue;
+ }
- cout << stringify(object);
+ // We consider all sockets that have non-zero rtt value.
+ if (info.tcpInfo.isSome() && info.tcpInfo.get().tcpi_rtt != 0) {
+ RTTs.push_back(info.tcpInfo.get().tcpi_rtt);
+ }
+ }
+
+ // Only print to stdout when we have results.
+ if (RTTs.size() > 0) {
+ std::sort(RTTs.begin(), RTTs.end());
+
+ // NOTE: The size of RTTs is usually within 1 million so we
+ // don't need to worry about overflow here.
+ // TODO(jieyu): Right now, we choose to use "Nearest rank" for
+ // simplicity. Consider directly using the Statistics abstraction
+ // which computes "Linear interpolation between closest ranks".
+ // http://en.wikipedia.org/wiki/Percentile
+ size_t p50 = RTTs.size() * 50 / 100;
+ size_t p90 = RTTs.size() * 90 / 100;
+ size_t p95 = RTTs.size() * 95 / 100;
+ size_t p99 = RTTs.size() * 99 / 100;
+
+ object.values["net_tcp_rtt_microsecs_p50"] = RTTs[p50];
+ object.values["net_tcp_rtt_microsecs_p90"] = RTTs[p90];
+ object.values["net_tcp_rtt_microsecs_p95"] = RTTs[p95];
+ object.values["net_tcp_rtt_microsecs_p99"] = RTTs[p99];
+ }
}
+ cout << stringify(object);
return 0;
}
+
/////////////////////////////////////////////////
// Implementation for the isolator.
/////////////////////////////////////////////////
@@ -2037,13 +2125,18 @@ Future<ResourceStatistics> PortMappingIsolatorProcess::usage(
result.set_net_tx_dropped(tx_dropped.get());
}
- if (!flags.network_enable_socket_statistics) {
+ if (!flags.network_enable_socket_statistics_summary &&
+ !flags.network_enable_socket_statistics_details) {
return result;
}
// Retrieve the socket information from inside the container.
PortMappingStatistics statistics;
statistics.flags.pid = info->pid.get();
+ statistics.flags.enable_socket_statistics_summary =
+ flags.network_enable_socket_statistics_summary;
+ statistics.flags.enable_socket_statistics_details =
+ flags.network_enable_socket_statistics_details;
vector<string> argv(2);
argv[0] = "mesos-network-helper";
@@ -2109,30 +2202,49 @@ Future<ResourceStatistics> PortMappingIsolatorProcess::__usage(
CHECK_READY(out);
// NOTE: It's possible the subprocess has no output.
- if (out.get().size() > 0) {
- Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
- if (object.isError()) {
- return Failure(
- "Failed to parse the output from the process that gets the "
- "network statistics: " + object.error());
- }
+ if (out.get().empty()) {
+ return result;
+ }
- Result<JSON::Number> p50 =
- object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p50");
- Result<JSON::Number> p90 =
- object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p90");
- Result<JSON::Number> p95 =
- object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p95");
- Result<JSON::Number> p99 =
- object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p99");
-
- if (!p50.isSome() || !p90.isSome() || !p95.isSome() || !p99.isSome()) {
- return Failure("Failed to get TCP RTT statistics");
- }
+ Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
+ if (object.isError()) {
+ return Failure("Failed to parse the output from the process that gets the "
+ "network statistics: " + object.error());
+ }
+ Result<JSON::Number> active =
+ object.get().find<JSON::Number>("net_tcp_active_connections");
+ if (active.isSome()) {
+ result.set_net_tcp_active_connections(active.get().value);
+ }
+
+ Result<JSON::Number> tw =
+ object.get().find<JSON::Number>("net_tcp_time_wait_connections");
+ if (tw.isSome()) {
+ result.set_net_tcp_time_wait_connections(tw.get().value);
+ }
+
+ Result<JSON::Number> p50 =
+ object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p50");
+ if (p50.isSome()) {
result.set_net_tcp_rtt_microsecs_p50(p50.get().value);
+ }
+
+ Result<JSON::Number> p90 =
+ object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p90");
+ if (p90.isSome()) {
result.set_net_tcp_rtt_microsecs_p90(p90.get().value);
+ }
+
+ Result<JSON::Number> p95 =
+ object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p95");
+ if (p95.isSome()) {
result.set_net_tcp_rtt_microsecs_p95(p95.get().value);
+ }
+
+ Result<JSON::Number> p99 =
+ object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p99");
+ if (p99.isSome()) {
result.set_net_tcp_rtt_microsecs_p99(p99.get().value);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/containerizer/isolators/network/port_mapping.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/isolators/network/port_mapping.hpp b/src/slave/containerizer/isolators/network/port_mapping.hpp
index f1e2352..90d19a9 100644
--- a/src/slave/containerizer/isolators/network/port_mapping.hpp
+++ b/src/slave/containerizer/isolators/network/port_mapping.hpp
@@ -347,6 +347,8 @@ public:
bool help;
Option<pid_t> pid;
+ bool enable_socket_statistics_summary;
+ bool enable_socket_statistics_details;
};
PortMappingStatistics() : Subcommand(NAME) {}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 33a0cb6..a3c5c68 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -355,32 +355,44 @@ public:
add(&Flags::ephemeral_ports_per_container,
"ephemeral_ports_per_container",
"Number of ephemeral ports allocated to a container by the network\n"
- "isolator. This number has to be a power of 2.\n",
+ "isolator. This number has to be a power of 2. This flag is used\n"
+ "for the 'network/port_mapping' isolator.",
DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER);
add(&Flags::eth0_name,
"eth0_name",
"The name of the public network interface (e.g., eth0). If it is\n"
"not specified, the network isolator will try to guess it based\n"
- "on the host default gateway.");
+ "on the host default gateway. This flag is used for the\n"
+ "'network/port_mapping' isolator.");
add(&Flags::lo_name,
"lo_name",
"The name of the loopback network interface (e.g., lo). If it is\n"
- "not specified, the network isolator will try to guess it.");
+ "not specified, the network isolator will try to guess it. This\n"
+ "flag is used for the 'network/port_mapping' isolator.");
add(&Flags::egress_rate_limit_per_container,
"egress_rate_limit_per_container",
"The limit of the egress traffic for each container, in Bytes/s.\n"
"If not specified or specified as zero, the network isolator will\n"
"impose no limits to containers' egress traffic throughput.\n"
- "This flag uses the Bytes type, defined in stout.");
+ "This flag uses the Bytes type (defined in stout) and is used for\n"
+ "the 'network/port_mapping' isolator.");
- add(&Flags::network_enable_socket_statistics,
- "network_enable_socket_statistics",
- "Whether to collect socket statistics (e.g., TCP RTT) for\n"
- "each container.",
+ add(&Flags::network_enable_socket_statistics_summary,
+ "network_enable_socket_statistics_summary",
+ "Whether to collect socket statistics summary for each container.\n"
+ "This flag is used for the 'network/port_mapping' isolator.",
false);
+
+ add(&Flags::network_enable_socket_statistics_details,
+ "network_enable_socket_statistics_details",
+ "Whether to collect socket statistics details (e.g., TCP RTT) for\n"
+ "each container. This flag is used for the 'network/port_mapping'\n"
+ "isolator.",
+ false);
+
#endif // WITH_NETWORK_ISOLATOR
add(&Flags::container_disk_watch_interval,
@@ -503,7 +515,8 @@ public:
Option<std::string> eth0_name;
Option<std::string> lo_name;
Option<Bytes> egress_rate_limit_per_container;
- bool network_enable_socket_statistics;
+ bool network_enable_socket_statistics_summary;
+ bool network_enable_socket_statistics_details;
#endif
Duration container_disk_watch_interval;
bool enforce_container_disk_quota;
http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/tests/port_mapping_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/port_mapping_tests.cpp b/src/tests/port_mapping_tests.cpp
index d57d3e6..18d58ff 100644
--- a/src/tests/port_mapping_tests.cpp
+++ b/src/tests/port_mapping_tests.cpp
@@ -23,6 +23,7 @@
#include <vector>
#include <process/future.hpp>
+#include <process/io.hpp>
#include <process/reap.hpp>
#include <process/subprocess.hpp>
@@ -308,6 +309,47 @@ protected:
return pid;
}
+
+ JSON::Object statisticsHelper(
+ pid_t pid,
+ bool enable_summary,
+ bool enable_details)
+ {
+ // Retrieve the socket information from inside the container.
+ PortMappingStatistics statistics;
+ statistics.flags.pid = pid;
+ statistics.flags.enable_socket_statistics_summary = enable_summary;
+ statistics.flags.enable_socket_statistics_details = enable_details;
+
+ vector<string> argv(2);
+ argv[0] = "mesos-network-helper";
+ argv[1] = PortMappingStatistics::NAME;
+
+ // We don't need STDIN; we need STDOUT for the result; we leave
+ // STDERR as is to log to slave process.
+ Try<Subprocess> s = subprocess(
+ path::join(flags.launcher_dir, "mesos-network-helper"),
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PIPE(),
+ Subprocess::FD(STDERR_FILENO),
+ statistics.flags);
+
+ CHECK_SOME(s);
+
+ Future<Option<int> > status = s.get().status();
+ AWAIT_EXPECT_READY(status);
+ EXPECT_SOME_EQ(0, status.get());
+
+ Future<string> out = io::read(s.get().out().get());
+ AWAIT_EXPECT_READY(out);
+
+ Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
+ CHECK_SOME(object);
+
+ return object.get();
+ }
+
slave::Flags flags;
// Name of the host eth0 and lo.
@@ -1442,9 +1484,38 @@ TEST_F(PortMappingIsolatorTest, ROOT_SmallEgressLimitTest)
}
+bool HasTCPSocketsCount(const JSON::Object& object)
+{
+ return object.find<JSON::Number>("net_tcp_active_connections").isSome() &&
+ object.find<JSON::Number>("net_tcp_time_wait_connections").isSome();
+}
+
+
+bool HasTCPSocketsRTT(const JSON::Object& object)
+{
+ Result<JSON::Number> p50 =
+ object.find<JSON::Number>("net_tcp_rtt_microsecs_p50");
+ Result<JSON::Number> p90 =
+ object.find<JSON::Number>("net_tcp_rtt_microsecs_p90");
+ Result<JSON::Number> p95 =
+ object.find<JSON::Number>("net_tcp_rtt_microsecs_p95");
+ Result<JSON::Number> p99 =
+ object.find<JSON::Number>("net_tcp_rtt_microsecs_p99");
+
+ // We either have all of the following metrics or we have nothing.
+ if (!p50.isSome() && !p90.isSome() && !p95.isSome() && !p99.isSome()) {
+ return false;
+ }
+
+ EXPECT_TRUE(p50.isSome() && p90.isSome() && p95.isSome() && p99.isSome());
+
+ return true;
+}
+
+
// Test that RTT can be returned properly from usage(). This test is
-// very similar to SmallEgressLimitTest in its set up.
-TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
+// very similar to SmallEgressLimitTest in its setup.
+TEST_F(PortMappingIsolatorTest, ROOT_PortMappingStatisticsTest)
{
// To-be-tested egress rate limit, in Bytes/s.
const Bytes rate = 2000;
@@ -1453,7 +1524,8 @@ TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
// Use a very small egress limit.
flags.egress_rate_limit_per_container = rate;
- flags.network_enable_socket_statistics = true;
+ flags.network_enable_socket_statistics_summary = true;
+ flags.network_enable_socket_statistics_details = true;
Try<Isolator*> isolator = PortMappingIsolatorProcess::create(flags);
CHECK_SOME(isolator);
@@ -1544,15 +1616,32 @@ TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
os::sleep(Milliseconds(200));
waited += Milliseconds(200);
+ // Do an end-to-end test by calling `usage`.
Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
AWAIT_READY(usage);
- if (usage.get().has_net_tcp_rtt_microsecs_p50()) {
+ if (usage.get().has_net_tcp_rtt_microsecs_p50() &&
+ usage.get().has_net_tcp_active_connections()) {
+ EXPECT_GT(usage.get().net_tcp_active_connections(), 0);
break;
}
} while (waited < Seconds(5));
ASSERT_LT(waited, Seconds(5));
+ // While the connection is still active, try out different flag
+ // combinations.
+ JSON::Object object = statisticsHelper(pid.get(), true, true);
+ ASSERT_TRUE(HasTCPSocketsCount(object) && HasTCPSocketsRTT(object));
+
+ object = statisticsHelper(pid.get(), true, false);
+ ASSERT_TRUE(HasTCPSocketsCount(object) && !HasTCPSocketsRTT(object));
+
+ object = statisticsHelper(pid.get(), false, true);
+ ASSERT_TRUE(!HasTCPSocketsCount(object) && HasTCPSocketsRTT(object));
+
+ object = statisticsHelper(pid.get(), false, false);
+ ASSERT_TRUE(!HasTCPSocketsCount(object) && !HasTCPSocketsRTT(object));
+
// Wait for the command to finish.
while (!os::exists(container1Ready));