You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2015/01/20 22:56:20 UTC

mesos git commit: Added metrics in PortMappingIsolator for the number of tcp connections in active or time wait status.

Repository: mesos
Updated Branches:
  refs/heads/master 393e82cb0 -> b6b087e57


Added metrics in PortMappingIsolator for the number of tcp connections
in active or time wait status.

Review: https://reviews.apache.org/r/29033


Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b6b087e5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b6b087e5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b6b087e5

Branch: refs/heads/master
Commit: b6b087e57ed67abe2c38173299615f90e1b0461e
Parents: 393e82c
Author: Chi Zhang <ch...@gmail.com>
Authored: Tue Jan 20 12:59:46 2015 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Tue Jan 20 13:56:19 2015 -0800

----------------------------------------------------------------------
 include/mesos/mesos.proto                       |   5 +
 .../isolators/network/port_mapping.cpp          | 220 ++++++++++++++-----
 .../isolators/network/port_mapping.hpp          |   2 +
 src/slave/flags.hpp                             |  31 ++-
 src/tests/port_mapping_tests.cpp                |  97 +++++++-
 5 files changed, 288 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 859aa2f..26003fa 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -488,6 +488,11 @@ message ResourceStatistics {
   optional double net_tcp_rtt_microsecs_p90 = 23;
   optional double net_tcp_rtt_microsecs_p95 = 24;
   optional double net_tcp_rtt_microsecs_p99 = 25;
+
+  // Total number of active tcp connections in a container.
+  optional double net_tcp_active_connections = 28;
+  // Total number of TIME_WAIT tcp connections in a container.
+  optional double net_tcp_time_wait_connections = 29;
 }
 
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/containerizer/isolators/network/port_mapping.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/isolators/network/port_mapping.cpp b/src/slave/containerizer/isolators/network/port_mapping.cpp
index 2d81336..b484bbf 100644
--- a/src/slave/containerizer/isolators/network/port_mapping.cpp
+++ b/src/slave/containerizer/isolators/network/port_mapping.cpp
@@ -38,11 +38,13 @@
 #include <stout/hashset.hpp>
 #include <stout/json.hpp>
 #include <stout/lambda.hpp>
+#include <stout/numify.hpp>
 #include <stout/os.hpp>
 #include <stout/option.hpp>
 #include <stout/protobuf.hpp>
 #include <stout/result.hpp>
 #include <stout/stringify.hpp>
+#include <stout/strings.hpp>
 
 #include <stout/os/exists.hpp>
 
@@ -486,6 +488,17 @@ PortMappingStatistics::Flags::Flags()
   add(&pid,
       "pid",
       "The pid of the process whose namespaces we will enter");
+
+  add(&enable_socket_statistics_summary,
+      "enable_socket_statistics_summary",
+      "Whether to collect socket statistics summary for this container\n",
+      false);
+
+  add(&enable_socket_statistics_details,
+      "enable_socket_statistics_details",
+      "Whether to collect socket statistics details (e.g., TCP RTT)\n"
+      "for this container.",
+      false);
 }
 
 
@@ -511,55 +524,130 @@ int PortMappingStatistics::execute()
     return 1;
   }
 
-  // NOTE: If the underlying library uses the older version of kernel
-  // API, the family argument passed in may not be honored.
-  Try<vector<diagnosis::socket::Info> > infos =
-    diagnosis::socket::infos(AF_INET, diagnosis::socket::state::ALL);
+  JSON::Object object;
 
-  if (infos.isError()) {
-    cerr << "Failed to retrieve the socket information" << endl;
-  }
+  if (flags.enable_socket_statistics_summary) {
+    // Collections for socket statistics summary are below.
 
-  vector<uint32_t> RTTs;
-  foreach (const diagnosis::socket::Info& info, infos.get()) {
-    // We double check on family regardless.
-    if (info.family != AF_INET) {
-      continue;
+    // For TCP, get the number of ACTIVE and TIME_WAIT connections,
+    // from reading /proc/net/sockstat (/proc/net/sockstat6 for IPV6).
+    // This is not as expensive in the kernel because only counter
+    // values are accessed instead of a dump of all the sockets.
+    // Example output:
+
+    // $ cat /proc/net/sockstat
+    // sockets: used 1391
+    // TCP: inuse 33 orphan 0 tw 0 alloc 37 mem 6
+    // UDP: inuse 15 mem 7
+    // UDPLITE: inuse 0
+    // RAW: inuse 0
+    // FRAG: inuse 0 memory 0
+
+    Try<string> value = os::read("/proc/net/sockstat");
+    if (value.isError()) {
+      cerr << "Failed to read /proc/net/sockstat: " << value.error() << endl;
+      return 1;
     }
 
-    // We consider all sockets that have non-zero rtt value.
-    if (info.tcpInfo.isSome() && info.tcpInfo.get().tcpi_rtt != 0) {
-      RTTs.push_back(info.tcpInfo.get().tcpi_rtt);
+    foreach (const string& line, strings::tokenize(value.get(), "\n")) {
+      if (!strings::startsWith(line, "TCP")) {
+        continue;
+      }
+
+      vector<string> tokens = strings::tokenize(line, " ");
+      for (size_t i = 0; i < tokens.size(); i++) {
+        if (tokens[i] == "inuse") {
+          if (i + 1 >= tokens.size()) {
+            cerr << "Unexpected output from /proc/net/sockstat" << endl;
+            // Be a bit forgiving here here since the /proc file
+            // output format can change, though not very likely.
+            continue;
+          }
+
+          // Set number of active TCP connections.
+          Try<size_t> inuse = numify<size_t>(tokens[i+1]);
+          if (inuse.isError()) {
+            cerr << "Failed to parse the number of tcp connections in use: "
+                 << inuse.error() << endl;
+            continue;
+          }
+
+          object.values["net_tcp_active_connections"] = inuse.get();
+        } else if (tokens[i] == "tw") {
+          if (i + 1 >= tokens.size()) {
+            cerr << "Unexpected output from /proc/net/sockstat" << endl;
+            // Be a bit forgiving here here since the /proc file
+            // output format can change, though not very likely.
+            continue;
+          }
+
+          // Set number of TIME_WAIT TCP connections.
+          Try<size_t> tw = numify<size_t>(tokens[i+1]);
+          if (tw.isError()) {
+            cerr << "Failed to parse the number of tcp connections in"
+                 << " TIME_WAIT: " << tw.error() << endl;
+            continue;
+          }
+
+          object.values["net_tcp_time_wait_connections"] = tw.get();
+        }
+      }
     }
   }
 
-  // Only print to stdout when we have results.
-  if (RTTs.size() > 0) {
-    std::sort(RTTs.begin(), RTTs.end());
+  if (flags.enable_socket_statistics_details) {
+    // Collections for socket statistics details are below.
+
+    // NOTE: If the underlying library uses the older version of
+    // kernel API, the family argument passed in may not be honored.
+    Try<vector<diagnosis::socket::Info> > infos =
+      diagnosis::socket::infos(AF_INET, diagnosis::socket::state::ALL);
 
-    // NOTE: The size of RTTs is usually within 1 million so we don't
-    // need to worry about overflow here.
-    // TODO(jieyu): Right now, we choose to use "Nearest rank" for
-    // simplicity. Consider directly using the Statistics abstraction
-    // which computes "Linear interpolation between closest ranks".
-    // http://en.wikipedia.org/wiki/Percentile
-    size_t p50 = RTTs.size() * 50 / 100;
-    size_t p90 = RTTs.size() * 90 / 100;
-    size_t p95 = RTTs.size() * 95 / 100;
-    size_t p99 = RTTs.size() * 99 / 100;
+    if (infos.isError()) {
+      cerr << "Failed to retrieve the socket information" << endl;
+      return 1;
+    }
 
-    JSON::Object object;
-    object.values["net_tcp_rtt_microsecs_p50"] = RTTs[p50];
-    object.values["net_tcp_rtt_microsecs_p90"] = RTTs[p90];
-    object.values["net_tcp_rtt_microsecs_p95"] = RTTs[p95];
-    object.values["net_tcp_rtt_microsecs_p99"] = RTTs[p99];
+    vector<uint32_t> RTTs;
+    foreach (const diagnosis::socket::Info& info, infos.get()) {
+      // We double check on family regardless.
+      if (info.family != AF_INET) {
+        continue;
+      }
 
-    cout << stringify(object);
+      // We consider all sockets that have non-zero rtt value.
+      if (info.tcpInfo.isSome() && info.tcpInfo.get().tcpi_rtt != 0) {
+        RTTs.push_back(info.tcpInfo.get().tcpi_rtt);
+      }
+    }
+
+    // Only print to stdout when we have results.
+    if (RTTs.size() > 0) {
+      std::sort(RTTs.begin(), RTTs.end());
+
+      // NOTE: The size of RTTs is usually within 1 million so we
+      // don't need to worry about overflow here.
+      // TODO(jieyu): Right now, we choose to use "Nearest rank" for
+      // simplicity. Consider directly using the Statistics abstraction
+      // which computes "Linear interpolation between closest ranks".
+      // http://en.wikipedia.org/wiki/Percentile
+      size_t p50 = RTTs.size() * 50 / 100;
+      size_t p90 = RTTs.size() * 90 / 100;
+      size_t p95 = RTTs.size() * 95 / 100;
+      size_t p99 = RTTs.size() * 99 / 100;
+
+      object.values["net_tcp_rtt_microsecs_p50"] = RTTs[p50];
+      object.values["net_tcp_rtt_microsecs_p90"] = RTTs[p90];
+      object.values["net_tcp_rtt_microsecs_p95"] = RTTs[p95];
+      object.values["net_tcp_rtt_microsecs_p99"] = RTTs[p99];
+    }
   }
 
+  cout << stringify(object);
   return 0;
 }
 
+
 /////////////////////////////////////////////////
 // Implementation for the isolator.
 /////////////////////////////////////////////////
@@ -2037,13 +2125,18 @@ Future<ResourceStatistics> PortMappingIsolatorProcess::usage(
     result.set_net_tx_dropped(tx_dropped.get());
   }
 
-  if (!flags.network_enable_socket_statistics) {
+  if (!flags.network_enable_socket_statistics_summary &&
+      !flags.network_enable_socket_statistics_details) {
     return result;
   }
 
   // Retrieve the socket information from inside the container.
   PortMappingStatistics statistics;
   statistics.flags.pid = info->pid.get();
+  statistics.flags.enable_socket_statistics_summary =
+    flags.network_enable_socket_statistics_summary;
+  statistics.flags.enable_socket_statistics_details =
+    flags.network_enable_socket_statistics_details;
 
   vector<string> argv(2);
   argv[0] = "mesos-network-helper";
@@ -2109,30 +2202,49 @@ Future<ResourceStatistics> PortMappingIsolatorProcess::__usage(
   CHECK_READY(out);
 
   // NOTE: It's possible the subprocess has no output.
-  if (out.get().size() > 0) {
-    Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
-    if (object.isError()) {
-      return Failure(
-          "Failed to parse the output from the process that gets the "
-          "network statistics: " + object.error());
-    }
+  if (out.get().empty()) {
+    return result;
+  }
 
-    Result<JSON::Number> p50 =
-      object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p50");
-    Result<JSON::Number> p90 =
-      object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p90");
-    Result<JSON::Number> p95 =
-      object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p95");
-    Result<JSON::Number> p99 =
-      object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p99");
-
-    if (!p50.isSome() || !p90.isSome() || !p95.isSome() || !p99.isSome()) {
-      return Failure("Failed to get TCP RTT statistics");
-    }
+  Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
+  if (object.isError()) {
+    return Failure("Failed to parse the output from the process that gets the "
+       "network statistics: " + object.error());
+  }
 
+  Result<JSON::Number> active =
+    object.get().find<JSON::Number>("net_tcp_active_connections");
+  if (active.isSome()) {
+    result.set_net_tcp_active_connections(active.get().value);
+  }
+
+  Result<JSON::Number> tw =
+    object.get().find<JSON::Number>("net_tcp_time_wait_connections");
+  if (tw.isSome()) {
+    result.set_net_tcp_time_wait_connections(tw.get().value);
+  }
+
+  Result<JSON::Number> p50 =
+    object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p50");
+  if (p50.isSome()) {
     result.set_net_tcp_rtt_microsecs_p50(p50.get().value);
+  }
+
+  Result<JSON::Number> p90 =
+    object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p90");
+  if (p90.isSome()) {
     result.set_net_tcp_rtt_microsecs_p90(p90.get().value);
+  }
+
+  Result<JSON::Number> p95 =
+    object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p95");
+  if (p95.isSome()) {
     result.set_net_tcp_rtt_microsecs_p95(p95.get().value);
+  }
+
+  Result<JSON::Number> p99 =
+    object.get().find<JSON::Number>("net_tcp_rtt_microsecs_p99");
+  if (p99.isSome()) {
     result.set_net_tcp_rtt_microsecs_p99(p99.get().value);
   }
 

http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/containerizer/isolators/network/port_mapping.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/isolators/network/port_mapping.hpp b/src/slave/containerizer/isolators/network/port_mapping.hpp
index f1e2352..90d19a9 100644
--- a/src/slave/containerizer/isolators/network/port_mapping.hpp
+++ b/src/slave/containerizer/isolators/network/port_mapping.hpp
@@ -347,6 +347,8 @@ public:
 
     bool help;
     Option<pid_t> pid;
+    bool enable_socket_statistics_summary;
+    bool enable_socket_statistics_details;
   };
 
   PortMappingStatistics() : Subcommand(NAME) {}

http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 33a0cb6..a3c5c68 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -355,32 +355,44 @@ public:
     add(&Flags::ephemeral_ports_per_container,
         "ephemeral_ports_per_container",
         "Number of ephemeral ports allocated to a container by the network\n"
-        "isolator. This number has to be a power of 2.\n",
+        "isolator. This number has to be a power of 2. This flag is used\n"
+        "for the 'network/port_mapping' isolator.",
         DEFAULT_EPHEMERAL_PORTS_PER_CONTAINER);
 
     add(&Flags::eth0_name,
         "eth0_name",
         "The name of the public network interface (e.g., eth0). If it is\n"
         "not specified, the network isolator will try to guess it based\n"
-        "on the host default gateway.");
+        "on the host default gateway. This flag is used for the\n"
+        "'network/port_mapping' isolator.");
 
     add(&Flags::lo_name,
         "lo_name",
         "The name of the loopback network interface (e.g., lo). If it is\n"
-        "not specified, the network isolator will try to guess it.");
+        "not specified, the network isolator will try to guess it. This\n"
+        "flag is used for the 'network/port_mapping' isolator.");
 
     add(&Flags::egress_rate_limit_per_container,
         "egress_rate_limit_per_container",
         "The limit of the egress traffic for each container, in Bytes/s.\n"
         "If not specified or specified as zero, the network isolator will\n"
         "impose no limits to containers' egress traffic throughput.\n"
-        "This flag uses the Bytes type, defined in stout.");
+        "This flag uses the Bytes type (defined in stout) and is used for\n"
+        "the 'network/port_mapping' isolator.");
 
-    add(&Flags::network_enable_socket_statistics,
-        "network_enable_socket_statistics",
-        "Whether to collect socket statistics (e.g., TCP RTT) for\n"
-        "each container.",
+    add(&Flags::network_enable_socket_statistics_summary,
+        "network_enable_socket_statistics_summary",
+        "Whether to collect socket statistics summary for each container.\n"
+        "This flag is used for the 'network/port_mapping' isolator.",
         false);
+
+    add(&Flags::network_enable_socket_statistics_details,
+        "network_enable_socket_statistics_details",
+        "Whether to collect socket statistics details (e.g., TCP RTT) for\n"
+        "each container. This flag is used for the 'network/port_mapping'\n"
+        "isolator.",
+        false);
+
 #endif // WITH_NETWORK_ISOLATOR
 
     add(&Flags::container_disk_watch_interval,
@@ -503,7 +515,8 @@ public:
   Option<std::string> eth0_name;
   Option<std::string> lo_name;
   Option<Bytes> egress_rate_limit_per_container;
-  bool network_enable_socket_statistics;
+  bool network_enable_socket_statistics_summary;
+  bool network_enable_socket_statistics_details;
 #endif
   Duration container_disk_watch_interval;
   bool enforce_container_disk_quota;

http://git-wip-us.apache.org/repos/asf/mesos/blob/b6b087e5/src/tests/port_mapping_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/port_mapping_tests.cpp b/src/tests/port_mapping_tests.cpp
index d57d3e6..18d58ff 100644
--- a/src/tests/port_mapping_tests.cpp
+++ b/src/tests/port_mapping_tests.cpp
@@ -23,6 +23,7 @@
 #include <vector>
 
 #include <process/future.hpp>
+#include <process/io.hpp>
 #include <process/reap.hpp>
 #include <process/subprocess.hpp>
 
@@ -308,6 +309,47 @@ protected:
     return pid;
   }
 
+
+  JSON::Object statisticsHelper(
+      pid_t pid,
+      bool enable_summary,
+      bool enable_details)
+  {
+    // Retrieve the socket information from inside the container.
+    PortMappingStatistics statistics;
+    statistics.flags.pid = pid;
+    statistics.flags.enable_socket_statistics_summary = enable_summary;
+    statistics.flags.enable_socket_statistics_details = enable_details;
+
+    vector<string> argv(2);
+    argv[0] = "mesos-network-helper";
+    argv[1] = PortMappingStatistics::NAME;
+
+    // We don't need STDIN; we need STDOUT for the result; we leave
+    // STDERR as is to log to slave process.
+    Try<Subprocess> s = subprocess(
+        path::join(flags.launcher_dir, "mesos-network-helper"),
+        argv,
+        Subprocess::PATH("/dev/null"),
+        Subprocess::PIPE(),
+        Subprocess::FD(STDERR_FILENO),
+        statistics.flags);
+
+    CHECK_SOME(s);
+
+    Future<Option<int> > status = s.get().status();
+    AWAIT_EXPECT_READY(status);
+    EXPECT_SOME_EQ(0, status.get());
+
+    Future<string> out = io::read(s.get().out().get());
+    AWAIT_EXPECT_READY(out);
+
+    Try<JSON::Object> object = JSON::parse<JSON::Object>(out.get());
+    CHECK_SOME(object);
+
+    return object.get();
+  }
+
   slave::Flags flags;
 
   // Name of the host eth0 and lo.
@@ -1442,9 +1484,38 @@ TEST_F(PortMappingIsolatorTest, ROOT_SmallEgressLimitTest)
 }
 
 
+bool HasTCPSocketsCount(const JSON::Object& object)
+{
+  return object.find<JSON::Number>("net_tcp_active_connections").isSome() &&
+    object.find<JSON::Number>("net_tcp_time_wait_connections").isSome();
+}
+
+
+bool HasTCPSocketsRTT(const JSON::Object& object)
+{
+  Result<JSON::Number> p50 =
+    object.find<JSON::Number>("net_tcp_rtt_microsecs_p50");
+  Result<JSON::Number> p90 =
+    object.find<JSON::Number>("net_tcp_rtt_microsecs_p90");
+  Result<JSON::Number> p95 =
+    object.find<JSON::Number>("net_tcp_rtt_microsecs_p95");
+  Result<JSON::Number> p99 =
+    object.find<JSON::Number>("net_tcp_rtt_microsecs_p99");
+
+  // We either have all of the following metrics or we have nothing.
+  if (!p50.isSome() && !p90.isSome() && !p95.isSome() && !p99.isSome()) {
+    return false;
+  }
+
+  EXPECT_TRUE(p50.isSome() && p90.isSome() && p95.isSome() && p99.isSome());
+
+  return true;
+}
+
+
 // Test that RTT can be returned properly from usage(). This test is
-// very similar to SmallEgressLimitTest in its set up.
-TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
+// very similar to SmallEgressLimitTest in its setup.
+TEST_F(PortMappingIsolatorTest, ROOT_PortMappingStatisticsTest)
 {
   // To-be-tested egress rate limit, in Bytes/s.
   const Bytes rate = 2000;
@@ -1453,7 +1524,8 @@ TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
 
   // Use a very small egress limit.
   flags.egress_rate_limit_per_container = rate;
-  flags.network_enable_socket_statistics = true;
+  flags.network_enable_socket_statistics_summary = true;
+  flags.network_enable_socket_statistics_details = true;
 
   Try<Isolator*> isolator = PortMappingIsolatorProcess::create(flags);
   CHECK_SOME(isolator);
@@ -1544,15 +1616,32 @@ TEST_F(PortMappingIsolatorTest, ROOT_ExportRTTTest)
     os::sleep(Milliseconds(200));
     waited += Milliseconds(200);
 
+    // Do an end-to-end test by calling `usage`.
     Future<ResourceStatistics> usage = isolator.get()->usage(containerId);
     AWAIT_READY(usage);
 
-    if (usage.get().has_net_tcp_rtt_microsecs_p50()) {
+    if (usage.get().has_net_tcp_rtt_microsecs_p50() &&
+        usage.get().has_net_tcp_active_connections()) {
+      EXPECT_GT(usage.get().net_tcp_active_connections(), 0);
       break;
     }
   } while (waited < Seconds(5));
   ASSERT_LT(waited, Seconds(5));
 
+  // While the connection is still active, try out different flag
+  // combinations.
+  JSON::Object object = statisticsHelper(pid.get(), true, true);
+  ASSERT_TRUE(HasTCPSocketsCount(object) && HasTCPSocketsRTT(object));
+
+  object = statisticsHelper(pid.get(), true, false);
+  ASSERT_TRUE(HasTCPSocketsCount(object) && !HasTCPSocketsRTT(object));
+
+  object = statisticsHelper(pid.get(), false, true);
+  ASSERT_TRUE(!HasTCPSocketsCount(object) && HasTCPSocketsRTT(object));
+
+  object = statisticsHelper(pid.get(), false, false);
+  ASSERT_TRUE(!HasTCPSocketsCount(object) && !HasTCPSocketsRTT(object));
+
   // Wait for the command to finish.
   while (!os::exists(container1Ready));