You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/11/13 00:26:50 UTC
[1/2] git commit: Improved cgroups isolator to report additional
memory statistics.
Updated Branches:
refs/heads/master 8e13a26e8 -> f45214ab8
Improved cgroups isolator to report additional memory statistics.
From: Eric Biederman <eb...@xmission.com>
Review: https://reviews.apache.org/r/15015
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f45214ab
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f45214ab
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f45214ab
Branch: refs/heads/master
Commit: f45214ab89d3506f6b75f95e3852b1eaf4ed8194
Parents: 9fb5524
Author: Vinod Kone <vi...@twitter.com>
Authored: Tue Nov 12 14:42:09 2013 -0800
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Nov 12 15:00:09 2013 -0800
----------------------------------------------------------------------
include/mesos/mesos.proto | 5 +++++
src/slave/cgroups_isolator.cpp | 18 ++++++++++++++++++
src/slave/monitor.cpp | 36 ++++++++++++++++++++++++++++++++++++
src/tests/monitor_tests.cpp | 12 ++++++++++++
4 files changed, 71 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f45214ab/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 87132af..ad4ba5b 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -284,6 +284,11 @@ message ResourceStatistics {
// Amount of memory resources allocated.
optional uint64 mem_limit_bytes = 6;
+ // Broken out memory usage information (files, anonymous, and mmaped files)
+ optional uint64 mem_file_bytes = 10;
+ optional uint64 mem_anon_bytes = 11;
+ optional uint64 mem_mapped_file_bytes = 12;
+
// TODO(bmahler): Add disk usage.
// TODO(bmahler): Add network usage?
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/f45214ab/src/slave/cgroups_isolator.cpp
----------------------------------------------------------------------
diff --git a/src/slave/cgroups_isolator.cpp b/src/slave/cgroups_isolator.cpp
index a1cf26e..c769ae0 100644
--- a/src/slave/cgroups_isolator.cpp
+++ b/src/slave/cgroups_isolator.cpp
@@ -785,6 +785,24 @@ Future<ResourceStatistics> CgroupsIsolator::usage(
// structure, e.g, cgroups::memory::stat.
result.set_mem_rss_bytes(usage.get().bytes());
+ stat = cgroups::stat(hierarchy, info->name(), "memory.stat");
+ if (stat.isError()) {
+ return Future<ResourceStatistics>::failed(
+ "Failed to read memory.stat: " + stat.error());
+ }
+
+ if (stat.get().contains("total_cache")) {
+ result.set_mem_file_bytes(stat.get()["total_cache"]);
+ }
+
+ if (stat.get().contains("total_rss")) {
+ result.set_mem_anon_bytes(stat.get()["total_rss"]);
+ }
+
+ if (stat.get().contains("total_mapped_file")) {
+ result.set_mem_mapped_file_bytes(stat.get()["total_mapped_file"]);
+ }
+
// Add the cpu.stat information.
stat = cgroups::stat(hierarchy, info->name(), "cpu.stat");
http://git-wip-us.apache.org/repos/asf/mesos/blob/f45214ab/src/slave/monitor.cpp
----------------------------------------------------------------------
diff --git a/src/slave/monitor.cpp b/src/slave/monitor.cpp
index 9cb6256..b49fd05 100644
--- a/src/slave/monitor.cpp
+++ b/src/slave/monitor.cpp
@@ -59,6 +59,9 @@ const std::string CPUS_SYSTEM_TIME_SECS = "cpus_system_time_secs";
const std::string CPUS_LIMIT = "cpus_limit";
const std::string MEM_RSS_BYTES = "mem_rss_bytes";
const std::string MEM_LIMIT_BYTES = "mem_limit_bytes";
+const std::string MEM_FILE_BYTES = "mem_file_bytes";
+const std::string MEM_ANON_BYTES = "mem_anon_bytes";
+const std::string MEM_MAPPED_FILE_BYTES = "mem_mapped_file_bytes";
const std::string CPUS_NR_PERIODS = "cpus_nr_periods";
const std::string CPUS_NR_THROTTLED = "cpus_nr_throttled";
const std::string CPUS_THROTTLED_TIME_SECS = "cpus_throttled_time_secs";
@@ -130,6 +133,9 @@ Future<Nothing> ResourceMonitorProcess::unwatch(
::statistics->archive("monitor", prefix + CPUS_LIMIT);
::statistics->archive("monitor", prefix + MEM_RSS_BYTES);
::statistics->archive("monitor", prefix + MEM_LIMIT_BYTES);
+ ::statistics->archive("monitor", prefix + MEM_FILE_BYTES);
+ ::statistics->archive("monitor", prefix + MEM_ANON_BYTES);
+ ::statistics->archive("monitor", prefix + MEM_MAPPED_FILE_BYTES);
::statistics->archive("monitor", prefix + CPUS_NR_PERIODS);
::statistics->archive("monitor", prefix + CPUS_NR_THROTTLED);
::statistics->archive("monitor", prefix + CPUS_THROTTLED_TIME_SECS);
@@ -255,6 +261,21 @@ void publish(
prefix + MEM_LIMIT_BYTES,
statistics.mem_limit_bytes(),
time);
+ ::statistics->set(
+ "monitor",
+ prefix + MEM_FILE_BYTES,
+ statistics.mem_file_bytes(),
+ time);
+ ::statistics->set(
+ "monitor",
+ prefix + MEM_ANON_BYTES,
+ statistics.mem_anon_bytes(),
+ time);
+ ::statistics->set(
+ "monitor",
+ prefix + MEM_MAPPED_FILE_BYTES,
+ statistics.mem_mapped_file_bytes(),
+ time);
// Publish cpu.stat statistics.
::statistics->set(
@@ -310,6 +331,9 @@ Future<http::Response> _statisticsJSON(
usage.values[CPUS_LIMIT] = 0;
usage.values[MEM_RSS_BYTES] = 0;
usage.values[MEM_LIMIT_BYTES] = 0;
+ usage.values[MEM_FILE_BYTES] = 0;
+ usage.values[MEM_ANON_BYTES] = 0;
+ usage.values[MEM_MAPPED_FILE_BYTES] = 0;
usage.values[CPUS_NR_PERIODS] = 0;
usage.values[CPUS_NR_THROTTLED] = 0;
usage.values[CPUS_THROTTLED_TIME_SECS] = 0;
@@ -336,6 +360,18 @@ Future<http::Response> _statisticsJSON(
usage.values[MEM_LIMIT_BYTES] =
statistics.find(prefix + MEM_LIMIT_BYTES)->second;
}
+ if (statistics.count(prefix + MEM_FILE_BYTES) > 0) {
+ usage.values[MEM_FILE_BYTES] =
+ statistics.find(prefix + MEM_FILE_BYTES)->second;
+ }
+ if (statistics.count(prefix + MEM_ANON_BYTES) > 0) {
+ usage.values[MEM_ANON_BYTES] =
+ statistics.find(prefix + MEM_ANON_BYTES)->second;
+ }
+ if (statistics.count(prefix + MEM_MAPPED_FILE_BYTES) > 0) {
+ usage.values[MEM_MAPPED_FILE_BYTES] =
+ statistics.find(prefix + MEM_MAPPED_FILE_BYTES)->second;
+ }
// Set the cpu.stat data if present.
if (statistics.count(prefix + CPUS_NR_PERIODS) > 0) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/f45214ab/src/tests/monitor_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/monitor_tests.cpp b/src/tests/monitor_tests.cpp
index 3d3f8af..a341893 100644
--- a/src/tests/monitor_tests.cpp
+++ b/src/tests/monitor_tests.cpp
@@ -77,6 +77,9 @@ TEST(MonitorTest, WatchUnwatch)
initialStatistics.set_cpus_system_time_secs(0);
initialStatistics.set_cpus_limit(2.5);
initialStatistics.set_mem_rss_bytes(0);
+ initialStatistics.set_mem_file_bytes(0);
+ initialStatistics.set_mem_anon_bytes(0);
+ initialStatistics.set_mem_mapped_file_bytes(0);
initialStatistics.set_mem_limit_bytes(2048);
initialStatistics.set_timestamp(Clock::now().secs());
@@ -88,6 +91,9 @@ TEST(MonitorTest, WatchUnwatch)
statistics.set_cpus_throttled_time_secs(0.5);
statistics.set_cpus_limit(2.5);
statistics.set_mem_rss_bytes(1024);
+ statistics.set_mem_file_bytes(512);
+ statistics.set_mem_anon_bytes(512);
+ statistics.set_mem_mapped_file_bytes(256);
statistics.set_mem_limit_bytes(2048);
statistics.set_timestamp(
initialStatistics.timestamp() +
@@ -191,7 +197,10 @@ TEST(MonitorTest, WatchUnwatch)
"\"cpus_system_time_secs\":%g,"
"\"cpus_throttled_time_secs\":%g,"
"\"cpus_user_time_secs\":%g,"
+ "\"mem_anon_bytes\":%lu,"
+ "\"mem_file_bytes\":%lu,"
"\"mem_limit_bytes\":%lu,"
+ "\"mem_mapped_file_bytes\":%lu,"
"\"mem_rss_bytes\":%lu"
"}"
"}]",
@@ -201,7 +210,10 @@ TEST(MonitorTest, WatchUnwatch)
statistics.cpus_system_time_secs(),
statistics.cpus_throttled_time_secs(),
statistics.cpus_user_time_secs(),
+ statistics.mem_anon_bytes(),
+ statistics.mem_file_bytes(),
statistics.mem_limit_bytes(),
+ statistics.mem_mapped_file_bytes(),
statistics.mem_rss_bytes()).get(),
response);
[2/2] git commit: Expanded stout::format() to handle 11 parameters.
Posted by vi...@apache.org.
Expanded stout::format() to handle 11 parameters.
From: Eric Biederman <eb...@xmission.com>
Review: https://reviews.apache.org/r/15013
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9fb55241
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9fb55241
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9fb55241
Branch: refs/heads/master
Commit: 9fb5524171fb995e86ab2decf885dd06cda3bc95
Parents: 8e13a26
Author: Vinod Kone <vi...@twitter.com>
Authored: Tue Nov 12 14:59:41 2013 -0800
Committer: Vinod Kone <vi...@twitter.com>
Committed: Tue Nov 12 15:00:09 2013 -0800
----------------------------------------------------------------------
.../3rdparty/stout/include/stout/format.hpp | 39 ++++++++++++++++++++
1 file changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9fb55241/3rdparty/libprocess/3rdparty/stout/include/stout/format.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/stout/format.hpp b/3rdparty/libprocess/3rdparty/stout/include/stout/format.hpp
index 71b5986..cae7fcb 100644
--- a/3rdparty/libprocess/3rdparty/stout/include/stout/format.hpp
+++ b/3rdparty/libprocess/3rdparty/stout/include/stout/format.hpp
@@ -279,6 +279,45 @@ Try<std::string> format(const std::string& s,
internal::stringify<T9, !std::tr1::is_pod<T9>::value>(t9).get(),
internal::stringify<T10, !std::tr1::is_pod<T10>::value>(t10).get());
}
+
+template <typename T1,
+ typename T2,
+ typename T3,
+ typename T4,
+ typename T5,
+ typename T6,
+ typename T7,
+ typename T8,
+ typename T9,
+ typename T10,
+ typename T11>
+Try<std::string> format(const std::string& s,
+ const T1& t1,
+ const T2& t2,
+ const T3& t3,
+ const T4& t4,
+ const T5& t5,
+ const T6& t6,
+ const T7& t7,
+ const T8& t8,
+ const T9& t9,
+ const T10& t10,
+ const T11& t11)
+{
+ return internal::format(
+ s,
+ internal::stringify<T1, !std::tr1::is_pod<T1>::value>(t1).get(),
+ internal::stringify<T2, !std::tr1::is_pod<T2>::value>(t2).get(),
+ internal::stringify<T3, !std::tr1::is_pod<T3>::value>(t3).get(),
+ internal::stringify<T4, !std::tr1::is_pod<T4>::value>(t4).get(),
+ internal::stringify<T5, !std::tr1::is_pod<T5>::value>(t5).get(),
+ internal::stringify<T6, !std::tr1::is_pod<T6>::value>(t6).get(),
+ internal::stringify<T7, !std::tr1::is_pod<T7>::value>(t7).get(),
+ internal::stringify<T8, !std::tr1::is_pod<T8>::value>(t8).get(),
+ internal::stringify<T9, !std::tr1::is_pod<T9>::value>(t9).get(),
+ internal::stringify<T10, !std::tr1::is_pod<T10>::value>(t10).get(),
+ internal::stringify<T11, !std::tr1::is_pod<T11>::value>(t11).get());
+}
#endif // __cplusplus >= 201103L