You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2013/02/25 20:44:43 UTC
svn commit: r1449837 - in /incubator/mesos/trunk/third_party/libprocess:
Makefile.am include/process/statistics.hpp include/stout/owned.hpp
src/statistics.cpp src/tests/statistics_tests.cpp
Author: vinodkone
Date: Mon Feb 25 19:44:43 2013
New Revision: 1449837
URL: http://svn.apache.org/r1449837
Log:
Resource Monitoring 4: Added metering to statistics.
We'll need to compute metered data on top incoming raw data. The challenge is that some meters will require previous data:
For example:
1. cpu_time = 10 @ time 10;
2. cpu_time = 20 @ time 20; --> cpu_usage = 1.0 (100%) this is 20-10 / 20-10
3. cpu_time = 25 @ time 30; --> cpu_usage = 0.5 (50%) this is 25-20 / 30-20
From: Ben Mahler <be...@gmail.com>
Review: https://reviews.apache.org/r/9092
Added:
incubator/mesos/trunk/third_party/libprocess/include/stout/owned.hpp
Modified:
incubator/mesos/trunk/third_party/libprocess/Makefile.am
incubator/mesos/trunk/third_party/libprocess/include/process/statistics.hpp
incubator/mesos/trunk/third_party/libprocess/src/statistics.cpp
incubator/mesos/trunk/third_party/libprocess/src/tests/statistics_tests.cpp
Modified: incubator/mesos/trunk/third_party/libprocess/Makefile.am
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/Makefile.am?rev=1449837&r1=1449836&r2=1449837&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/Makefile.am (original)
+++ incubator/mesos/trunk/third_party/libprocess/Makefile.am Mon Feb 25 19:44:43 2013
@@ -87,6 +87,7 @@ libprocess_la_SOURCES += $(top_srcdir)/i
$(top_srcdir)/include/stout/numify.hpp \
$(top_srcdir)/include/stout/option.hpp \
$(top_srcdir)/include/stout/os.hpp \
+ $(top_srcdir)/include/stout/owned.hpp \
$(top_srcdir)/include/stout/path.hpp \
$(top_srcdir)/include/stout/protobuf.hpp \
$(top_srcdir)/include/stout/result.hpp \
Modified: incubator/mesos/trunk/third_party/libprocess/include/process/statistics.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/include/process/statistics.hpp?rev=1449837&r1=1449836&r2=1449837&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/include/process/statistics.hpp (original)
+++ incubator/mesos/trunk/third_party/libprocess/include/process/statistics.hpp Mon Feb 25 19:44:43 2013
@@ -6,7 +6,9 @@
#include <stout/duration.hpp>
#include <stout/none.hpp>
+#include <stout/nothing.hpp>
#include <stout/option.hpp>
+#include <stout/owned.hpp>
namespace process {
@@ -14,6 +16,11 @@ namespace process {
class Statistics;
class StatisticsProcess;
+namespace meters {
+ class Meter;
+ class TimeRate;
+}
+
// Libprocess statistics handle.
// To be used from anywhere to manage statistics.
@@ -28,6 +35,9 @@ extern Statistics* statistics;
// Provides an in-memory time series of statistics over some window
// (values are truncated outside of the window, but no limit is
// currently placed on the number of values within a window).
+// TODO(bmahler): We need to consider truncation when we have
+// statistics that are ephemeral. They stop being updated and therefore
+// never get truncated when the window passes.
class Statistics
{
public:
@@ -41,6 +51,16 @@ public:
const Option<Seconds>& start = None(),
const Option<Seconds>& stop = None());
+ // Adds a meter for the statistic with the provided context and name.
+ // get(context, meter->name) will return the metered time series.
+ // Returns an error if:
+ // -meter->name == name, or
+ // -The meter already exists.
+ Future<Try<Nothing> > meter(
+ const std::string& context,
+ const std::string& name,
+ Owned<meters::Meter> meter);
+
// Sets the current value of a statistic at the current clock time
// or at a specified time.
void set(
@@ -61,9 +81,54 @@ private:
StatisticsProcess* process;
};
-// Global statistics instance provided by libprocess.
-extern Statistics* _statistics;
+namespace meters {
+
+// This is the interface for statistical meters.
+// Meters provide additional metering on top of the raw statistical
+// value. Ex: Track the maximum, average, rate, etc.
+class Meter
+{
+protected:
+ Meter(const std::string& _name) : name(_name) {}
+
+public:
+ virtual ~Meter() {}
+
+ // Updates the meter with another input value.
+ // Returns the new metered value, or none if no metered value can be produced.
+ virtual Option<double> update(const Seconds& time, double value) = 0;
+
+ const std::string name;
+};
+
+
+// Tracks the percent of time 'used' since the last update.
+// Input values to this meter must be in seconds.
+class TimeRate : public Meter
+{
+public:
+ TimeRate(const std::string& name) : Meter(name), time(-1), value(0) {}
+ virtual ~TimeRate() {}
+
+ virtual Option<double> update(const Seconds& _time, double _value)
+ {
+ Option<double> rate;
+ if (time.secs() > 0) {
+ rate = (_value - value) / (_time.secs() - time.secs());
+ }
+
+ time = _time;
+ value = _value;
+ return rate;
+ }
+
+private:
+ Seconds time;
+ double value;
+};
+
+} // namespace meters {
} // namespace process {
#endif // __PROCESS_STATISTICS_HPP__
Added: incubator/mesos/trunk/third_party/libprocess/include/stout/owned.hpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/include/stout/owned.hpp?rev=1449837&view=auto
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/include/stout/owned.hpp (added)
+++ incubator/mesos/trunk/third_party/libprocess/include/stout/owned.hpp Mon Feb 25 19:44:43 2013
@@ -0,0 +1,20 @@
+#ifndef __STOUT_OWNED_HPP__
+#define __STOUT_OWNED_HPP__
+
+#include <boost/shared_ptr.hpp>
+
+// Represents a uniquely owned pointer.
+//
+// TODO(bmahler): For now, Owned only provides shared_ptr semantics.
+// When we make the switch to C++11, we will change to provide
+// unique_ptr semantics. Consequently, each usage of Owned that
+// invoked a copy will have to be adjusted to use move semantics.
+template <typename T>
+class Owned : public boost::shared_ptr<T>
+{
+public:
+ Owned(T* t) : boost::shared_ptr<T>(t) {}
+};
+
+
+#endif // __STOUT_OWNED_HPP__
Modified: incubator/mesos/trunk/third_party/libprocess/src/statistics.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/src/statistics.cpp?rev=1449837&r1=1449836&r2=1449837&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/src/statistics.cpp (original)
+++ incubator/mesos/trunk/third_party/libprocess/src/statistics.cpp Mon Feb 25 19:44:43 2013
@@ -16,6 +16,7 @@
#include <process/process.hpp>
#include <process/statistics.hpp>
+#include <stout/error.hpp>
#include <stout/duration.hpp>
#include <stout/foreach.hpp>
#include <stout/hashmap.hpp>
@@ -56,6 +57,11 @@ public:
const Option<Seconds>& start,
const Option<Seconds>& stop);
+ Try<Nothing> meter(
+ const string& context,
+ const string& name,
+ const Owned<meters::Meter>& meter);
+
void set(
const string& context,
const string& name,
@@ -88,10 +94,42 @@ private:
// We use a map instead of a hashmap to store the values because
// that way we can retrieve a series in sorted order efficiently.
+ // This maps from {context: {name: {time: value} } }.
hashmap<string, hashmap<string, map<Seconds, double> > > statistics;
+
+ // Each statistic can have many meters.
+ // This maps from {context: {name: [meters] } }.
+ hashmap<string, hashmap<string, list<Owned<meters::Meter> > > > meters;
};
+Try<Nothing> StatisticsProcess::meter(
+ const string& context,
+ const string& name,
+ const Owned<meters::Meter>& meter)
+{
+ if (meter->name == name) {
+ return Error("Meter name must not match the statistic name");
+ }
+
+ // Check for a duplicate meter.
+ foreachkey (const string& context, meters) {
+ foreachkey (const string& name, meters[context]) {
+ foreach (Owned<meters::Meter>& existing, meters[context][name]) {
+ if (meter->name == existing->name) {
+ return Error("Meter name matched existing meter name");
+ }
+ }
+ }
+ }
+
+ // Add the meter.
+ meters[context][name].push_back(meter);
+
+ return Nothing();
+}
+
+
map<Seconds, double> StatisticsProcess::get(
const string& context,
const string& name,
@@ -124,6 +162,17 @@ void StatisticsProcess::set(
// Update the raw value.
statistics[context][name][time] = value;
truncate(context, name);
+
+ // Update the metered values, if necessary.
+ if (meters.contains(context) && meters[context].contains(name)) {
+ foreach (Owned<meters::Meter>& meter, meters[context][name]) {
+ const Option<double>& update = meter->update(time, value);
+ if (update.isSome()) {
+ statistics[context][meter->name][time] = update.get();
+ truncate(context, meter->name);
+ }
+ }
+ }
}
@@ -209,7 +258,7 @@ Future<Response> StatisticsProcess::seri
if (result.isError()) {
return BadRequest("Failed to parse start: " + result.error());
}
- start = Option<Seconds>::some(Seconds(result.get()));
+ start = Seconds(result.get());
}
if (request.query.get("stop").isSome()) {
@@ -217,7 +266,7 @@ Future<Response> StatisticsProcess::seri
if (result.isError()) {
return BadRequest("Failed to parse stop: " + result.error());
}
- stop = Option<Seconds>::some(Seconds(result.get()));
+ stop = Seconds(result.get());
}
JSON::Array array;
@@ -259,6 +308,16 @@ Future<map<Seconds, double> > Statistics
}
+Future<Try<Nothing> > Statistics::meter(
+ const string& context,
+ const string& name,
+ Owned<meters::Meter> meter)
+{
+
+ return dispatch(process, &StatisticsProcess::meter, context, name, meter);
+}
+
+
void Statistics::set(
const string& context,
const string& name,
Modified: incubator/mesos/trunk/third_party/libprocess/src/tests/statistics_tests.cpp
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/third_party/libprocess/src/tests/statistics_tests.cpp?rev=1449837&r1=1449836&r2=1449837&view=diff
==============================================================================
--- incubator/mesos/trunk/third_party/libprocess/src/tests/statistics_tests.cpp (original)
+++ incubator/mesos/trunk/third_party/libprocess/src/tests/statistics_tests.cpp Mon Feb 25 19:44:43 2013
@@ -15,19 +15,27 @@ using std::map;
TEST(Statistics, set)
{
- Statistics statistics(Seconds(60*60*24));
+ Statistics statistics(Days(1));
- statistics.set("test", "statistic", 3.14);
+ // Set one using Clock::now() implicitly.
+ statistics.set("test", "statistic", 3.0);
- Future<map<Seconds, double> > values =
- statistics.get("test", "statistic");
+ // Set one using Clock::now() explicitly.
+ Seconds now(Clock::now());
+ statistics.set("test", "statistic", 4.0, now);
+
+ Future<map<Seconds, double> > values = statistics.get("test", "statistic");
values.await();
ASSERT_TRUE(values.isReady());
- EXPECT_EQ(1, values.get().size());
+ EXPECT_EQ(2, values.get().size());
+
EXPECT_GE(Clock::now(), values.get().begin()->first.secs());
- EXPECT_DOUBLE_EQ(3.14, values.get().begin()->second);
+ EXPECT_DOUBLE_EQ(3.0, values.get().begin()->second);
+
+ EXPECT_EQ(1, values.get().count(now));
+ EXPECT_DOUBLE_EQ(4.0, values.get()[now]);
}
@@ -35,19 +43,18 @@ TEST(Statistics, truncate)
{
Clock::pause();
- Statistics statistics(Seconds(60*60*24));
+ Statistics statistics(Days(1));
- statistics.set("test", "statistic", 3.14);
+ statistics.set("test", "statistic", 3.0);
- Future<map<Seconds, double> > values =
- statistics.get("test", "statistic");
+ Future<map<Seconds, double> > values = statistics.get("test", "statistic");
values.await();
ASSERT_TRUE(values.isReady());
EXPECT_EQ(1, values.get().size());
EXPECT_GE(Clock::now(), values.get().begin()->first.secs());
- EXPECT_DOUBLE_EQ(3.14, values.get().begin()->second);
+ EXPECT_DOUBLE_EQ(3.0, values.get().begin()->second);
Clock::advance((60*60*24) + 1);
@@ -60,7 +67,55 @@ TEST(Statistics, truncate)
ASSERT_TRUE(values.isReady());
EXPECT_EQ(1, values.get().size());
EXPECT_GE(Clock::now(), values.get().begin()->first.secs());
- EXPECT_DOUBLE_EQ(4.14, values.get().begin()->second);
+ EXPECT_DOUBLE_EQ(4.0, values.get().begin()->second);
Clock::resume();
}
+
+
+TEST(Statistics, meter) {
+ Statistics statistics(Days(1));
+
+ // Set up a meter, and ensure it captures the expected time rate.
+ Future<Try<Nothing> > meter =
+ statistics.meter("test", "statistic", new meters::TimeRate("metered"));
+
+ meter.await();
+
+ ASSERT_TRUE(meter.isReady());
+ ASSERT_TRUE(meter.get().isSome());
+
+ Seconds now(Clock::now());
+ statistics.set("test", "statistic", 1.0, now);
+ statistics.set("test", "statistic", 2.0, Seconds(now.secs() + 1.0));
+ statistics.set("test", "statistic", 4.0, Seconds(now.secs() + 2.0));
+
+ // Check the raw statistic values.
+ Future<map<Seconds, double> > values = statistics.get("test", "statistic");
+
+ values.await();
+
+ ASSERT_TRUE(values.isReady());
+ EXPECT_EQ(3, values.get().size());
+ EXPECT_EQ(1, values.get().count(now));
+ EXPECT_EQ(1, values.get().count(Seconds(now.secs() + 1.0)));
+ EXPECT_EQ(1, values.get().count(Seconds(now.secs() + 2.0)));
+
+ EXPECT_EQ(1.0, values.get()[now]);
+ EXPECT_EQ(2.0, values.get()[Seconds(now.secs() + 1.0)]);
+ EXPECT_EQ(4.0, values.get()[Seconds(now.secs() + 2.0)]);
+
+ // Now check the metered values.
+ values = statistics.get("test", "metered");
+
+ values.await();
+
+ ASSERT_TRUE(values.isReady());
+ EXPECT_EQ(2, values.get().size());
+ EXPECT_EQ(1, values.get().count(Seconds(now.secs() + 1.0)));
+ EXPECT_EQ(1, values.get().count(Seconds(now.secs() + 2.0)));
+
+ EXPECT_EQ(0., values.get()[now]);
+ EXPECT_EQ(1.0, values.get()[Seconds(now.secs() + 1.0)]); // 100%.
+ EXPECT_EQ(2.0, values.get()[Seconds(now.secs() + 2.0)]); // 200%.
+}