You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ya...@apache.org on 2015/06/09 02:41:08 UTC
[1/2] mesos git commit: Made updateSlave() update its
'totalResources'.
Repository: mesos
Updated Branches:
refs/heads/master dd12418ac -> b24da74e4
Made updateSlave() update its 'totalResources'.
- This way Master::Slave::totalResources includes revocable resources, which we need for metrics for revocable resources.
Review: https://reviews.apache.org/r/35118
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/baf80d7c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/baf80d7c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/baf80d7c
Branch: refs/heads/master
Commit: baf80d7ccabaa300f4ef7cb739f09c1bc1da507c
Parents: dd12418
Author: Jiang Yan Xu <ya...@jxu.me>
Authored: Fri Jun 5 00:34:47 2015 -0700
Committer: Jiang Yan Xu <ya...@jxu.me>
Committed: Mon Jun 8 17:39:59 2015 -0700
----------------------------------------------------------------------
src/master/master.cpp | 8 +++++++-
src/master/master.hpp | 7 ++++---
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/baf80d7c/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index d436f84..68ca437 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -3461,7 +3461,7 @@ void Master::unregisterSlave(const UPID& from, const SlaveID& slaveId)
void Master::updateSlave(
const SlaveID& slaveId,
- const vector<Resource>& oversubscribedResources)
+ const Resources& oversubscribedResources)
{
++metrics->messages_update_slave;
@@ -3509,6 +3509,12 @@ void Master::updateSlave(
}
}
+ // Check that all the oversubscribed resources are revocable.
+ CHECK_EQ(oversubscribedResources, oversubscribedResources.revocable());
+
+ slave->totalResources -= slave->totalResources.revocable();
+ slave->totalResources += oversubscribedResources;
+
// Now, update the allocator with the new estimate.
allocator->updateSlave(slaveId, oversubscribedResources);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/baf80d7c/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index deeb0d8..9f19765 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -310,8 +310,9 @@ struct Slave
Resources checkpointedResources;
// The current total resources of the slave. Note that this is
- // different from 'info.resources()' because this also consider
- // operations (e.g., CREATE, RESERVE) that have been applied.
+ // different from 'info.resources()' because this also considers
+ // operations (e.g., CREATE, RESERVE) that have been applied and
+ // includes revocable resources as well.
Resources totalResources;
SlaveObserver* observer;
@@ -752,7 +753,7 @@ public:
void updateSlave(
const SlaveID& slaveId,
- const std::vector<Resource>& oversubscribedResources);
+ const Resources& oversubscribedResources);
void shutdownSlave(
const SlaveID& slaveId,
[2/2] mesos git commit: Introduced metrics for revocable resources.
Posted by ya...@apache.org.
Introduced metrics for revocable resources.
Review: https://reviews.apache.org/r/35119
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b24da74e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b24da74e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b24da74e
Branch: refs/heads/master
Commit: b24da74e48aa2566e1410d39054f863bd469c393
Parents: baf80d7
Author: Jiang Yan Xu <ya...@jxu.me>
Authored: Fri Jun 5 14:03:44 2015 -0700
Committer: Jiang Yan Xu <ya...@jxu.me>
Committed: Mon Jun 8 17:40:08 2015 -0700
----------------------------------------------------------------------
src/master/master.cpp | 46 +++++++++++++++++++++++++++++++
src/master/master.hpp | 4 +++
src/master/metrics.cpp | 37 +++++++++++++++++++++++++
src/master/metrics.hpp | 6 ++++
src/tests/oversubscription_tests.cpp | 9 +++++-
5 files changed, 101 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b24da74e/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 68ca437..5ed990e 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -5398,6 +5398,52 @@ double Master::_resources_percent(const std::string& name)
}
}
+
+double Master::_resources_revocable_total(const std::string& name)
+{
+ double total = 0.0;
+
+ foreachvalue (Slave* slave, slaves.registered) {
+ foreach (const Resource& resource, slave->totalResources.revocable()) {
+ if (resource.name() == name && resource.type() == Value::SCALAR) {
+ total += resource.scalar().value();
+ }
+ }
+ }
+
+ return total;
+}
+
+
+double Master::_resources_revocable_used(const std::string& name)
+{
+ double used = 0.0;
+
+ foreachvalue (Slave* slave, slaves.registered) {
+ foreachvalue (const Resources& resources, slave->usedResources) {
+ foreach (const Resource& resource, resources.revocable()) {
+ if (resource.name() == name && resource.type() == Value::SCALAR) {
+ used += resource.scalar().value();
+ }
+ }
+ }
+ }
+
+ return used;
+}
+
+
+double Master::_resources_revocable_percent(const std::string& name)
+{
+ double total = _resources_revocable_total(name);
+
+ if (total == 0.0) {
+ return total;
+ } else {
+ return _resources_revocable_used(name) / total;
+ }
+}
+
} // namespace master {
} // namespace internal {
} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/b24da74e/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 9f19765..af83d3e 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1366,6 +1366,10 @@ private:
double _resources_used(const std::string& name);
double _resources_percent(const std::string& name);
+ double _resources_revocable_total(const std::string& name);
+ double _resources_revocable_used(const std::string& name);
+ double _resources_revocable_percent(const std::string& name);
+
process::Time startTime; // Start time used to calculate uptime.
Option<process::Time> electedTime; // Time when this master is elected.
http://git-wip-us.apache.org/repos/asf/mesos/blob/b24da74e/src/master/metrics.cpp
----------------------------------------------------------------------
diff --git a/src/master/metrics.cpp b/src/master/metrics.cpp
index 264252c..d2489c8 100644
--- a/src/master/metrics.cpp
+++ b/src/master/metrics.cpp
@@ -246,6 +246,7 @@ Metrics::Metrics(const Master& master)
// resources the slave exposes.
const std::string resources[] = {"cpus", "mem", "disk"};
+ // Regular (non-revocable) resources.
foreach (const std::string& resource, resources) {
process::metrics::Gauge totalGauge(
"master/" + resource + "_total",
@@ -265,6 +266,27 @@ Metrics::Metrics(const Master& master)
resources_percent.push_back(percentGauge);
process::metrics::add(percentGauge);
}
+
+ // Revocable resources.
+ foreach (const std::string& resource, resources) {
+ process::metrics::Gauge totalGauge(
+ "master/" + resource + "_revocable_total",
+ defer(master, &Master::_resources_revocable_total, resource));
+ resources_revocable_total.push_back(totalGauge);
+ process::metrics::add(totalGauge);
+
+ process::metrics::Gauge usedGauge(
+ "master/" + resource + "_revocable_used",
+ defer(master, &Master::_resources_revocable_used, resource));
+ resources_revocable_used.push_back(usedGauge);
+ process::metrics::add(usedGauge);
+
+ process::metrics::Gauge percentGauge(
+ "master/" + resource + "_revocable_percent",
+ defer(master, &Master::_resources_revocable_percent, resource));
+ resources_revocable_percent.push_back(percentGauge);
+ process::metrics::add(percentGauge);
+ }
}
@@ -363,6 +385,21 @@ Metrics::~Metrics()
}
resources_percent.clear();
+ foreach (const process::metrics::Gauge& gauge, resources_revocable_total) {
+ process::metrics::remove(gauge);
+ }
+ resources_revocable_total.clear();
+
+ foreach (const process::metrics::Gauge& gauge, resources_revocable_used) {
+ process::metrics::remove(gauge);
+ }
+ resources_revocable_used.clear();
+
+ foreach (const process::metrics::Gauge& gauge, resources_revocable_percent) {
+ process::metrics::remove(gauge);
+ }
+ resources_revocable_percent.clear();
+
foreachvalue (const auto& source_reason, tasks_states) {
foreachvalue (const auto& reason_counter, source_reason) {
foreachvalue (const process::metrics::Counter& counter, reason_counter) {
http://git-wip-us.apache.org/repos/asf/mesos/blob/b24da74e/src/master/metrics.hpp
----------------------------------------------------------------------
diff --git a/src/master/metrics.hpp b/src/master/metrics.hpp
index 833033c..3d389e6 100644
--- a/src/master/metrics.hpp
+++ b/src/master/metrics.hpp
@@ -174,10 +174,16 @@ struct Metrics
process::metrics::Counter slave_shutdowns_canceled;
// Resource metrics.
+ // Regular (non-revocable) resources.
std::vector<process::metrics::Gauge> resources_total;
std::vector<process::metrics::Gauge> resources_used;
std::vector<process::metrics::Gauge> resources_percent;
+ // Revocable resources.
+ std::vector<process::metrics::Gauge> resources_revocable_total;
+ std::vector<process::metrics::Gauge> resources_revocable_used;
+ std::vector<process::metrics::Gauge> resources_revocable_percent;
+
void incrementTasksStates(
TaskState state, TaskStatus::Source source, TaskStatus::Reason reason);
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/b24da74e/src/tests/oversubscription_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/oversubscription_tests.cpp b/src/tests/oversubscription_tests.cpp
index afd7ff4..59cf07e 100644
--- a/src/tests/oversubscription_tests.cpp
+++ b/src/tests/oversubscription_tests.cpp
@@ -181,7 +181,7 @@ TEST_F(OversubscriptionTest, ForwardUpdateSlaveMessage)
AWAIT_READY(update);
- EXPECT_EQ(Resources(update.get().oversubscribed_resources()), resources);
+ EXPECT_EQ(update.get().oversubscribed_resources(), resources);
// Ensure the metric is updated.
JSON::Object metrics = Metrics();
@@ -192,6 +192,13 @@ TEST_F(OversubscriptionTest, ForwardUpdateSlaveMessage)
1u,
metrics.values["master/messages_update_slave"]);
+ ASSERT_EQ(
+ 1u,
+ metrics.values.count("master/cpus_revocable_total"));
+ ASSERT_EQ(
+ 1.0,
+ metrics.values["master/cpus_revocable_total"]);
+
Shutdown();
}