You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ya...@apache.org on 2017/05/24 00:01:31 UTC
mesos git commit: Metric in the allocator to track latency in running
allocations.
Repository: mesos
Updated Branches:
refs/heads/master e40318975 -> 21caae3bd
Metric in the allocator to track latency in running allocations.
Added a metric 'allocator/mesos/allocation_run_latency_ms' which
tracks the latency due to the batching of allocation requests.
Review: https://reviews.apache.org/r/53840/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/21caae3b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/21caae3b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/21caae3b
Branch: refs/heads/master
Commit: 21caae3bdf72dec3962691da6e6ccbabbbe928d4
Parents: e403189
Author: Anindya Sinha <an...@apple.com>
Authored: Tue May 23 10:14:08 2017 -0700
Committer: Jiang Yan Xu <xu...@apple.com>
Committed: Tue May 23 17:01:26 2017 -0700
----------------------------------------------------------------------
docs/monitoring.md | 90 +++++++++++++++++++++---
src/master/allocator/mesos/hierarchical.cpp | 6 +-
src/master/allocator/mesos/metrics.cpp | 5 +-
src/master/allocator/mesos/metrics.hpp | 5 +-
src/tests/hierarchical_allocator_tests.cpp | 83 ++++++++++++++++++++++
5 files changed, 176 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/21caae3b/docs/monitoring.md
----------------------------------------------------------------------
diff --git a/docs/monitoring.md b/docs/monitoring.md
index a027f49..cb28336 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -973,70 +973,70 @@ and resource allocations in the allocator.
<td>
<code>allocator/mesos/allocation_run_ms</code>
</td>
- <td>Allocation algorithm latency in ms</td>
+ <td>Time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/count</code>
</td>
- <td>Number of allocation algorithm latency measurements in the window</td>
+ <td>Number of allocation algorithm time measurements in the window</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/max</code>
</td>
- <td>Maximum allocation algorithm latency in ms</td>
+ <td>Maximum time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/min</code>
</td>
- <td>Minimum allocation algorithm latency in ms</td>
+ <td>Minimum time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p50</code>
</td>
- <td>Median allocation algorithm latency in ms</td>
+ <td>Median time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p90</code>
</td>
- <td>90th percentile allocation algorithm latency in ms</td>
+ <td>90th percentile of time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p95</code>
</td>
- <td>95th percentile allocation algorithm latency in ms</td>
+ <td>95th percentile of time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p99</code>
</td>
- <td>99th percentile allocation algorithm latency in ms</td>
+ <td>99th percentile of time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p999</code>
</td>
- <td>99.9th percentile allocation algorithm latency in ms</td>
+ <td>99.9th percentile of time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
<td>
<code>allocator/mesos/allocation_run_ms/p9999</code>
</td>
- <td>99.99th percentile allocation algorithm latency in ms</td>
+ <td>99.99th percentile of time spent in allocation algorithm in ms</td>
<td>Gauge</td>
</tr>
<tr>
@@ -1048,6 +1048,76 @@ and resource allocations in the allocator.
</tr>
<tr>
<td>
+ <code>allocator/mesos/allocation_run_latency_ms</code>
+ </td>
+ <td>Allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/count</code>
+ </td>
+ <td>Number of allocation batch latency measurements in the window</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/max</code>
+ </td>
+ <td>Maximum allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/min</code>
+ </td>
+ <td>Minimum allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p50</code>
+ </td>
+ <td>Median allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p90</code>
+ </td>
+ <td>90th percentile allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p95</code>
+ </td>
+ <td>95th percentile allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p99</code>
+ </td>
+ <td>99th percentile allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p999</code>
+ </td>
+ <td>99.9th percentile allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>allocator/mesos/allocation_run_latency_ms/p9999</code>
+ </td>
+ <td>99.99th percentile allocation batch latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
<code>allocator/mesos/roles/<role>/shares/dominant</code>
</td>
<td>Dominant resource share for the role, exposed as a percentage (0.0-1.0)</td>
http://git-wip-us.apache.org/repos/asf/mesos/blob/21caae3b/src/master/allocator/mesos/hierarchical.cpp
----------------------------------------------------------------------
diff --git a/src/master/allocator/mesos/hierarchical.cpp b/src/master/allocator/mesos/hierarchical.cpp
index b75ed9a..5511bf6 100644
--- a/src/master/allocator/mesos/hierarchical.cpp
+++ b/src/master/allocator/mesos/hierarchical.cpp
@@ -1402,6 +1402,7 @@ Future<Nothing> HierarchicalAllocatorProcess::allocate(
allocationCandidates |= slaveIds;
if (allocation.isNone() || !allocation->isPending()) {
+ metrics.allocation_run_latency.start();
allocation = dispatch(self(), &Self::_allocate);
}
@@ -1409,7 +1410,10 @@ Future<Nothing> HierarchicalAllocatorProcess::allocate(
}
-Nothing HierarchicalAllocatorProcess::_allocate() {
+Nothing HierarchicalAllocatorProcess::_allocate()
+{
+ metrics.allocation_run_latency.stop();
+
if (paused) {
VLOG(1) << "Skipped allocation because the allocator is paused";
http://git-wip-us.apache.org/repos/asf/mesos/blob/21caae3b/src/master/allocator/mesos/metrics.cpp
----------------------------------------------------------------------
diff --git a/src/master/allocator/mesos/metrics.cpp b/src/master/allocator/mesos/metrics.cpp
index ec987fe..4892dbc 100644
--- a/src/master/allocator/mesos/metrics.cpp
+++ b/src/master/allocator/mesos/metrics.cpp
@@ -48,12 +48,14 @@ Metrics::Metrics(const HierarchicalAllocatorProcess& _allocator)
process::defer(
allocator, &HierarchicalAllocatorProcess::_event_queue_dispatches)),
allocation_runs("allocator/mesos/allocation_runs"),
- allocation_run("allocator/mesos/allocation_run", Hours(1))
+ allocation_run("allocator/mesos/allocation_run", Hours(1)),
+ allocation_run_latency("allocator/mesos/allocation_run_latency", Hours(1))
{
process::metrics::add(event_queue_dispatches);
process::metrics::add(event_queue_dispatches_);
process::metrics::add(allocation_runs);
process::metrics::add(allocation_run);
+ process::metrics::add(allocation_run_latency);
// Create and install gauges for the total and allocated
// amount of standard scalar resources.
@@ -92,6 +94,7 @@ Metrics::~Metrics()
process::metrics::remove(event_queue_dispatches_);
process::metrics::remove(allocation_runs);
process::metrics::remove(allocation_run);
+ process::metrics::remove(allocation_run_latency);
foreach (const Gauge& gauge, resources_total) {
process::metrics::remove(gauge);
http://git-wip-us.apache.org/repos/asf/mesos/blob/21caae3b/src/master/allocator/mesos/metrics.hpp
----------------------------------------------------------------------
diff --git a/src/master/allocator/mesos/metrics.hpp b/src/master/allocator/mesos/metrics.hpp
index ce486eb..d56a10b 100644
--- a/src/master/allocator/mesos/metrics.hpp
+++ b/src/master/allocator/mesos/metrics.hpp
@@ -66,9 +66,12 @@ struct Metrics
// Number of times the allocation algorithm has run.
process::metrics::Counter allocation_runs;
- // Latency of the allocation algorithm.
+ // Time spent in the allocation algorithm.
process::metrics::Timer<Milliseconds> allocation_run;
+ // The latency of allocation runs due to the batching of allocation requests.
+ process::metrics::Timer<Milliseconds> allocation_run_latency;
+
// Gauges for the total amount of each resource in the cluster.
std::vector<process::metrics::Gauge> resources_total;
http://git-wip-us.apache.org/repos/asf/mesos/blob/21caae3b/src/tests/hierarchical_allocator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hierarchical_allocator_tests.cpp b/src/tests/hierarchical_allocator_tests.cpp
index 6dee229..f911110 100644
--- a/src/tests/hierarchical_allocator_tests.cpp
+++ b/src/tests/hierarchical_allocator_tests.cpp
@@ -3575,6 +3575,7 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(HierarchicalAllocatorTest, AllocationRunsMetric)
++allocations; // Adding a framework triggers allocations.
+ // Allocation count is set based on adding an agent and a framework.
expected.values = { {"allocator/mesos/allocation_runs", allocations} };
metrics = Metrics();
@@ -3670,6 +3671,88 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(
}
+// This test checks that the allocation run latency
+// metrics are reported in the metrics endpoint.
+// TODO(xujyan): This test is structurally similar to
+// `AllocationRunTimerMetrics` above. Consider a refactor.
+TEST_F_TEMP_DISABLED_ON_WINDOWS(
+ HierarchicalAllocatorTest,
+ AllocationRunLatencyMetrics)
+{
+ Clock::pause();
+
+ initialize();
+
+ // These time series statistics will be generated
+ // once at least 2 allocation runs occur.
+ auto statistics = {
+ "allocator/mesos/allocation_run_latency_ms/count",
+ "allocator/mesos/allocation_run_latency_ms/min",
+ "allocator/mesos/allocation_run_latency_ms/max",
+ "allocator/mesos/allocation_run_latency_ms/p50",
+ "allocator/mesos/allocation_run_latency_ms/p95",
+ "allocator/mesos/allocation_run_latency_ms/p99",
+ "allocator/mesos/allocation_run_latency_ms/p999",
+ "allocator/mesos/allocation_run_latency_ms/p9999",
+ };
+
+ JSON::Object metrics = Metrics();
+ map<string, JSON::Value> values = metrics.values;
+
+ EXPECT_EQ(0u, values.count("allocator/mesos/allocation_run_latency_ms"));
+
+ // No allocation latency statistics should appear.
+ foreach (const string& statistic, statistics) {
+ EXPECT_EQ(0u, values.count(statistic))
+ << "Expected " << statistic << " to be absent";
+ }
+
+ // Allow the allocation timer to measure time.
+ Clock::resume();
+
+ // Trigger at least two calls to allocate to generate the window statistics.
+ SlaveInfo agent = createSlaveInfo("cpus:2;mem:1024;disk:0");
+ allocator->addSlave(
+ agent.id(),
+ agent,
+ AGENT_CAPABILITIES(),
+ None(),
+ agent.resources(),
+ {});
+
+ // Wait for the allocation triggered by `addSlave()` to complete.
+ Clock::pause();
+ Clock::settle();
+ Clock::resume();
+
+ FrameworkInfo framework = createFrameworkInfo({"role1"});
+ allocator->addFramework(framework.id(), framework, {}, true);
+
+ // Wait for the allocation triggered by `addFramework()` to complete.
+ AWAIT_READY(allocations.get());
+
+ metrics = Metrics();
+ values = metrics.values;
+
+ // A non-zero measurement should be present.
+ EXPECT_EQ(1u, values.count("allocator/mesos/allocation_run_latency_ms"));
+
+ JSON::Value value =
+ metrics.values["allocator/mesos/allocation_run_latency_ms"];
+ ASSERT_TRUE(value.is<JSON::Number>()) << value.which();
+
+ JSON::Number timing = value.as<JSON::Number>();
+ ASSERT_EQ(JSON::Number::FLOATING, timing.type);
+ EXPECT_GE(timing.as<double>(), 0.0);
+
+ // The statistics should be generated.
+ foreach (const string& statistic, statistics) {
+ EXPECT_EQ(1u, values.count(statistic))
+ << "Expected " << statistic << " to be present";
+ }
+}
+
+
// This test checks that per-role active offer filter metrics
// are correctly reported in the metrics endpoint.
TEST_F_TEMP_DISABLED_ON_WINDOWS(