You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2017/01/24 01:08:01 UTC
[21/25] mesos git commit: Changed "master/tasks_unreachable" metric
from counter to gauge.
Changed "master/tasks_unreachable" metric from counter to gauge.
The number of unreachable tasks should be a gauge, because it can
increase and decrease over time. Note that we report the number of
unreachable tasks that the master knows about; this is a subset of the
"true" number of unreachable tasks, because the master only keeps a
limited-size cache of unreachable tasks for each framework.
Review: https://reviews.apache.org/r/55808/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/95aa7ff8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/95aa7ff8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/95aa7ff8
Branch: refs/heads/master
Commit: 95aa7ff83a6e511b0b5f255dc1887ce71550dbde
Parents: e8ff54f
Author: Neil Conway <ne...@gmail.com>
Authored: Mon Jan 23 17:05:28 2017 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Jan 23 17:07:27 2017 -0800
----------------------------------------------------------------------
docs/monitoring.md | 21 +++++++++++-
src/master/master.cpp | 16 ++++++++--
src/master/master.hpp | 1 +
src/master/metrics.cpp | 5 +--
src/master/metrics.hpp | 2 +-
src/tests/partition_tests.cpp | 65 +++++++++++++++++++++++++++++---------
6 files changed, 88 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/docs/monitoring.md
----------------------------------------------------------------------
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 59ee364..c42afce 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -14,12 +14,17 @@ you should monitor to detect abnormal situations in your cluster.
## Overview
Mesos master and agent nodes report a set of statistics and metrics that enable
-you to monitor resource usage and detect abnormal situations early. The
+cluster operators to monitor resource usage and detect abnormal situations early. The
information reported by Mesos includes details about available resources, used
resources, registered frameworks, active agents, and task state. You can use
this information to create automated alerts and to plot different metrics over
time inside a monitoring dashboard.
+Metric information is not persisted to disk at either master or agent
+nodes, which means that metrics will be reset when masters and agents
+are restarted. Similarly, if the current leading master fails and a new
+leading master is elected, metrics at the new master will be reset.
+
## Metric Types
@@ -501,6 +506,13 @@ The task states listed here match those of the task state machine.
</tr>
<tr>
<td>
+ <code>master/tasks_killing</code>
+ </td>
+ <td>Number of tasks currently being killed</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
<code>master/tasks_lost</code>
</td>
<td>Number of lost tasks</td>
@@ -527,6 +539,13 @@ The task states listed here match those of the task state machine.
<td>Number of starting tasks</td>
<td>Gauge</td>
</tr>
+<tr>
+ <td>
+ <code>master/tasks_unreachable</code>
+ </td>
+ <td>Number of unreachable tasks</td>
+ <td>Gauge</td>
+</tr>
</table>
#### Messages
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/src/master/master.cpp
----------------------------------------------------------------------
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 994b43b..d2aee2b 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -8119,9 +8119,6 @@ void Master::updateTask(Task* task, const StatusUpdate& update)
case TASK_ERROR:
++metrics->tasks_error;
break;
- case TASK_UNREACHABLE:
- ++metrics->tasks_unreachable;
- break;
case TASK_DROPPED:
++metrics->tasks_dropped;
break;
@@ -8135,6 +8132,7 @@ void Master::updateTask(Task* task, const StatusUpdate& update)
case TASK_STAGING:
case TASK_RUNNING:
case TASK_KILLING:
+ case TASK_UNREACHABLE:
break;
case TASK_UNKNOWN:
// Should not happen.
@@ -8578,6 +8576,18 @@ double Master::_tasks_running()
}
+double Master::_tasks_unreachable()
+{
+ double count = 0.0;
+
+ foreachvalue (Framework* framework, frameworks.registered) {
+ count += framework->unreachableTasks.size();
+ }
+
+ return count;
+}
+
+
double Master::_tasks_killing()
{
double count = 0.0;
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index 9c610d1..e4dd677 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1830,6 +1830,7 @@ private:
double _tasks_staging();
double _tasks_starting();
double _tasks_running();
+ double _tasks_unreachable();
double _tasks_killing();
double _resources_total(const std::string& name);
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/src/master/metrics.cpp
----------------------------------------------------------------------
diff --git a/src/master/metrics.cpp b/src/master/metrics.cpp
index 4c7072c..646041d 100644
--- a/src/master/metrics.cpp
+++ b/src/master/metrics.cpp
@@ -83,6 +83,9 @@ Metrics::Metrics(const Master& master)
tasks_running(
"master/tasks_running",
defer(master, &Master::_tasks_running)),
+ tasks_unreachable(
+ "master/tasks_unreachable",
+ defer(master, &Master::_tasks_unreachable)),
tasks_killing(
"master/tasks_killing",
defer(master, &Master::_tasks_killing)),
@@ -98,8 +101,6 @@ Metrics::Metrics(const Master& master)
"master/tasks_error"),
tasks_dropped(
"master/tasks_dropped"),
- tasks_unreachable(
- "master/tasks_unreachable"),
tasks_gone(
"master/tasks_gone"),
tasks_gone_by_operator(
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/src/master/metrics.hpp
----------------------------------------------------------------------
diff --git a/src/master/metrics.hpp b/src/master/metrics.hpp
index 62c3620..f701efe 100644
--- a/src/master/metrics.hpp
+++ b/src/master/metrics.hpp
@@ -61,6 +61,7 @@ struct Metrics
process::metrics::Gauge tasks_staging;
process::metrics::Gauge tasks_starting;
process::metrics::Gauge tasks_running;
+ process::metrics::Gauge tasks_unreachable;
process::metrics::Gauge tasks_killing;
process::metrics::Counter tasks_finished;
process::metrics::Counter tasks_failed;
@@ -68,7 +69,6 @@ struct Metrics
process::metrics::Counter tasks_lost;
process::metrics::Counter tasks_error;
process::metrics::Counter tasks_dropped;
- process::metrics::Counter tasks_unreachable;
process::metrics::Counter tasks_gone;
process::metrics::Counter tasks_gone_by_operator;
http://git-wip-us.apache.org/repos/asf/mesos/blob/95aa7ff8/src/tests/partition_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/partition_tests.cpp b/src/tests/partition_tests.cpp
index 3bcb463..f03c5bd 100644
--- a/src/tests/partition_tests.cpp
+++ b/src/tests/partition_tests.cpp
@@ -267,13 +267,15 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(PartitionTest, ReregisterSlavePartitionAware)
AWAIT_READY(slaveLost);
- JSON::Object stats = Metrics();
- EXPECT_EQ(0, stats.values["master/tasks_lost"]);
- EXPECT_EQ(1, stats.values["master/tasks_unreachable"]);
- EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
- EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
- EXPECT_EQ(1, stats.values["master/slave_removals"]);
- EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ {
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(0, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(1, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ }
// Check the master's "/state" endpoint. There should be a single
// unreachable agent. The "tasks" and "completed_tasks" fields
@@ -506,6 +508,17 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(PartitionTest, ReregisterSlavePartitionAware)
EXPECT_EQ(1, framework.values["TASK_RUNNING"].as<JSON::Number>());
}
+ {
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(1, stats.values["master/tasks_running"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
+ }
+
driver.stop();
driver.join();
}
@@ -621,14 +634,16 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(PartitionTest, ReregisterSlaveNotPartitionAware)
AWAIT_READY(slaveLost);
- JSON::Object stats = Metrics();
- EXPECT_EQ(1, stats.values["master/tasks_lost"]);
- EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
- EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
- EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
- EXPECT_EQ(1, stats.values["master/slave_removals"]);
- EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
- EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
+ {
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(1, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
+ }
// Check the master's "/state" endpoint. The "tasks" and
// "unreachable_tasks" fields should be empty; there should be a
@@ -783,6 +798,18 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(PartitionTest, ReregisterSlaveNotPartitionAware)
EXPECT_EQ(1, framework.values["TASK_LOST"].as<JSON::Number>());
}
+ {
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(1, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(0, stats.values["master/tasks_killed"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_scheduled"]);
+ EXPECT_EQ(1, stats.values["master/slave_unreachable_completed"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals"]);
+ EXPECT_EQ(1, stats.values["master/slave_removals/reason_unhealthy"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals/reason_unregistered"]);
+ }
+
driver.stop();
driver.join();
}
@@ -1038,6 +1065,14 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(
Clock::resume();
+ {
+ JSON::Object stats = Metrics();
+ EXPECT_EQ(2, stats.values["master/tasks_running"]);
+ EXPECT_EQ(0, stats.values["master/tasks_lost"]);
+ EXPECT_EQ(0, stats.values["master/tasks_unreachable"]);
+ EXPECT_EQ(0, stats.values["master/slave_removals"]);
+ }
+
driver1.stop();
driver1.join();