You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by zh...@apache.org on 2018/06/21 04:56:42 UTC
[1/3] mesos git commit: Added doc for new pull latency metrics.
Repository: mesos
Updated Branches:
refs/heads/master 79547292b -> 528b763aa
Added doc for new pull latency metrics.
Review: https://reviews.apache.org/r/67677
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/528b763a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/528b763a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/528b763a
Branch: refs/heads/master
Commit: 528b763aae3bf26358776404d8c0ca37621374e4
Parents: 8f425c0
Author: Zhitao Li <zh...@gmail.com>
Authored: Wed Jun 20 13:24:08 2018 -0700
Committer: Zhitao Li <zh...@gmail.com>
Committed: Wed Jun 20 21:52:18 2018 -0700
----------------------------------------------------------------------
docs/monitoring.md | 151 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 151 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/528b763a/docs/monitoring.md
----------------------------------------------------------------------
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 2985f68..2f06e9b 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -1765,6 +1765,157 @@ the master it is registered with.
</tr>
</table>
+#### Containerizers
+
+The following metrics provide information about both Mesos and Docker
+containerizers.
+
+<table class="table table-striped">
+<thead>
+<tr><th>Metric</th><th>Description</th><th>Type</th>
+</thead>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms</code>
+ </td>
+ <td>Docker containerizer image pull latency in ms </td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/count</code>
+ </td>
+ <td>Number of Docker containerizer image pulls</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/max</code>
+ </td>
+ <td>Maximum Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/min</code>
+ </td>
+ <td>Minimum Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p50</code>
+ </td>
+ <td>Median Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p90</code>
+ </td>
+ <td>90th percentile Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p95</code>
+ </td>
+ <td>95th percentile Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p99</code>
+ </td>
+ <td>99th percentile Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p999</code>
+ </td>
+ <td>99.9th percentile Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/docker/image_pull_ms/p9999</code>
+ </td>
+ <td>99.99th percentile Docker containerizer image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms</code>
+ </td>
+ <td>Mesos containerizer docker image pull latency in ms </td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/count</code>
+ </td>
+ <td>Number of Mesos containerizer docker image pulls</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/max</code>
+ </td>
+ <td>Maximum Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/min</code>
+ </td>
+ <td>Minimum Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p50</code>
+ </td>
+ <td>Median Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p90</code>
+ </td>
+ <td>90th percentile Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p95</code>
+ </td>
+ <td>95th percentile Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p99</code>
+ </td>
+ <td>99th percentile Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p999</code>
+ </td>
+ <td>99.9th percentile Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+<tr>
+ <td>
+ <code>containerizer/mesos/provisioner/docker_store/image_pull_ms/p9999</code>
+ </td>
+ <td>99.99th percentile Mesos containerizer docker image pull latency in ms</td>
+ <td>Gauge</td>
+</tr>
+</table>
+
#### Resource Providers
The following metrics provide information about ongoing and completed
[3/3] mesos git commit: Added an hourly timer for
`containerizer/docker/image_pull`.
Posted by zh...@apache.org.
Added an hourly timer for `containerizer/docker/image_pull`.
Review: https://reviews.apache.org/r/53105/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/68d96ad7
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/68d96ad7
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/68d96ad7
Branch: refs/heads/master
Commit: 68d96ad76f9b26217fa7e0bc19f9177398e85e21
Parents: 7954729
Author: Zhitao Li <zh...@gmail.com>
Authored: Wed Jun 20 10:24:42 2018 -0700
Committer: Zhitao Li <zh...@gmail.com>
Committed: Wed Jun 20 21:52:18 2018 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 8 ++++----
src/slave/containerizer/docker.hpp | 21 +++++++++++++++++++++
2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/68d96ad7/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index f83036a..7737107 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -445,10 +445,10 @@ Future<Nothing> DockerContainerizerProcess::pull(
string image = container->image();
- Future<Docker::Image> future = docker->pull(
- container->containerWorkDir,
- image,
- container->forcePullImage());
+ Future<Docker::Image> future = metrics.image_pull.time(docker->pull(
+ container->containerWorkDir,
+ image,
+ container->forcePullImage()));
containers_.at(containerId)->pull = future;
http://git-wip-us.apache.org/repos/asf/mesos/blob/68d96ad7/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index 1ed47c8..438d96a 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -26,6 +26,10 @@
#include <process/owned.hpp>
#include <process/shared.hpp>
+#include <process/metrics/metrics.hpp>
+#include <process/metrics/timer.hpp>
+
+#include <stout/duration.hpp>
#include <stout/flags.hpp>
#include <stout/hashset.hpp>
@@ -170,6 +174,21 @@ public:
virtual process::Future<hashset<ContainerID>> containers();
private:
+ struct Metrics
+ {
+ Metrics() : image_pull("containerizer/docker/image_pull", Hours(1))
+ {
+ process::metrics::add(image_pull);
+ }
+
+ ~Metrics()
+ {
+ process::metrics::remove(image_pull);
+ }
+
+ process::metrics::Timer<Milliseconds> image_pull;
+ };
+
// Continuations and helpers.
process::Future<Nothing> _fetch(
const ContainerID& containerId,
@@ -293,6 +312,8 @@ private:
Option<NvidiaComponents> nvidia;
+ Metrics metrics;
+
struct Container
{
static Try<Container*> create(
[2/3] mesos git commit: Added an hourly timer for docker store pull
latency.
Posted by zh...@apache.org.
Added an hourly timer for docker store pull latency.
This patch added pull latency tracking for docker store, which can tell
us both latency distribution of pull as well as number of pulls.
Review: https://reviews.apache.org/r/66875/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8f425c0d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8f425c0d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8f425c0d
Branch: refs/heads/master
Commit: 8f425c0d69fe996ed2c6fbcbd28b258bf4f3bb19
Parents: 68d96ad
Author: Zhitao Li <zh...@gmail.com>
Authored: Wed Jun 20 10:24:46 2018 -0700
Committer: Zhitao Li <zh...@gmail.com>
Committed: Wed Jun 20 21:52:18 2018 -0700
----------------------------------------------------------------------
.../mesos/provisioner/docker/store.cpp | 27 +++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8f425c0d/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index 6e7dc44..85aad25 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -33,7 +33,9 @@
#include <process/dispatch.hpp>
#include <process/executor.hpp>
#include <process/id.hpp>
-#include <process/metrics/counter.hpp>
+
+#include <process/metrics/metrics.hpp>
+#include <process/metrics/timer.hpp>
#include "slave/containerizer/mesos/provisioner/constants.hpp"
#include "slave/containerizer/mesos/provisioner/utils.hpp"
@@ -95,6 +97,23 @@ public:
const hashset<string>& activeLayerPaths);
private:
+ struct Metrics
+ {
+ Metrics() :
+ image_pull(
+ "containerizer/mesos/provisioner/docker_store/image_pull", Hours(1))
+ {
+ process::metrics::add(image_pull);
+ }
+
+ ~Metrics()
+ {
+ process::metrics::remove(image_pull);
+ }
+
+ process::metrics::Timer<Milliseconds> image_pull;
+ };
+
Future<Image> _get(
const spec::ImageReference& reference,
const Option<Secret>& config,
@@ -127,6 +146,8 @@ private:
// For executing path removals in a separated actor.
process::Executor executor;
+
+ Metrics metrics;
};
@@ -325,7 +346,7 @@ Future<Image> StoreProcess::_get(
Owned<Promise<Image>> promise(new Promise<Image>());
- Future<Image> future = puller->pull(
+ Future<Image> future = metrics.image_pull.time(puller->pull(
reference,
staging.get(),
backend,
@@ -346,7 +367,7 @@ Future<Image> StoreProcess::_get(
LOG(WARNING) << "Failed to remove staging directory: "
<< rmdir.error();
}
- }));
+ })));
promise->associate(future);
pulling[name] = promise;