You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2019/09/06 09:41:20 UTC
[ambari] branch branch-2.7 updated: AMBARI-25370 Producer and
Customer Request /s graphs are failing on Kafa Grafana dashboards (#3080)
This is an automated email from the ASF dual-hosted git repository.
dmitriusan pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 7405c68 AMBARI-25370 Producer and Customer Request /s graphs are failing on Kafa Grafana dashboards (#3080)
7405c68 is described below
commit 7405c68f5daca31353455722cf78724fca584fd8
Author: payert <35...@users.noreply.github.com>
AuthorDate: Fri Sep 6 11:41:13 2019 +0200
AMBARI-25370 Producer and Customer Request /s graphs are failing on Kafa Grafana dashboards (#3080)
From kafka 2.0.0 there has been addition of version tag in kafka.network.RequestMetrics.RequestsPerSec.request.* metrics.
This is breaking the the default Grafana dashboard provided by Ambari. On the Kafka - Home and Kafka - Hosts dashboards
the Producer requests /s and Consumer requests /s graphs are failing to show any data.
Change-Id: I08656a6f43923b08b3bbc1f1e0346bd93211bbe8
---
.../ambari-metrics/datasource.js | 31 +++++++++++++++++-----
.../conf/unix/metrics_whitelist | 2 ++
.../conf/windows/metrics_whitelist | 2 ++
.../grafana-dashboards/HDP/grafana-kafka-home.json | 12 +++++----
.../HDP/grafana-kafka-hosts.json | 12 +++++----
5 files changed, 42 insertions(+), 17 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
index ec5ee10..b838ad5 100644
--- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -99,7 +99,7 @@ define([
if(!_.isEmpty(templateSrv.variables) && templateSrv.variables[0].query === "yarnqueues") {
alias = alias + ' on ' + target.qmetric; }
if(!_.isEmpty(templateSrv.variables) && templateSrv.variables[0].query === "kafka-topics") {
- alias = alias + ' on ' + target.kbTopic; }
+ alias = alias + ' on ' + target.kbTopic; }
return function (res) {
res = res.data;
console.log('processing metric ' + target.metric);
@@ -595,12 +595,29 @@ define([
if (indexOfHosts >= 0) {
var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? templateSrv._values.hosts.slice(1,-1) :
templateSrv._values.hosts;
- allHosts = templateSrv._texts.hosts === "All" ? '%' : allHosts;
- metricsPromises.push(_.map(options.targets, function(target) {
- target.templatedHost = allHosts? allHosts : '';
- target.templatedCluster = templatedCluster;
- return getAllHostData(target);
- }));
+
+ /* The Producer & Comsumer Requests graphs on the Kafka Hosts dashboard should display metrics that are
+ * versioned, thus the value of different versions should be aggregated and grouped by hosts.
+ * In order to have a 'grouped by hosts' like view the metric results are queried for each hosts separately.
+ */
+ if (!_.isEmpty(options.targets.filter(function(target) {
+ return target.metric.endsWith(".%.count"); }))) {
+ allHosts = allHosts.split(',');
+ _.forEach(allHosts, function(host) {
+ metricsPromises.push(_.map(options.targets, function(target) {
+ target.templatedHost = host;
+ target.templatedCluster = templatedCluster;
+ return getAllHostData(target);
+ }));
+ });
+ } else {
+ allHosts = templateSrv._texts.hosts === "All" ? '%' : allHosts;
+ metricsPromises.push(_.map(options.targets, function(target) {
+ target.templatedHost = allHosts? allHosts : '';
+ target.templatedCluster = templatedCluster;
+ return getAllHostData(target);
+ }));
+ }
}
metricsPromises = _.flatten(metricsPromises);
} else {
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
index fd03d6e..689fd0d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/metrics_whitelist
@@ -273,6 +273,8 @@ kafka.controller.KafkaController.OfflinePartitionsCount
kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.version.*.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.version.*.count
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
index fd03d6e..689fd0d 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/windows/metrics_whitelist
@@ -273,6 +273,8 @@ kafka.controller.KafkaController.OfflinePartitionsCount
kafka.log.LogFlushStats.LogFlushRateAndTimeMs.1MinuteRate
kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count
kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count
+kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.version.*.count
+kafka.network.RequestMetrics.RequestsPerSec.request.Produce.version.*.count
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.99percentile
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.max
kafka.network.RequestMetrics.TotalTimeMs.request.FetchConsumer.mean
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
index 983142a..e9b6fa2 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-home.json
@@ -555,11 +555,12 @@
"app": "kafka_broker",
"downsampleAggregator": "avg",
"errors": {},
- "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count",
+ "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.Produce.%.count",
"precision": "default",
"refId": "A",
"transform": "rate",
- "transformData": "none"
+ "transformData": "none",
+ "seriesAggregator": "sum"
}
],
"timeFrom": null,
@@ -626,11 +627,12 @@
"app": "kafka_broker",
"downsampleAggregator": "avg",
"errors": {},
- "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count",
+ "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.%.count",
"precision": "default",
"refId": "A",
"transform": "rate",
- "transformData": "none"
+ "transformData": "none",
+ "seriesAggregator": "sum"
}
],
"timeFrom": null,
@@ -1073,7 +1075,7 @@
"list": []
},
"schemaVersion": 8,
- "version": 16,
+ "version": 17,
"links": [
{
"asDropdown": true,
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
index 85deb92..61b2e77 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-kafka-hosts.json
@@ -323,12 +323,13 @@
"app": "kafka_broker",
"downsampleAggregator": "avg",
"errors": {},
- "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.Produce.count",
+ "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.Produce.%.count",
"precision": "default",
"refId": "A",
"templatedHost": "",
"transform": "rate",
- "transformData": "none"
+ "transformData": "none",
+ "seriesAggregator": "sum"
}
],
"timeFrom": null,
@@ -395,12 +396,13 @@
"app": "kafka_broker",
"downsampleAggregator": "avg",
"errors": {},
- "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.count",
+ "metric": "kafka.network.RequestMetrics.RequestsPerSec.request.FetchConsumer.%.count",
"precision": "default",
"refId": "A",
"templatedHost": "",
"transform": "rate",
- "transformData": "none"
+ "transformData": "none",
+ "seriesAggregator": "sum"
}
],
"timeFrom": null,
@@ -1991,7 +1993,7 @@
"list": []
},
"schemaVersion": 8,
- "version": 16,
+ "version": 17,
"links": [
{
"asDropdown": true,