You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by pr...@apache.org on 2016/09/26 20:52:42 UTC
ambari git commit: AMBARI-18439 - [Grafana] Add Kafka-Offset
dashboard for Storm (prajwal)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 bc14ad942 -> 9b4828f05
AMBARI-18439 - [Grafana] Add Kafka-Offset dashboard for Storm (prajwal)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b4828f0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b4828f0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b4828f0
Branch: refs/heads/branch-2.5
Commit: 9b4828f05db2bc21974869a7c54ebb55667422b4
Parents: bc14ad9
Author: Prajwal Rao <pr...@gmail.com>
Authored: Mon Sep 26 13:51:49 2016 -0700
Committer: Prajwal Rao <pr...@gmail.com>
Committed: Mon Sep 26 13:51:49 2016 -0700
----------------------------------------------------------------------
.../ambari-metrics/datasource.js | 64 +++++
.../HDF/grafana-storm-kafka-offset.json | 258 +++++++++++++++++++
.../HDP/grafana-storm-kafka-offset.json | 258 +++++++++++++++++++
3 files changed, 580 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/9b4828f0/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
index fa0cc47..626439d 100644
--- a/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
+++ b/ambari-metrics/ambari-metrics-grafana/ambari-metrics/datasource.js
@@ -418,6 +418,21 @@ define([
}));
}
+ //Templatized Dashboard for Storm Kafka Offset
+ if (templateSrv.variables[0].query === "topologies" && templateSrv.variables[1] &&
+ templateSrv.variables[1].name === "topic") {
+ var selectedTopology = templateSrv._values.topologies;
+ var selectedTopic = templateSrv._values.topic;
+ metricsPromises.push(_.map(options.targets, function(target) {
+ target.sTopology = selectedTopology;
+ target.sTopic = selectedTopic;
+ target.sPartition = options.scopedVars.partition.value;
+ target.sTopoMetric = target.metric.replace('*', target.sTopology).replace('*', target.sTopic)
+ .replace('*', target.sPartition);
+ return getStormData(target);
+ }));
+ }
+
// To speed up querying on templatized dashboards.
if (templateSrv.variables[1] && templateSrv.variables[1].name === "hosts") {
var allHosts = templateSrv._values.hosts.lastIndexOf('}') > 0 ? templateSrv._values.hosts.slice(1,-1) :
@@ -616,6 +631,55 @@ define([
});
});
}
+ var stormEntities = {};
+ AmbariMetricsDatasource.prototype.getStormEntities = function () {
+ return this.initMetricAppidMapping()
+ .then(function () {
+ var storm = allMetrics["nimbus"];
+ var extractTopologies = storm.filter(/./.test.bind(new
+ RegExp("partition", 'g')));
+ _.map(extractTopologies, function(topology){
+ topology = topology.split('.').slice(0,5);
+ var topologyName = topologyN = topology[1]; // Topology
+ var topologyTopicName = topicN = topology[3]; // Topic
+ var topologyTopicPartitionName = topology[4]; // Partition
+ if (stormEntities[topologyName]) {
+ if (stormEntities[topologyName][topologyTopicName]) {
+ stormEntities[topologyName][topologyTopicName].push(topologyTopicPartitionName);
+ } else {
+ stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName];
+ }
+ } else {
+ stormEntities[topologyName] = {};
+ stormEntities[topologyName][topologyTopicName] = [topologyTopicPartitionName];
+ }
+ });
+ });
+ };
+ //Templated Variables for Storm Topics per Topology
+ if (interpolated.includes("stormTopic")) {
+ var topicName = interpolated.substring(0,interpolated.indexOf('.'));
+ return this.getStormEntities().then(function () {
+ var topicNames = Object.keys(stormEntities[topicName]);
+ return _.map(topicNames, function(names){
+ return {
+ text: names
+ };
+ });
+ });
+ }
+ //Templated Variables for Storm Partitions per Topic
+ if (interpolated.includes("stormPartition")) {
+ var topicN, topologyN;
+ return this.getStormEntities().then(function () {
+ var partitionNames = _.uniq(stormEntities[topologyN][topicN]);
+ return _.map(partitionNames, function(names){
+ return {
+ text: names
+ };
+ });
+ });
+ }
// Templated Variable for YARN Queues.
// It will search the cluster and populate the queues.
if(interpolated === "yarnqueues") {
http://git-wip-us.apache.org/repos/asf/ambari/blob/9b4828f0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json
new file mode 100644
index 0000000..ac1f829
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDF/grafana-storm-kafka-offset.json
@@ -0,0 +1,258 @@
+{
+ "id": null,
+ "title": "Storm - Kafka-Offset",
+ "originalTitle": "Storm - Kafka-Offset",
+ "tags": [
+ "storm",
+ "builtin",
+ "2.4.0.0"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": false,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "25px",
+ "panels": [
+ {
+ "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari. You may lose any changes made to this dashboard. If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>",
+ "editable": true,
+ "error": false,
+ "id": 14,
+ "isNew": true,
+ "links": [],
+ "mode": "html",
+ "span": 12,
+ "style": {},
+ "title": "",
+ "type": "text"
+ }
+ ],
+ "title": "New row"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 15,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "minSpan": 4,
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": "partition",
+ "scopedVars": {
+ "partition": {
+ "text": "partition-0",
+ "value": "partition-0",
+ "selected": true
+ }
+ },
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "avg",
+ "alias": "Kafka Offset",
+ "app": "nimbus",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "hosts": "",
+ "metric": "topology.*.kafka-topic.*.*.latestTimeOffset",
+ "precision": "default",
+ "refId": "B",
+ "sPartition": "partition-0",
+ "sTopic": "myKafkaTopic",
+ "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset",
+ "sTopology": "kafka-topology",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "avg",
+ "alias": "Storm Offset",
+ "app": "nimbus",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset",
+ "precision": "default",
+ "refId": "A",
+ "sComponent": "--acker",
+ "sPartition": "partition-0",
+ "sTopic": "myKafkaTopic",
+ "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset",
+ "sTopology": "kafka-topology",
+ "seriesAggregator": "none",
+ "transform": "none"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "$partition",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ }
+ ],
+ "title": "New row"
+ }
+ ],
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": [
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "kafka-topology",
+ "value": "kafka-topology"
+ },
+ "datasource": null,
+ "includeAll": false,
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "topologies",
+ "options": [
+ ],
+ "query": "topologies",
+ "refresh": true,
+ "regex": "",
+ "type": "query"
+ },
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "",
+ "value": ""
+ },
+ "datasource": null,
+ "includeAll": false,
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "topic",
+ "options": [
+
+ ],
+ "query": "$topologies.stormTopic",
+ "refresh": true,
+ "regex": "",
+ "type": "query"
+ },
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "partition-0",
+ "value": "partition-0"
+ },
+ "datasource": null,
+ "includeAll": true,
+ "multi": true,
+ "multiFormat": "glob",
+ "name": "partition",
+ "options": [
+ {
+ "text": "partition-0",
+ "value": "partition-0",
+ "selected": true
+ }
+ ],
+ "query": "$topic.stormPartition",
+ "refresh": true,
+ "type": "query"
+ }
+ ]
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": false,
+ "schemaVersion": 8,
+ "version": 23,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "tags": [
+ "storm"
+ ],
+ "title": "Storm Dashboards",
+ "type": "dashboards"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/9b4828f0/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json
new file mode 100644
index 0000000..ac1f829
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/grafana-dashboards/HDP/grafana-storm-kafka-offset.json
@@ -0,0 +1,258 @@
+{
+ "id": null,
+ "title": "Storm - Kafka-Offset",
+ "originalTitle": "Storm - Kafka-Offset",
+ "tags": [
+ "storm",
+ "builtin",
+ "2.4.0.0"
+ ],
+ "style": "dark",
+ "timezone": "browser",
+ "editable": true,
+ "hideControls": false,
+ "sharedCrosshair": false,
+ "rows": [
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "25px",
+ "panels": [
+ {
+ "content": "<h4 align=\"center\">Metrics to see the status for the Storm topics on a per partition level. Click on each row title to expand on demand to look at various metrics. </h4>\n<h6 style=\"color:red;\" align=\"center\">This dashboard is managed by Ambari. You may lose any changes made to this dashboard. If you want to customize, make your own copy.</h6>\n<h5 align=\"center\">Note: Period ('.') contained topology names are not supported.</h5>",
+ "editable": true,
+ "error": false,
+ "id": 14,
+ "isNew": true,
+ "links": [],
+ "mode": "html",
+ "span": 12,
+ "style": {},
+ "title": "",
+ "type": "text"
+ }
+ ],
+ "title": "New row"
+ },
+ {
+ "collapse": false,
+ "editable": true,
+ "height": "250px",
+ "panels": [
+ {
+ "aliasColors": {},
+ "bars": false,
+ "datasource": null,
+ "editable": true,
+ "error": false,
+ "fill": 1,
+ "grid": {
+ "leftLogBase": 1,
+ "leftMax": null,
+ "leftMin": null,
+ "rightLogBase": 1,
+ "rightMax": null,
+ "rightMin": null,
+ "threshold1": null,
+ "threshold1Color": "rgba(216, 200, 27, 0.27)",
+ "threshold2": null,
+ "threshold2Color": "rgba(234, 112, 112, 0.22)"
+ },
+ "id": 15,
+ "isNew": true,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 2,
+ "links": [],
+ "minSpan": 4,
+ "nullPointMode": "connected",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "repeat": "partition",
+ "scopedVars": {
+ "partition": {
+ "text": "partition-0",
+ "value": "partition-0",
+ "selected": true
+ }
+ },
+ "seriesOverrides": [],
+ "span": 12,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "aggregator": "avg",
+ "alias": "Kafka Offset",
+ "app": "nimbus",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "hosts": "",
+ "metric": "topology.*.kafka-topic.*.*.latestTimeOffset",
+ "precision": "default",
+ "refId": "B",
+ "sPartition": "partition-0",
+ "sTopic": "myKafkaTopic",
+ "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestTimeOffset",
+ "sTopology": "kafka-topology",
+ "seriesAggregator": "none",
+ "transform": "none"
+ },
+ {
+ "aggregator": "avg",
+ "alias": "Storm Offset",
+ "app": "nimbus",
+ "downsampleAggregator": "avg",
+ "errors": {},
+ "metric": "topology.*.kafka-topic.*.*.latestCompletedOffset",
+ "precision": "default",
+ "refId": "A",
+ "sComponent": "--acker",
+ "sPartition": "partition-0",
+ "sTopic": "myKafkaTopic",
+ "sTopoMetric": "topology.kafka-topology.kafka-topic.myKafkaTopic.partition-0.latestCompletedOffset",
+ "sTopology": "kafka-topology",
+ "seriesAggregator": "none",
+ "transform": "none"
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "$partition",
+ "tooltip": {
+ "shared": true,
+ "value_type": "cumulative"
+ },
+ "type": "graph",
+ "x-axis": true,
+ "y-axis": true,
+ "y_formats": [
+ "short",
+ "short"
+ ]
+ }
+ ],
+ "title": "New row"
+ }
+ ],
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "now": true,
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "templating": {
+ "list": [
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "kafka-topology",
+ "value": "kafka-topology"
+ },
+ "datasource": null,
+ "includeAll": false,
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "topologies",
+ "options": [
+ ],
+ "query": "topologies",
+ "refresh": true,
+ "regex": "",
+ "type": "query"
+ },
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "",
+ "value": ""
+ },
+ "datasource": null,
+ "includeAll": false,
+ "multi": false,
+ "multiFormat": "glob",
+ "name": "topic",
+ "options": [
+
+ ],
+ "query": "$topologies.stormTopic",
+ "refresh": true,
+ "regex": "",
+ "type": "query"
+ },
+ {
+ "allFormat": "glob",
+ "current": {
+ "text": "partition-0",
+ "value": "partition-0"
+ },
+ "datasource": null,
+ "includeAll": true,
+ "multi": true,
+ "multiFormat": "glob",
+ "name": "partition",
+ "options": [
+ {
+ "text": "partition-0",
+ "value": "partition-0",
+ "selected": true
+ }
+ ],
+ "query": "$topic.stormPartition",
+ "refresh": true,
+ "type": "query"
+ }
+ ]
+ },
+ "annotations": {
+ "list": []
+ },
+ "refresh": false,
+ "schemaVersion": 8,
+ "version": 23,
+ "links": [
+ {
+ "asDropdown": true,
+ "icon": "external link",
+ "tags": [
+ "storm"
+ ],
+ "title": "Storm Dashboards",
+ "type": "dashboards"
+ }
+ ]
+}
\ No newline at end of file