You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2015/05/17 21:41:17 UTC
ambari git commit: AMBARI-11195. Need to be able to create customized
graphs based on YARN queue-specific metrics. (jaimin)
Repository: ambari
Updated Branches:
refs/heads/trunk 67c65aba9 -> 421770b27
AMBARI-11195. Need to be able to create customized graphs based on YARN queue-specific metrics. (jaimin)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/421770b2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/421770b2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/421770b2
Branch: refs/heads/trunk
Commit: 421770b27048fedfdfbff7c9f659e5f1ade387b7
Parents: 67c65ab
Author: Jaimin Jetly <ja...@hortonworks.com>
Authored: Sun May 17 12:40:57 2015 -0700
Committer: Jaimin Jetly <ja...@hortonworks.com>
Committed: Sun May 17 12:41:05 2015 -0700
----------------------------------------------------------------------
ambari-web/app/assets/test/tests.js | 1 +
.../service/widgets/create/step2_controller.js | 13 +-
.../service/widgets/create/step3_controller.js | 10 +-
.../service/widgets/create/wizard_controller.js | 67 +++++-
.../widgets/create/wizard_controller_test.js | 235 +++++++++++++++++++
5 files changed, 303 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/assets/test/tests.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/assets/test/tests.js b/ambari-web/app/assets/test/tests.js
index 64456a9..95e9f35 100644
--- a/ambari-web/app/assets/test/tests.js
+++ b/ambari-web/app/assets/test/tests.js
@@ -87,6 +87,7 @@ var files = ['test/init_model_test',
'test/controllers/main/service/reassign/step4_controller_test',
'test/controllers/main/service/reassign/step6_controller_test',
'test/controllers/main/service/reassign/step7_controller_test',
+ 'test/controllers/main/service/widgets/create/wizard_controller_test',
'test/controllers/main/service/widgets/create/step1_controller_test',
'test/controllers/main/service/widgets/create/step2_controller_test',
'test/controllers/main/dashboard_test',
http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
index 9c48872..65af9cd 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/step2_controller.js
@@ -343,14 +343,17 @@ App.WidgetWizardStep2Controller = Em.Controller.extend({
value = '${';
expression.data.forEach(function (element) {
if (element.isMetric) {
- metrics.push({
+ var metricObj = {
"name": element.name,
"service_name": element.serviceName,
"component_name": element.componentName,
- "metric_path": element.metricPath,
- "host_component_criteria": element.hostComponentCriteria,
- "category": element.category
- });
+ "metric_path": element.metricPath
+ };
+ if (element.hostComponentCriteria) {
+ metricObj.host_component_criteria = element.hostComponentCriteria;
+ }
+ metrics.push(metricObj);
+
}
value += element.name;
}, this);
http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
index f946b74..24b32f8 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/step3_controller.js
@@ -122,15 +122,7 @@ App.WidgetWizardStep3Controller = Em.Controller.extend({
description: this.get('widgetDescription') || "",
scope: this.get('widgetScope').toUpperCase(),
author: this.get('widgetAuthor'),
- "metrics": this.get('widgetMetrics').map(function (metric) {
- return {
- "name": metric.name,
- "service_name": metric.serviceName,
- "component_name": metric.componentName,
- "metric_path": metric.metricPath,
- "host_component_criteria": metric.hostComponentCriteria
- }
- }),
+ metrics: this.get('widgetMetrics'),
values: this.get('widgetValues').map(function (value) {
delete value.computedValue;
return value;
http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
index 08a5582..4c1dd06 100644
--- a/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
+++ b/ambari-web/app/controllers/main/service/widgets/create/wizard_controller.js
@@ -180,17 +180,17 @@ App.WidgetWizardController = App.WizardController.extend({
* @returns {$.Deferred}
*/
loadAllMetrics: function () {
- var widgetMetrics = this.getDBProperty('allMetrics');
+ var allMetrics = this.getDBProperty('allMetrics');
var self = this;
var dfd = $.Deferred();
- if (widgetMetrics.length === 0) {
+ if (allMetrics.length === 0) {
this.loadAllMetricsFromServer(function () {
dfd.resolve(self.get('content.allMetrics'));
});
} else {
- this.set('content.allMetrics', widgetMetrics);
- dfd.resolve(widgetMetrics);
+ this.set('content.allMetrics', allMetrics);
+ dfd.resolve(allMetrics);
}
return dfd.promise();
},
@@ -223,23 +223,25 @@ App.WidgetWizardController = App.WizardController.extend({
var self = this;
var result = [];
var metrics = {};
-
+ var slaveComponents = App.StackServiceComponent.find().filterProperty('isSlave').mapProperty('componentName');
if (json) {
json.items.forEach(function (service) {
var data = service.artifacts[0].artifact_data[service.StackServices.service_name];
for (var componentName in data) {
+ var isSlave = slaveComponents.contains(componentName);
for (var level in data[componentName]) {
var metricTypes = data[componentName][level]; //Ganglia or JMX
metricTypes.forEach(function (_metricType) {
metrics = _metricType['metrics']['default'];
var type = _metricType["type"].toUpperCase();
- if (!(type === 'JMX' && level.toUpperCase() === 'COMPONENT')) {
+ if (!((type === 'JMX' && level.toUpperCase() === 'COMPONENT') || (isSlave && level.toUpperCase() === 'HOSTCOMPONENT'))) {
for (var widgetId in metrics) {
+ var _metrics = metrics[widgetId];
var metricObj = {
widget_id: widgetId,
- point_in_time: metrics[widgetId].pointInTime,
- temporal: metrics[widgetId].temporal,
- name: metrics[widgetId].name,
+ point_in_time: _metrics.pointInTime,
+ temporal: _metrics.temporal,
+ name: _metrics.name,
level: level.toUpperCase(),
type: type,
component_name: componentName,
@@ -256,9 +258,56 @@ App.WidgetWizardController = App.WizardController.extend({
}
}, this);
}
+ if (!!App.YARNService.find("YARN")) {
+ result = this.substitueQueueMetrics(result);
+ }
this.save('allMetrics', result);
},
+
+ /**
+ * @name substitueQueueMetrics
+ * @param metrics
+ * @return {Array} array of metric objects with regex substituted with actual metrics names
+ */
+ substitueQueueMetrics: function (metrics) {
+ var result = [];
+ var queuePaths = App.YARNService.find("YARN").get("allQueueNames");
+ var queueNames = [];
+ var queueMetricName;
+ queuePaths.forEach(function (_queuePath) {
+ var queueName = _queuePath.replace(/\//g, ".");
+ queueNames.push(queueName);
+ }, this);
+ var regexpForAMS = new RegExp("^yarn.QueueMetrics.Queue=\\(\\.\\+\\).*$");
+ var regexpForJMX = new RegExp("\\(\\.\\+\\)", "g");
+ var replaceRegexForMetricName = regexpForJMX;
+ var replaceRegexForMetricPath = new RegExp("\\$\\d\\..*\\)(?=\\/)", "g");
+ metrics.forEach(function (_metric) {
+ var isAMSQueueMetric = regexpForAMS.test(_metric.name);
+ var isJMXQueueMetrics = regexpForJMX.test(_metric.name);
+ if ((_metric.type === 'GANGLIA' && isAMSQueueMetric) || (_metric.type === 'JMX' && isJMXQueueMetrics)) {
+ queuePaths.forEach(function (_queuePath) {
+ queueMetricName = '';
+ if (_metric.type === 'GANGLIA') {
+ queueMetricName = _queuePath.replace(/\//g, ".");
+ } else if (_metric.type === 'JMX') {
+ _queuePath.split("/").forEach(function(_metricName, index){
+ queueMetricName = queueMetricName + ',q' + index + '=' + _metricName;
+ }, this);
+ }
+ var metricName = _metric.name.replace(replaceRegexForMetricName, queueMetricName);
+ var newMetricPath = _metric.widget_id.replace(replaceRegexForMetricPath, _queuePath);
+ var newQueueMetric = $.extend(true, {}, _metric, {name: metricName, widget_id: newMetricPath});
+ result.pushObject(newQueueMetric);
+ }, this);
+ } else {
+ result.pushObject(_metric);
+ }
+ }, this);
+ return result;
+ },
+
/**
*
* @param metricObj {Object}
http://git-wip-us.apache.org/repos/asf/ambari/blob/421770b2/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
----------------------------------------------------------------------
diff --git a/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
new file mode 100644
index 0000000..83ce39f
--- /dev/null
+++ b/ambari-web/test/controllers/main/service/widgets/create/wizard_controller_test.js
@@ -0,0 +1,235 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+App = require('app');
+
+require('controllers/main/service/widgets/create/wizard_controller');
+
+
+describe('App.WidgetWizardController', function () {
+ var controller;
+
+ /**
+ * tests the function with following hierarchical queue scenario
+ * root
+ * |
+ * queue1
+ * / \
+ * queue2 queue3
+ *
+ */
+ describe("#substitueQueueMetrics", function () {
+ beforeEach(function () {
+ controller = App.WidgetWizardController.create();
+ sinon.stub(App.YARNService, 'find', function (k) {
+ if ('YARN' === k) return Em.Object.create({
+ 'allQueueNames': ["root", "root/queue1", "root/queue1/queue2", "root/queue1/queue3"]
+ });
+ });
+ });
+ afterEach(function () {
+ controller = '';
+ App.YARNService.find.restore();
+ });
+
+
+ var testCases = [
+ {
+ msg: 'AMS Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues',
+ inputMetrics: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ level: 'COMPONENT',
+ name: 'yarn.QueueMetrics.Queue=(.+).AppsFailed',
+ point_in_time: false,
+ service_name: 'YARN',
+ temporal: true,
+ type: 'GANGLIA',
+ widget_id: 'metrics/yarn/Queue/$1.replaceAll("([.])","/")/AppsFailed'
+ }
+ ],
+ expectedResult: [
+ {
+ "component_name": "RESOURCEMANAGER",
+ "level": "COMPONENT",
+ "name": "yarn.QueueMetrics.Queue=root.AppsFailed",
+ "point_in_time": false,
+ "service_name": "YARN",
+ "temporal": true,
+ "type": "GANGLIA",
+ "widget_id": "metrics/yarn/Queue/root/AppsFailed"
+ },
+ {
+ "component_name": "RESOURCEMANAGER",
+ "level": "COMPONENT",
+ "name": "yarn.QueueMetrics.Queue=root.queue1.AppsFailed",
+ "point_in_time": false,
+ "service_name": "YARN",
+ "temporal": true,
+ "type": "GANGLIA",
+ "widget_id": "metrics/yarn/Queue/root/queue1/AppsFailed"
+ },
+ {
+ "component_name": "RESOURCEMANAGER",
+ "level": "COMPONENT",
+ "name": "yarn.QueueMetrics.Queue=root.queue1.queue2.AppsFailed",
+ "point_in_time": false,
+ "service_name": "YARN",
+ "temporal": true,
+ "type": "GANGLIA",
+ "widget_id": "metrics/yarn/Queue/root/queue1/queue2/AppsFailed"
+ },
+ {
+ "component_name": "RESOURCEMANAGER",
+ "level": "COMPONENT",
+ "name": "yarn.QueueMetrics.Queue=root.queue1.queue3.AppsFailed",
+ "point_in_time": false,
+ "service_name": "YARN",
+ "temporal": true,
+ "type": "GANGLIA",
+ "widget_id": "metrics/yarn/Queue/root/queue1/queue3/AppsFailed"
+ }
+ ]
+ },
+ {
+ msg: 'JMX Queue metric with regex as name and regex as path should be replaced with actual metric name and path of all existing queues',
+ inputMetrics: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics(.+).AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/$1.replaceAll(",q(\d+)=","/").substring(1)/AppsFailed'
+ }
+ ],
+ expectedResult: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/root/AppsFailed'
+ },
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/root/queue1/AppsFailed'
+ },
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue2.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/root/queue1/queue2/AppsFailed'
+ },
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics,q0=root,q1=queue1,q2=queue3.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/root/queue1/queue3/AppsFailed'
+ }
+ ]
+ },
+ {
+ msg: 'AMS Queue metric without regex in name and path should retain same name and path',
+ inputMetrics: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ level: 'COMPONENT',
+ name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+ point_in_time: false,
+ service_name: 'YARN',
+ temporal: true,
+ type: 'GANGLIA',
+ widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+ }
+ ],
+ expectedResult: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ level: 'COMPONENT',
+ name: 'yarn.QueueMetrics.Queue.Clustermetrics.AppsFailed',
+ point_in_time: false,
+ service_name: 'YARN',
+ temporal: true,
+ type: 'GANGLIA',
+ widget_id: 'metrics/yarn/Queue/Clustermetrics/AppsFailed'
+ }
+ ]
+ },
+ {
+ msg: 'JMX Queue metric without regex in name and path should retain same name and path',
+ inputMetrics: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+ }
+ ],
+ expectedResult: [
+ {
+ component_name: 'RESOURCEMANAGER',
+ host_component_criteria: 'host_components/HostRoles/ha_state=ACTIVE',
+ level: 'HOSTCOMPONENT',
+ name: 'Hadoop:service=ResourceManager,name=QueueMetrics.clusterMetric.AppsFailed',
+ point_in_time: true,
+ service_name: 'YARN',
+ temporal: false,
+ type: 'JMX',
+ widget_id: 'metrics/yarn/Queue/clusterMetric/AppsFailed'
+ }
+ ]
+ }
+ ];
+ testCases.forEach(function (_testCase) {
+ it(_testCase.msg, function () {
+ var result = controller.substitueQueueMetrics(_testCase.inputMetrics);
+ expect(JSON.stringify(result)).to.equal(JSON.stringify(_testCase.expectedResult));
+ });
+ });
+ });
+});