You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by pr...@apache.org on 2015/09/14 19:31:15 UTC
[2/2] tez git commit: TEZ-2817. Tez UI: update in progress counter
data for the dag vertices and tasks table (sree via pramachandran)
TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table (sree via pramachandran)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b3ad5957
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b3ad5957
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b3ad5957
Branch: refs/heads/master
Commit: b3ad59578ba801a44c8e97b39534cf5efb90c1f1
Parents: 1e58a0e
Author: Prakash Ramachandran <pr...@hortonworks.com>
Authored: Mon Sep 14 23:00:13 2015 +0530
Committer: Prakash Ramachandran <pr...@hortonworks.com>
Committed: Mon Sep 14 23:00:13 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
tez-ui/src/main/webapp/app/scripts/app.js | 27 ++++---
.../app/scripts/controllers/dag_controller.js | 14 +++-
.../webapp/app/scripts/controllers/dag_tasks.js | 29 +++-----
.../app/scripts/controllers/dag_vertices.js | 23 +++++-
.../scripts/helpers/entity-array-pollster.js | 26 ++++++-
.../src/main/webapp/app/scripts/helpers/misc.js | 75 +++++++++++++++++++-
.../src/main/webapp/app/scripts/models/dag.js | 7 +-
8 files changed, 165 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bb023b1..e15b08d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -176,6 +176,7 @@ Release 0.7.1: Unreleased
INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-2817. Tez UI: update in progress counter data for the dag vertices and tasks table
TEZ-2813. Tez UI: add counter data for rest api calls to AM Web Services v2
TEZ-2660. Tez UI: need to show application page even if system metrics publish is disabled.
TEZ-2787. Tez AM should have java.io.tmpdir=./tmp to be consistent with tasks
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/app.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/app.js b/tez-ui/src/main/webapp/app/scripts/app.js
index 13c08cb..2ad5fdd 100644
--- a/tez-ui/src/main/webapp/app/scripts/app.js
+++ b/tez-ui/src/main/webapp/app/scripts/app.js
@@ -206,16 +206,20 @@ App.ready = function () {
App.VertexInfoAdapter = App.AMInfoAdapter.extend({
namespace: App.Configs.restNamespace.aminfoV2,
- ajax: function(url, method, hash) {
- var options = hash.data || {};
- url = url.replace('__app_id__', options.appId)
- .fmt(options.dagIdx);
- delete options['dagIdx'];
- delete options['appId'];
- return this._super(url, method, hash);
+ findQuery: function(store, type, query) {
+ var record = query.metadata;
+ delete query.metadata;
+ return this.ajax(
+ this.buildURL(Ember.String.pluralize(type.typeKey),
+ record.vertexID, Em.Object.create(record)), 'GET', { data: query});
},
- pathForType: function() {
- return 'verticesInfo?dagID=%@';
+ buildURL: function(type, id, record) {
+ var url = this._super(type, undefined, record);
+ return url.replace('__app_id__', record.get('appID'))
+ .fmt(record.get('dagID'), id, record.get('counters'));
+ },
+ pathForType: function(typeName) {
+ return 'verticesInfo?dagID=%@&vertexID=%@&counters=%@';
}
});
@@ -231,12 +235,13 @@ App.ready = function () {
buildURL: function(type, id, record) {
var url = this._super(type, undefined, record);
return url.replace('__app_id__', record.get('appID'))
- .fmt(record.get('dagID'), id);
+ .fmt(record.get('dagID'), id, record.get('counters'));
},
pathForType: function(typeName) {
- return 'tasksInfo?dagID=%@&taskID=%@';
+ return 'tasksInfo?dagID=%@&taskID=%@&counters=%@';
}
});
+
};
$.ajaxPrefilter(function(options, originalOptions, jqXHR) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
index 8804813..cd750d0 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_controller.js
@@ -80,6 +80,7 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
["loading", "id", "model.status"].forEach(function(item) {
Em.addObserver(that, item, that.startAMInfoUpdateService);
});
+ that.startAMInfoUpdateService();
}
}
});
@@ -121,9 +122,12 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
if (!dagId) return;
- verticesInfoLoader = this.store.find('vertexInfo', {
- appId: that.get('applicationId'),
- dagIdx: that.get('idx')
+ verticesInfoLoader = this.store.findQuery('vertexInfo', {
+ metadata: {
+ appID: that.get('applicationId'),
+ dagID: that.get('idx'),
+ counters: App.get('vertexCounters')
+ }
});
verticesInfoLoader.then(function(verticesInfo) {
@@ -156,6 +160,10 @@ App.DagController = Em.ObjectController.extend(App.Helpers.DisplayHelper, {
Em.addObserver(that, item, that.stopAMInfoUpdateService);
});
}
+ else {
+ that.updateAMDagInfo();
+ that.updateAMVerticesInfo();
+ }
},
dostopAMInfoUpdateService: function() {
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
index 0bd9df0..567b608 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_tasks.js
@@ -39,8 +39,8 @@ App.DagTasksController = App.TablePageController.extend({
},
pollsterControl: function () {
- if(this.get('controllers.dag.status') == 'RUNNING' &&
- this.get('controllers.dag.amWebServiceVersion') != '1' &&
+ if(this.get('status') == 'RUNNING' &&
+ this.get('amWebServiceVersion') != '1' &&
!this.get('loading') &&
this.get('isActive') &&
this. get('rowsDisplayed.length') > 0) {
@@ -49,34 +49,27 @@ App.DagTasksController = App.TablePageController.extend({
else {
this.get('pollster').stop();
}
- }.observes('controllers.dag.status',
- 'controllers.dag.amWebServiceVersion',
- 'rowsDisplayed',
- 'loading',
- 'isActive'),
-
- reset: function () {
- this.set('pollster.options', null);
- },
+ }.observes('status', 'amWebServiceVersion', 'rowsDisplayed', 'loading', 'isActive'),
pollsterOptionsObserver: function () {
var rows = this.get('rowsDisplayed');
this.set('pollster.targetRecords', rows);
- rows = rows.filter(function (row) {
- return row.get('status') != 'SUCCEEDED';
- });
-
this.set('pollster.options', (rows && rows.length) ? {
- appID: this.get('controllers.dag.model.applicationId'),
- dagID: this.get('controllers.dag.model.idx'),
+ appID: this.get('applicationId'),
+ dagID: this.get('idx'),
+ counters: this.get('countersDisplayed'),
taskID: rows.map(function (row) {
var taskIndex = App.Helpers.misc.getIndexFromId(row.get('id')),
vertexIndex = App.Helpers.misc.getIndexFromId(row.get('vertexID'));
return '%@_%@'.fmt(vertexIndex, taskIndex);
}).join(',')
} : null);
- }.observes('controllers.dag.model.applicationId', 'controllers.dag.model.idx', 'rowsDisplayed'),
+ }.observes('applicationId', 'idx', 'rowsDisplayed', 'counters'),
+
+ countersDisplayed: function () {
+ return App.Helpers.misc.getCounterQueryParam(this.get('columns'));
+ }.property('columns'),
beforeLoad: function () {
var dagController = this.get('controllers.dag'),
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
index ea25818..5f60d5d 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dag_vertices.js
@@ -26,6 +26,11 @@ App.DagVerticesController = App.TablePageController.extend({
cacheDomain: Ember.computed.alias('controllers.dag.id'),
+ pollster: {},
+ amInfoUpdateServiceObserver: function () {
+ this.set('pollster.isRunning', !!this.get('amInfoUpdateService'));
+ }.observes('amInfoUpdateService').on('init'),
+
beforeLoad: function () {
var dagController = this.get('controllers.dag'),
model = dagController.get('model');
@@ -69,7 +74,8 @@ App.DagVerticesController = App.TablePageController.extend({
metadata: {
appId: that.get('applicationId'),
dagIdx: that.get('idx'),
- vertexIds: runningVerticesIdx.join(',')
+ vertexIds: runningVerticesIdx.join(','),
+ counters: counters
}
}).then(function(vertexProgressInfo) {
that.set('controllers.dag.amVertexInfo', vertexProgressInfo);
@@ -79,10 +85,18 @@ App.DagVerticesController = App.TablePageController.extend({
}
},
+ _onColumnChange: function () {
+ App.set('vertexCounters', App.Helpers.misc.getCounterQueryParam(this.get('columns')));
+ }.observes('columns').on('init'),
+
overlayVertexInfo: function(vertex, amVertexInfo) {
if (Em.isNone(amVertexInfo) || Em.isNone(vertex)) return;
amVertexInfo.set('_amInfoLastUpdatedTime', moment());
vertex.setProperties(amVertexInfo.getProperties('status', 'progress', '_amInfoLastUpdatedTime'));
+
+ vertex.set('counterGroups',
+ App.Helpers.misc.mergeCounterInfo(vertex.get('counterGroups'), amVertexInfo.get('counters')).slice(0)
+ );
},
updateVertexInfo: function() {
@@ -152,6 +166,13 @@ App.DagVerticesController = App.TablePageController.extend({
}
},
{
+ id: 'progress',
+ headerCellName: 'Progress',
+ contentPath: 'progress',
+ observePath: true,
+ templateName: 'components/basic-table/progress-cell'
+ },
+ {
id: 'startTime',
headerCellName: 'Start Time',
contentPath: 'startTime',
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
index 040ec8f..fb3dd76 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/entity-array-pollster.js
@@ -64,6 +64,10 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({
}
},
+ _optionObserver: function () {
+ Em.run.later(this, this.onPoll, 10);
+ }.observes('options'),
+
_callIfRunning: function (that, funName) {
return function (data) {
var fun = that.get(funName);
@@ -93,9 +97,27 @@ App.Helpers.EntityArrayPollster = App.Helpers.Pollster.extend({
if(polledRecords && targetRecords) {
targetRecords.forEach(function (row) {
- var info = polledRecords.findBy('id', row.get('id'));
- if(info) {
+ var info = polledRecords.findBy('id', row.get('id')),
+ merge = !!info;
+
+ if(merge && row.get('progress') && info.get('progress')) {
+ if(row.get('progress') >= info.get('progress')) {
+ merge = false;
+ }
+ }
+
+ if(merge) {
row.setProperties(info.getProperties.apply(info, mergeProperties));
+
+ if(info.get('counters')) {
+ row.set('counterGroups',
+ App.Helpers.misc.mergeCounterInfo(
+ row.get('counterGroups'),
+ info.get('counters')
+ ).slice(0)
+ );
+ }
+
}
});
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index 223b686..ddd7d6a 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -47,6 +47,8 @@ App.Helpers.misc = {
return 'success';
case 'UNDEFINED':
return 'unknown';
+ case 'SCHEDULED':
+ return 'schedule';
default:
return 'submitted';
}
@@ -158,7 +160,7 @@ App.Helpers.misc = {
* @param counterConfigs Array
* @return Normalized configurations
*/
- normalizeCounterConfigs: function (counterConfigs) {
+ normalizeCounterConfigs: function (counterConfigs, inProgress) {
return counterConfigs.map(function (configuration) {
var groupName = configuration.counterGroupName || configuration.groupId,
counterName = configuration.counterName || configuration.counterId;
@@ -169,6 +171,12 @@ App.Helpers.misc = {
);
configuration.id = '%@/%@'.fmt(groupName, counterName),
+ configuration.observePath = true;
+ configuration.contentPath = 'counterGroups';
+ configuration.counterGroupName = groupName;
+ configuration.counterName = counterName;
+ configuration.searchAndSortable = !inProgress;
+
configuration.getSortValue = App.Helpers.misc.getCounterCellContent;
configuration.getCellContent =
configuration.getSearchValue = App.Helpers.misc.getCounterCellContentFormatted;
@@ -176,6 +184,71 @@ App.Helpers.misc = {
});
},
+ getCounterQueryParam: function (columns) {
+ var counterHash = {},
+ counters = [];
+
+ columns.forEach(function (column) {
+ var groupName = column.get('counterGroupName'),
+ counterName = column.get('counterName');
+ if(column.get('contentPath') == 'counterGroups') {
+ counterHash[groupName] = counterHash[groupName] || [];
+ counterHash[groupName].push(counterName);
+ }
+ });
+ for(var groupName in counterHash) {
+ counters.push('%@/%@'.fmt(groupName, counterHash[groupName].join(',')));
+ }
+
+ return counters.join(';');
+ },
+
+ /*
+ * Merges counter information from AM counter object into ATS counters array
+ */
+ mergeCounterInfo: function (targetATSCounters, sourceAMCounters) {
+ var atsCounters, atsCounter,
+ counters;
+
+ targetATSCounters = targetATSCounters || [];
+
+ try{
+ for(var counterGroupName in sourceAMCounters) {
+ counters = sourceAMCounters[counterGroupName],
+ atsCounters = targetATSCounters.findBy('counterGroupName', counterGroupName);
+ if(!atsCounters) {
+ atsCounters = [];
+ targetATSCounters.push({
+ counterGroupName: counterGroupName,
+ counterGroupDisplayName: counterGroupName,
+ counters: atsCounters
+ });
+ }
+ else {
+ atsCounters = atsCounters.counters;
+ }
+ for(var counterName in counters) {
+ atsCounter = atsCounters.findBy('counterName', counterName);
+ if(atsCounter) {
+ Em.set(atsCounter, 'counterValue', counters[counterName]);
+ }
+ else {
+ atsCounters.push({
+ "counterName": counterName,
+ "counterDisplayName": counterName,
+ "counterValue": counters[counterName]
+ });
+ }
+ }
+ }
+ }
+ catch(e){
+ Em.Logger.info("Counter merge failed", e);
+ }
+
+ return targetATSCounters;
+ },
+
/*
* Creates column definitions form configuration object array
* @param columnConfigs Array
http://git-wip-us.apache.org/repos/asf/tez/blob/b3ad5957/tez-ui/src/main/webapp/app/scripts/models/dag.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js
index c00e591..3e48cfe 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/dag.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js
@@ -138,6 +138,8 @@ App.Vertex = App.AbstractEntity.extend({
return !!f && f > 0;
}.property('failedTasks'),
+ progress: DS.attr('number'),
+
/**
* Vertex type has to be one of the types defined in 'App.VertexType'
* @return {string}
@@ -428,12 +430,15 @@ App.VertexInfo = DS.Model.extend({
killedTaskAttempts: DS.attr('number'),
pendingTasks: function() {
return this.get('totalTasks') - this.get('runningTasks') - this.get('succeededTasks');
- }.property('totalTasks', 'runningTasks', 'succeededTasks')
+ }.property('totalTasks', 'runningTasks', 'succeededTasks'),
+
+ counters: DS.attr('object')
});
App.TaskInfo = DS.Model.extend({
progress: DS.attr('number'),
status: DS.attr('string'),
+ counters: DS.attr('object')
});
App.KVDatum = DS.Model.extend({