You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2015/12/08 00:27:14 UTC
tez git commit: TEZ-2346. TEZ-UI: Lazy load other info / counter data
(Sreenath Somarajapuram via jeagles)
Repository: tez
Updated Branches:
refs/heads/master 69e3de818 -> 5af06047f
TEZ-2346. TEZ-UI: Lazy load other info / counter data (Sreenath Somarajapuram via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5af06047
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5af06047
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5af06047
Branch: refs/heads/master
Commit: 5af06047f03eafbfa97ba0b5213b79bb6e38165e
Parents: 69e3de8
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Mon Dec 7 17:26:58 2015 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Mon Dec 7 17:26:58 2015 -0600
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../app/scripts/controllers/dags_controller.js | 136 ++++++++++++-------
.../src/main/webapp/app/scripts/helpers/misc.js | 5 +
.../app/scripts/mixins/paginated_content.js | 14 +-
.../app/scripts/models/TimelineRestAdapter.js | 66 ++++++++-
.../src/main/webapp/app/scripts/models/dag.js | 2 +
6 files changed, 168 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b8619dc..9031566 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES:
+ TEZ-2346. TEZ-UI: Lazy load other info / counter data
TEZ-2975. Bump up apache commons dependency.
TEZ-2970. Re-localization in TezChild does not use correct UGI.
TEZ-2968. Counter limits exception causes AM to crash.
@@ -273,6 +274,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES
+ TEZ-2346. TEZ-UI: Lazy load other info / counter data
TEZ-2975. Bump up apache commons dependency.
TEZ-2970. Re-localization in TezChild does not use correct UGI.
TEZ-2968. Counter limits exception causes AM to crash.
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
index ad9d636..7a52837 100644
--- a/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
+++ b/tez-ui/src/main/webapp/app/scripts/controllers/dags_controller.js
@@ -25,6 +25,8 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
pageSubTitle: 'All Tez DAGs',
+ columnSelectorMessage: 'Logs and counter columns needs more time to load.',
+
// query parameters supported through url. The same named variables in this controller get
// bound automatically to the ones defined in the route.
queryParams: {
@@ -35,6 +37,8 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
dagName_filter: 'dag_name'
},
+ _loadedAllData: false,
+
fromID: null,
status_filter: null,
@@ -63,6 +67,25 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
}));
}.observes('status_filter', 'user_filter', 'appId_filter', 'dagName_filter', 'id_filter'),
+ _otherInfoFieldsVisible: function () {
+ var visibleColumns = this.get('visibleColumnIds') || {},
+ columnIds;
+
+ if(visibleColumns['logs']) {
+ return true;
+ }
+
+ columnIds = Object.keys(visibleColumns);
+ for(var i = 0, length = columnIds.length, id; i < length; i++) {
+ id = columnIds[i];
+ if(visibleColumns[id] && id.indexOf('/') != -1) {
+ return true;
+ }
+ }
+
+ return false;
+ }.property('visibleColumnIds'),
+
_filterVisiblilityObserver: function () {
var visibleFilters = Em.Object.create();
this.get('columns').forEach(function (column) {
@@ -103,25 +126,50 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
store.unloadAll(childEntityType);
store.unloadAll('dagProgress');
- if(this.id_filter) {
- finder = store.find(childEntityType, this.id_filter).then(function (entity) {
- return (
+ that.set('_loadedAllData', false);
+ function loadAllData() {
+ return store.findQuery(childEntityType, that.getFilterProperties()).then(function (entities) {
+ that.set('_loadedAllData', true);
+ return entities;
+ });
+ }
+
+ function setEntities(entities) {
+ that.set('entities', entities);
+ that.set('loading', false);
+
+ return entities;
+ }
+
+ if(that.id_filter) {
+ finder = store.find(childEntityType, that.id_filter).then(function (entity) {
+ var entities = (
(that.dagName_filter && entity.get('name') != that.dagName_filter) ||
(that.appId_filter && entity.get('applicationId') != that.appId_filter) ||
(that.user_filter && entity.get('user') != that.user_filter) ||
(that.status_filter && entity.get('status') != that.status_filter)
) ? [] : [entity];
- }).catch(function () {
- return [];
+
+ return setEntities(entities);
});
}
else {
- finder = store.findQuery(childEntityType, this.getFilterProperties());
+ // Query just basic data
+ finder = store.findQuery(childEntityType, that.getFilterProperties('events,primaryfilters'));
+ finder = finder.then(setEntities).then(function (entities) {
+
+ // If countersVisible lazy load counters
+ return that.get('_otherInfoFieldsVisible') ? new Promise(function (fullfill) {
+ setTimeout(fullfill, 100); // Lazyload delay
+ }).then(loadAllData) : entities;
+
+ }).catch(function () {
+ // Basic data query failed, probably YARN-3530 fix is not in ATS. Load all data.
+ return loadAllData().then(setEntities);
+ });
}
- finder.then(function(entities){
- that.set('entities', entities);
- that.set('loading', false);
+ finder = finder.then(function(entities){
entities.forEach(function (dag) {
var appId = dag.get('applicationId');
@@ -133,8 +181,7 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
app.get('status'),
app.get('finalStatus')
));
- }).catch(function(error) {})
- .finally(function () {
+ }).finally(function (entities) {
if(dag.get('status') === 'RUNNING') {
App.Helpers.misc.removeRecord(store, 'dagProgress', dag.get('id'));
store.find('dagProgress', dag.get('id'), {
@@ -155,13 +202,20 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
});
}
});
+
}).catch(function(error){
Em.Logger.error(error);
var err = App.Helpers.misc.formatError(error, defaultErrMsg);
var msg = 'error code: %@, message: %@'.fmt(err.errCode, err.msg);
App.Helpers.ErrorBar.getInstance().show(msg, err.details);
});
- }.observes('fields'),
+ },
+
+ _onCountersVisible: function () {
+ if(this.get('_otherInfoFieldsVisible') && !this.get('_loadedAllData')) {
+ Em.run.once(this, this.loadEntities);
+ }
+ }.observes('_otherInfoFieldsVisible'),
actions : {
filterUpdated: function() {
@@ -185,24 +239,6 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
defaultColumnConfigs: function () {
var store = this.get('store');
- function onProgressChange() {
- var progress = this.get('dag.progress'),
- pct;
- if (Ember.typeOf(progress) === 'number') {
- pct = App.Helpers.number.fractionToPercentage(progress);
- this.set('progress', pct);
- }
- }
-
- function onStatusChange() {
- var status = this.get('dag.status');
- this.setProperties({
- status: status,
- statusIcon: App.Helpers.misc.getStatusClassForEntity(status,
- this.get('dag.hasFailedTaskAttempts'))
- });
- }
-
return [
{
id: 'dagName',
@@ -234,24 +270,26 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
headerCellName: 'Status',
templateName: 'components/basic-table/status-cell',
enableFilter: true,
+ contentPath: 'status',
+ observePath: true,
getCellContent: function(row) {
- var status = row.get('status'),
- content = Ember.Object.create({
- dag: row,
- status: status,
- statusIcon: App.Helpers.misc.getStatusClassForEntity(status,
- row.get('hasFailedTaskAttempts'))
- });
-
- if(status == 'RUNNING') {
- row.addObserver('progress', content, onProgressChange);
- row.addObserver('status', content, onStatusChange);
- }
-
- return content;
+ var status = row.get('status');
+ return {
+ status: status,
+ statusIcon: App.Helpers.misc.getStatusClassForEntity(status,
+ row.get('hasFailedTaskAttempts'))
+ };
}
},
{
+ id: 'progress',
+ headerCellName: 'Progress',
+ contentPath: 'progress',
+ enableFilter: true,
+ observePath: true,
+ templateName: 'components/basic-table/progress-cell'
+ },
+ {
id: 'startTime',
headerCellName: 'Start Time',
contentPath: 'startTime',
@@ -303,11 +341,15 @@ App.DagsController = Em.ObjectController.extend(App.PaginatedContentMixin, App.C
id: 'logs',
headerCellName: 'Logs',
templateName: 'components/basic-table/multi-logs-cell',
+ contentPath: 'containerLogs',
+ observePath: true,
getCellContent: function(row) {
- var content = {
- logs: row.get('containerLogs')
+ var containerLogs = row.get('containerLogs');
+ return containerLogs ? {
+ logs: containerLogs
+ } : {
+ isPending: true
};
- return content;
}
}
];
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
index b6fd690..8e19686 100644
--- a/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
+++ b/tez-ui/src/main/webapp/app/scripts/helpers/misc.js
@@ -302,6 +302,11 @@ App.Helpers.misc = {
* @return value
*/
getCounterCellContentFormatted: function (row) {
+ if(row.get(this.get('contentPath')) == undefined) {
+ return {
+ isPending: true
+ };
+ }
var value = App.Helpers.misc.getCounterCellContent.call(this, row);
return App.Helpers.number.formatNumThousands(value);
},
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
index 3acce39..d393df7 100644
--- a/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
+++ b/tez-ui/src/main/webapp/app/scripts/mixins/paginated_content.js
@@ -44,11 +44,13 @@ App.PaginatedContentMixin = Em.Mixin.create({
entities: [],
_paginationFilters: {},
- loading: true,
+ loading: false,
load: function() {
- this.resetNavigation();
- this.loadEntities();
+ if(!this.get('loading')) {
+ this.resetNavigation();
+ this.loadEntities();
+ }
}.observes('rowCount'),
lastPage: function () {
@@ -149,7 +151,7 @@ App.PaginatedContentMixin = Em.Mixin.create({
return p.join(',');
},
- getFilterProperties: function() {
+ getFilterProperties: function(fields) {
var params = {
limit: Math.min(this.rowCount, this.get('maxRowCount')) + 1
};
@@ -191,6 +193,10 @@ App.PaginatedContentMixin = Em.Mixin.create({
params['fromId'] = this.get('fromID');
}
+ if (fields) {
+ params['fields'] = fields;
+ }
+
return params;
},
});
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
index b4a2062..71f96df 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/TimelineRestAdapter.js
@@ -85,27 +85,74 @@ App.TimelineSerializer = DS.RESTSerializer.extend({
}
});
+function getStatus(source) {
+ var status = Em.get(source, 'otherinfo.status') || Em.get(source, 'primaryfilters.status.0'),
+ event = source.events;
+
+ if(!status && event) {
+ if(event.findBy('eventtype', 'DAG_STARTED')) {
+ status = 'RUNNING';
+ }
+ }
+
+ return status;
+}
var timelineJsonToDagMap = {
id: 'entity',
submittedTime: 'starttime',
- startTime: 'otherinfo.startTime',
- endTime: 'otherinfo.endTime',
+
+ startTime: {
+ custom: function(source) {
+ var time = Em.get(source, 'otherinfo.startTime'),
+ event = source.events;
+
+ if(!time && event) {
+ event = event.findBy('eventtype', 'DAG_STARTED');
+ if(event) {
+ time = event.timestamp;
+ }
+ }
+
+ return time;
+ }
+ },
+ endTime: {
+ custom: function(source) {
+ var time = Em.get(source, 'otherinfo.endTime'),
+ event = source.events;
+
+ if(!time && event) {
+ event = event.findBy('eventtype', 'DAG_FINISHED');
+ if(event) {
+ time = event.timestamp;
+ }
+ }
+
+ return time;
+ }
+ },
+
name: 'primaryfilters.dagName.0',
user: 'primaryfilters.user.0',
- status: 'otherinfo.status',
+ status: {
+ custom: getStatus
+ },
progress: {
custom: function(source) {
- return Em.get(source, 'otherinfo.status') == 'SUCCEEDED' ? 1 : null;
+ var status = getStatus(source);
+ return status == 'SUCCEEDED' ? 1 : null;
}
},
containerLogs: {
custom: function(source) {
-
var containerLogs = [];
var otherinfo = Em.get(source, 'otherinfo');
+ if(!otherinfo) {
+ return undefined;
+ }
for (var key in otherinfo) {
if (key.indexOf('inProgressLogsURL_') === 0) {
var logs = Em.get(source, 'otherinfo.' + key);
@@ -130,7 +177,14 @@ var timelineJsonToDagMap = {
numFailedTasks: 'otherinfo.numFailedTasks',
diagnostics: 'otherinfo.diagnostics',
- counterGroups: 'otherinfo.counters.counterGroups',
+ counterGroups: {
+ custom: function(source) {
+ var otherinfo = source.otherinfo;
+ if(otherinfo) {
+ return Em.get(otherinfo, 'counters.counterGroups') || [];
+ }
+ }
+ },
planName: 'otherinfo.dagPlan.dagName',
planVersion: 'otherinfo.dagPlan.version',
http://git-wip-us.apache.org/repos/asf/tez/blob/5af06047/tez-ui/src/main/webapp/app/scripts/models/dag.js
----------------------------------------------------------------------
diff --git a/tez-ui/src/main/webapp/app/scripts/models/dag.js b/tez-ui/src/main/webapp/app/scripts/models/dag.js
index ff4a388..3ea76d2 100644
--- a/tez-ui/src/main/webapp/app/scripts/models/dag.js
+++ b/tez-ui/src/main/webapp/app/scripts/models/dag.js
@@ -21,6 +21,8 @@ App.Dag = App.AbstractEntity.extend({
return App.Helpers.misc.getDagIndexFromDagId(this.get('id'));
}.property('id'),
+ progress: DS.attr('number'),
+
submittedTime: DS.attr('number'),
// start time of the entity