You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by pa...@apache.org on 2017/02/08 10:00:52 UTC
ambari git commit: AMBARI-19890. Hive2: Query Runtime Prediction:
Compact Visual Explain Plan (pallavkul)
Repository: ambari
Updated Branches:
refs/heads/trunk 2720b5c1f -> df50b6c01
AMBARI-19890. Hive2: Query Runtime Prediction: Compact Visual Explain Plan (pallavkul)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/df50b6c0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/df50b6c0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/df50b6c0
Branch: refs/heads/trunk
Commit: df50b6c01a07685b5c5f446e334314b9620c5a99
Parents: 2720b5c
Author: pallavkul <pa...@gmail.com>
Authored: Wed Feb 8 15:30:07 2017 +0530
Committer: pallavkul <pa...@gmail.com>
Committed: Wed Feb 8 15:30:07 2017 +0530
----------------------------------------------------------------------
.../src/main/resources/ui/app/adapters/query.js | 5 +
.../ui/app/components/query-result-table.js | 4 +
.../ui/app/components/visual-explain.js | 65 ++
.../main/resources/ui/app/controllers/udfs.js | 22 +
.../resources/ui/app/routes/queries/query.js | 33 +-
.../src/main/resources/ui/app/services/query.js | 12 +-
.../src/main/resources/ui/app/styles/app.scss | 73 ++-
.../app/templates/components/visual-explain.hbs | 37 ++
.../ui/app/templates/queries/query.hbs | 50 +-
.../resources/ui/app/utils/hive-explainer.js | 645 +++++++++++++++++++
.../hive20/src/main/resources/ui/bower.json | 1 +
.../src/main/resources/ui/ember-cli-build.js | 1 +
12 files changed, 922 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/adapters/query.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/adapters/query.js b/contrib/views/hive20/src/main/resources/ui/app/adapters/query.js
index ccda9d4..e519e64 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/adapters/query.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/adapters/query.js
@@ -41,6 +41,11 @@ export default ApplicationAdapter.extend({
return this.ajax(url, 'GET')
},
+ getVisualExplainJson(jobId){
+ let url = this.buildURL() + jobId + '/results?first=true';
+ return this.ajax(url, 'GET');
+ },
+
retrieveQueryLog(logFile){
let url = '';
url = this.buildURL().replace('/jobs','') + '/files' + logFile;
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/components/query-result-table.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/query-result-table.js b/contrib/views/hive20/src/main/resources/ui/app/components/query-result-table.js
index 919127f..0373f72 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/components/query-result-table.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/components/query-result-table.js
@@ -126,6 +126,10 @@ export default Ember.Component.extend({
console.log('downloadAsCsv with jobId == ', jobId );
console.log('downloadAsCsv with pathName == ', pathName );
this.sendAction('downloadAsCsv', jobId, pathName);
+ },
+
+ showVisualExplain(){
+ this.sendAction('showVisualExplain');
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js b/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js
new file mode 100644
index 0000000..6551974
--- /dev/null
+++ b/contrib/views/hive20/src/main/resources/ui/app/components/visual-explain.js
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+import explain from '../utils/hive-explainer';
+
+export default Ember.Component.extend({
+ visualExplainJson:'',
+
+ visualExplainInput: Ember.computed('visualExplainJson', function () {
+ return this.get('visualExplainJson');
+ }),
+
+ isQueryRunning:false,
+
+ didInsertElement(){
+ this._super(...arguments);
+
+ const width = '100vw', height = '100vh';
+
+ d3.select('#explain-container').select('svg').remove();
+ const svg = d3.select('#explain-container').append('svg')
+ .attr('width', width)
+ .attr('height', height);
+
+ const container = svg.append('g');
+ const zoom =
+ d3.zoom()
+ .scaleExtent([1 / 10, 4])
+ .on('zoom', () => {
+ container.attr('transform', d3.event.transform);
+ });
+
+ svg
+ .call(zoom);
+
+ const onRequestDetail = data => this.sendAction('showStepDetail', data);
+
+ explain(JSON.parse(this.get('visualExplainInput')), svg, container, zoom, onRequestDetail);
+
+ },
+
+ actions:{
+ expandQueryResultPanel(){
+ this.sendAction('expandQueryResultPanel');
+ }
+ }
+
+});
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/controllers/udfs.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/controllers/udfs.js b/contrib/views/hive20/src/main/resources/ui/app/controllers/udfs.js
new file mode 100644
index 0000000..dc99fd1
--- /dev/null
+++ b/contrib/views/hive20/src/main/resources/ui/app/controllers/udfs.js
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+
+export default Ember.Controller.extend({
+});
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/routes/queries/query.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/routes/queries/query.js b/contrib/views/hive20/src/main/resources/ui/app/routes/queries/query.js
index 7d9a7c3..b904e4e 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/routes/queries/query.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/routes/queries/query.js
@@ -111,6 +111,9 @@ export default Ember.Route.extend({
controller.set('showQueryEditorLog', false);
controller.set('showQueryEditorResult', !controller.get('showQueryEditorLog'));
+ controller.set('isVisualExplainQuery', false);
+ controller.set('visualExplainJson', null);
+
},
@@ -159,11 +162,22 @@ export default Ember.Route.extend({
this.get('controller.model').set('selectedDb', db);
},
+
+ visualExplainQuery(){
+ this.get('controller').set('isVisualExplainQuery', true );
+ this.send('executeQuery');
+ },
+
executeQuery(isFirstCall){
let self = this;
this.get('controller').set('currentJobId', null);
- let queryInput = this.get('controller').get('currentQuery');
+
+ //let queryInput = this.get('controller').get('currentQuery');
+ let isVisualExplainQuery = this.get('controller').get('isVisualExplainQuery');
+ let queryInput = (isVisualExplainQuery) ? 'explain formatted ' + this.get('controller').get('currentQuery') : this.get('controller').get('currentQuery') ;
+
+
this.get('controller.model').set('query', queryInput);
let dbid = this.get('controller.model').get('selectedDb');
@@ -210,6 +224,10 @@ export default Ember.Route.extend({
self.get('controller').set('isJobSuccess', true);
self.send('getJob', data);
+ if(isVisualExplainQuery){
+ self.send('showVisualExplain');
+ }
+
//Last log
self.send('fetchLogs');
@@ -258,6 +276,19 @@ export default Ember.Route.extend({
});
},
+ showVisualExplain(){
+ let self = this;
+ let jobId = this.get('controller').get('currentJobId');
+ this.get('query').getVisualExplainJson(jobId).then(function(data) {
+ console.log('Successful getVisualExplainJson', data);
+
+ self.get('controller').set('visualExplainJson', data.rows[0][0]);
+
+ }, function(error){
+ console.log('error getVisualExplainJson', error);
+ });
+ },
+
getJob(data){
var self = this;
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/services/query.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/services/query.js b/contrib/views/hive20/src/main/resources/ui/app/services/query.js
index 1799f71..b484c74 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/services/query.js
+++ b/contrib/views/hive20/src/main/resources/ui/app/services/query.js
@@ -59,7 +59,6 @@ export default Ember.Service.extend({
},
retrieveQueryLog(logFile){
-
let self = this;
return new Promise( (resolve, reject) => {
this.get('store').adapterFor('query').retrieveQueryLog(logFile).then(function(data) {
@@ -68,8 +67,17 @@ export default Ember.Service.extend({
reject(err);
});
});
+ },
-
+ getVisualExplainJson(jobId){
+ let self = this;
+ return new Promise( (resolve, reject) => {
+ this.get('store').adapterFor('query').getVisualExplainJson(jobId).then(function(data) {
+ resolve(data);
+ }, function(err) {
+ reject(err);
+ });
+ });
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss
index 0b92d28..bce9f69 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss
+++ b/contrib/views/hive20/src/main/resources/ui/app/styles/app.scss
@@ -885,13 +885,82 @@ ul.dropdown-menu {
}
+.step {
+ font-family: Roboto;
+ background-color:rgb(255, 255, 255);
+ padding: 8px 10px;
+ border: 1px solid rgb(223, 223, 223);
+ cursor: pointer;
+ &:hover {
+ background-color: rgb(223, 240, 247);
+ }
+ body {
+ font-family: Roboto;
+ background-color: rgba(0,0,0,0);
+ }
+}
+.step-sink {
+ body {
+ color: rgb(255, 255, 255);
+ }
+ background-color: rgb(42, 179, 119);
+ padding: 20px;
+}
+.step-sink .step-body {
+ font-weight: lighter;
+ margin-top: 10px;
+}
+.step-source {
+ body {
+ color: rgb(255, 255, 255);
+ }
+ background-color: rgb(85, 100, 105);
+}
+.step-job,
+.step-vectorization {
+ border: none;
+ background: none;
+ padding: 0;
+}
+.step-job body,
+.step-vectorization body {
+ height: 100%;
+ width: 100%;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+}
+
+.step__pill {
+ font-size: 12px;
+ border-radius: 25px;
+ padding: 5px 10px;
+ min-width: 60px;
+ color: rgb(255, 255, 255);
+ background-color: rgb(85, 100, 105);
+ text-align: center;
+}
+.step-caption {
+ font-size: 10px;
+}
+.step-body {
+ font-size: 12px;
+}
+.edge {
+ stroke: rgb(83, 100, 106);
+ stroke-width: 2px;
+ fill: none;
+}
-
-
+#explain-container {
+ height: 100%;
+ width: 100%;
+ overflow: auto;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/templates/components/visual-explain.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/components/visual-explain.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/components/visual-explain.hbs
new file mode 100644
index 0000000..4238d43
--- /dev/null
+++ b/contrib/views/hive20/src/main/resources/ui/app/templates/components/visual-explain.hbs
@@ -0,0 +1,37 @@
+{{!
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+}}
+
+
+<div style="position: relative;">
+ <button class="btn btn-default" title="Expand/Collspse" {{action "expandQueryResultPanel" }} style="position: absolute;top: 10px; right: 10px;">{{fa-icon "expand"}}</button>
+</div>
+
+{{#if isQueryRunning}}
+ <div style="position:relative">
+ <div style="margin: auto;position: absolute;top: 0;left: 0;bottom: 0;right: 0;text-align: center">
+ {{fa-icon "spinner fa-2" spin=true}}
+ </div>
+ </div>
+{{/if}}
+
+
+{{#unless isQueryRunning}}
+ <div id="explain-container" ></div>
+{{/unless}}
+
+{{yield}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/templates/queries/query.hbs
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/templates/queries/query.hbs b/contrib/views/hive20/src/main/resources/ui/app/templates/queries/query.hbs
index e33e002..ce90883 100644
--- a/contrib/views/hive20/src/main/resources/ui/app/templates/queries/query.hbs
+++ b/contrib/views/hive20/src/main/resources/ui/app/templates/queries/query.hbs
@@ -30,8 +30,6 @@
<div class="row query-editor-controls">
<button class="btn btn-success" {{action "executeQuery" }}>{{fa-icon "check"}} Execute</button>
<button class="btn btn-default" {{action "openWorksheetModal" }}>{{fa-icon "save"}} Save As</button>
-
-
<div class="btn-group">
<button class="btn btn-default" type="button" data-toggle="dropdown">Insert UDF
<span class="caret"></span></button>
@@ -42,10 +40,8 @@
{{fileresource-item fileResource=fileResource createQuery='createQuery'}}
{{/each}}
</ul>
-
</div>
-
-
+ <button class="btn btn-default" {{action "visualExplainQuery" }}>{{fa-icon "link"}} Visual Explain</button>
{{#if worksheet.isQueryRunning}}
{{fa-icon "spinner fa-1-5" spin=true}}
{{/if}}
@@ -76,22 +72,34 @@
{{/if}}
{{#if showQueryEditorResult}}
<div class="clearfix row query-editor-results">
- {{query-result-table
- queryResult=queryResult
- jobId=currentJobId
- updateQuery='updateQuery'
- previousPage=worksheet.previousPage
- hidePreviousButton=hidePreviousButton
- goNextPage='goNextPage'
- goPrevPage='goPrevPage'
- expandQueryResultPanel='expandQueryResultPanel'
- saveToHDFS='saveToHDFS'
- downloadAsCsv='downloadAsCsv'
- isExportResultSuccessMessege=isExportResultSuccessMessege
- isExportResultFailureMessege=isExportResultFailureMessege
- showSaveHdfsModal=showSaveHdfsModal
- isQueryRunning=worksheet.isQueryRunning
- }}
+
+ {{#if isVisualExplainQuery}}
+ {{visual-explain
+ expandQueryResultPanel='expandQueryResultPanel'
+ isQueryRunning=worksheet.isQueryRunning
+ visualExplainJson=visualExplainJson
+ }}
+ {{else}}
+ {{query-result-table
+ queryResult=queryResult
+ jobId=currentJobId
+ updateQuery='updateQuery'
+ previousPage=worksheet.previousPage
+ hidePreviousButton=hidePreviousButton
+ goNextPage='goNextPage'
+ goPrevPage='goPrevPage'
+ expandQueryResultPanel='expandQueryResultPanel'
+ saveToHDFS='saveToHDFS'
+ downloadAsCsv='downloadAsCsv'
+ isExportResultSuccessMessege=isExportResultSuccessMessege
+ isExportResultFailureMessege=isExportResultFailureMessege
+ showSaveHdfsModal=showSaveHdfsModal
+ isQueryRunning=worksheet.isQueryRunning
+ }}
+ {{/if}}
+
+
+
</div>
{{/if}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js
new file mode 100644
index 0000000..2b59340
--- /dev/null
+++ b/contrib/views/hive20/src/main/resources/ui/app/utils/hive-explainer.js
@@ -0,0 +1,645 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+export default function render(data, svg, container, zoom, onRequestDetail){
+
+ const steps = createOrder(data).steps;
+ const plans = data['STAGE PLANS'];
+ const stageKey =
+ Object
+ .keys(plans)
+ .find(cStageKey => plans[cStageKey].hasOwnProperty('Fetch Operator'));
+ let rows = 'Unknown';
+ if(stageKey && plans[stageKey]['Fetch Operator']['limit:']) {
+ rows = plans[stageKey]['Fetch Operator']['limit:'];
+ }
+ const root = [{
+ "type": "sink",
+ "sink-type": "table",
+ "sink-label": "Limit",
+ "rows": rows,
+ "children": [{
+ steps: steps
+ }]
+ }];
+ const transformed = getTransformed(root);
+ update(transformed, svg, container, zoom, onRequestDetail);
+}
+
+const RENDER_GROUP = {
+ join: d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type, d['join-type'])}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>${d['join-type'] === 'merge' ? 'Merge' : 'Map'} Join</div>
+ <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div>
+ </div>
+ </div>
+ `,
+ vectorization: d => '<div class="step__pill">U</div>',
+ job: d => `<div class="step__pill">${d.label.toUpperCase()}</div>`,
+ broadcast: d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>Broadcast</div>
+ <!--div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div-->
+ </div>
+ </div>
+ `,
+ 'partition-sort': d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>Partition / Sort</div>
+ <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div>
+ </div>
+ </div>
+ `,
+ sink: d => `
+// TODO
+ `,
+ 'group-by': d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>Group By</div>
+ <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div>
+ </div>
+ </div>
+ `,
+ select: d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type)}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>Select</div>
+ <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(d.rows)}</div>
+ </div>
+ </div>
+ `,
+ source: d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getIcon(d.type, d['source-type'])}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>${d.label}</div>
+ <div><span style='font-weight: lighter;'>${d.isPartitioned ? 'Partitioned' : 'Unpartitioned'} | Rows:</span> ${abbreviate(d.rows)}</div>
+ </div>
+ </div>
+ `
+};
+
+function update(data, svg, container, zoom, onRequestDetail) {
+ const steps = container.selectAll('g.step')
+ .data(data)
+ .enter().append('g')
+ .attr('class', 'step');
+ steps
+ .append('foreignObject')
+ .attr('id', d => d.uuid)
+ .attr('class', 'step step-sink')
+ .attr('height', 300)
+ .attr('width', 220)
+ .append('xhtml:body')
+ .style('margin', 0)
+ .html(d => `
+ <div>
+ <div class='step-meta' style='display:flex;'>
+ <i class='fa ${getIcon(d.type, d['sink-type'])}' aria-hidden='true'></i>
+ <div class='step-header' style='margin-left: 10px;'>
+ <div class='step-title'>${d['sink-label']}</div>
+ <div class='step-caption'>${abbreviate(d.rows)} ${d.row === 1 ? 'row' : 'rows'}</div>
+ </div>
+ </div>
+ <div class='step-body'>${d['sink-description'] || ''}</div>
+ </div>
+ `)
+ .on('click', d => onRequestDetail(d));
+ steps
+ .call(recurse);
+ const edges =
+ container.selectAll('p.edge')
+ .data(getEdges(data))
+ .enter().insert('path', ':first-child')
+ .attr('class', 'edge')
+ .attr('d', d => getConnectionPath(d, svg, container));
+ reset(zoom, svg, container);
+
+
+ function recurse(step) {
+ const children =
+ step
+ .selectAll('g.child')
+ .data(d => d.children || []).enter()
+ .append('g')
+ .attr('class', 'child')
+ .style('transform', (d, index) => `translateY(${index * 100}px)`);
+ children.each(function(d) {
+ const child = d3.select(this);
+ const steps =
+ child.selectAll('g.step')
+ .data(d => d.steps || []).enter()
+ .append('g')
+ .attr('class', 'step')
+ .style('transform', (d, index) => `translateX(${250 + index * 150}px)`);
+ steps
+ .append('foreignObject')
+ .attr('id', d => d.uuid)
+ .attr('class', d => `step step-${d.type}`)
+ .classed('step-source', d => d.operator === 'TableScan')
+ .attr('height', 55)
+ .attr('width', d => d.type === 'source' ? 200 : 140)
+ .append('xhtml:body')
+ .style('margin', 0)
+ .html(d => getRenderer(d.type)(d))
+ .on('click', d => onRequestDetail(d));
+ steps.filter(d => Array.isArray(d.children))
+ .call(recurse);
+ });
+ }
+}
+
+function getRenderer(type) {
+ const renderer = RENDER_GROUP[type];
+ if(renderer) {
+ return renderer;
+ }
+
+ if(type === 'stage') {
+ return (d => `
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>Stage</div>
+ <!--div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div-->
+ </div>
+ </div>
+ `);
+ }
+
+ return (d => {
+ const isSource = d.operator === 'TableScan';
+ return (`
+ <div style='display:flex;'>
+ <div class='step-meta'>
+ <i class='fa ${getOperatorIcon(d.operator)}' aria-hidden='true'></i>
+ </div>
+ <div class='step-body' style='margin-left: 10px;'>
+ <div>${isSource ? d['alias:'] : getOperatorLabel(d.operator)}</div>
+ <div><span style='font-weight: lighter;'>Rows:</span> ${abbreviate(getNumberOfRows(d['Statistics:']))}</div>
+ </div>
+ </div>
+ `);
+ });
+}
+function getNumberOfRows(statistics) {
+ const match = statistics.match(/([^\?]*)\Num rows: (\d*)/);
+ return (match.length === 3 && Number.isNaN(Number(match[2])) === false) ? match[2] : 0;
+}
+function getOperatorLabel(operator) {
+ const operatorStr = operator.toString();
+ if(operatorStr.endsWith(' Operator')) {
+ return operatorStr.substring(0, operatorStr.length - ' Operator'.length);
+ }
+ if(operatorStr === 'TableScan') {
+ return 'Scan';
+ }
+ return operatorStr ? operatorStr : 'Unknown';
+}
+function getOperatorIcon(operator) {
+ switch(operator) {
+ case 'File Output Operator':
+ return 'fa-file-o';
+ case 'Reduce Output Operator':
+ return 'fa-compress';
+ case 'Filter Operator':
+ return 'fa-filter';
+ case 'Dynamic Partitioning Event Operator':
+ return 'fa-columns'
+ case 'Map Join Operator':
+ return 'fa-code-fork'
+ case 'Limit':
+ case 'Group By Operator':
+ case 'Select Operator':
+ case 'TableScan':
+ return 'fa-table';
+ default:
+ return '';
+ }
+}
+function getIcon (type, subtype) {
+ switch(type) {
+ case 'join':
+ return 'fa-code-fork'
+ case 'vectorization':
+ case 'job':
+ return;
+ case 'broadcast':
+ case 'partition-sort':
+ return 'fa-compress';
+ case 'source':
+ case 'sink':
+ case 'group-by':
+ case 'select':
+ return 'fa-table';
+ }
+};
+function abbreviate(value) {
+ let newValue = value;
+ if (value >= 1000) {
+ const suffixes = ["", "k", "m", "b","t"];
+ const suffixNum = Math.floor(("" + value).length / 3);
+ let shortValue = '';
+ for (var precision = 2; precision >= 1; precision--) {
+ shortValue = parseFloat( (suffixNum != 0 ? (value / Math.pow(1000,suffixNum) ) : value).toPrecision(precision));
+ const dotLessShortValue = (shortValue + '').replace(/[^a-zA-Z 0-9]+/g,'');
+ if (dotLessShortValue.length <= 2) { break; }
+ }
+ if (shortValue % 1 != 0) {
+ const shortNum = shortValue.toFixed(1);
+ }
+ newValue = shortValue+suffixes[suffixNum];
+ }
+ return newValue;
+}
+function reset(zoom, svg, container) {
+ const steps = container.selectAll('g.step');
+ const bounds = [];
+ steps.each(function(d) {
+ const cStep = d3.select(this);
+ const box = cStep.node().getBoundingClientRect();
+ bounds.push(box);
+ });
+ const PADDING_PERCENT = 0.95;
+ const fullWidth = svg.node().clientWidth;
+ const fullHeight = svg.node().clientHeight;
+ const offsetY = svg.node().getBoundingClientRect().top;
+ const top = Math.min(...bounds.map(cBound => cBound.top));
+ const left = Math.min(...bounds.map(cBound => cBound.left));
+ const width = Math.max(...bounds.map(cBound => cBound.right)) - left;
+ const height = Math.max(...bounds.map(cBound => cBound.bottom)) - top;
+ const midX = left + width / 2;
+ const midY = top + height / 2;
+ if (width == 0 || height == 0) return; // nothing to fit
+ const scale = PADDING_PERCENT / Math.max(width / fullWidth, height / fullHeight);
+ const translate = [fullWidth / 2 - scale * midX, fullHeight / 2 - scale * midY];
+ const zoomIdentity =
+ d3.zoomIdentity
+ .translate(translate[0], translate[1] + offsetY)
+ .scale(scale);
+ svg
+ .transition()
+ // .delay(750)
+ .duration(750)
+ // .call( zoom.transform, d3.zoomIdentity.translate(0, 0).scale(1) ); // not in d3 v4
+ .call(zoom.transform, zoomIdentity);
+}
+function getConnectionPath(edge, svg, container) {
+ const steps = container.selectAll('foreignObject.step');
+ const source = container.select(`#${edge.source}`);
+ const target = container.select(`#${edge.target}`);
+ const rSource = source.node().getBoundingClientRect();
+ const rTarget = target.node().getBoundingClientRect();
+ const pSource = {
+ x: (rSource.left + rSource.right) / 2,
+ y: (rSource.top + rSource.bottom) / 2,
+ };
+ const pTarget = {
+ x: (rTarget.left + rTarget.right) / 2,
+ y: (rTarget.top + rTarget.bottom) / 2,
+ };
+ const path = [
+ pSource
+ ];
+ if(pSource.y !== pTarget.y) {
+ path.push({
+ x: (pSource.x + pTarget.x) / 2,
+ y: pSource.y
+ }, {
+ x: (pSource.x + pTarget.x) / 2,
+ y: pTarget.y
+ })
+ }
+ path.push(pTarget);
+ const offsetY = svg.node().getBoundingClientRect().top;
+ return path.reduce((accumulator, cPoint, index) => {
+ if(index === 0) {
+ return accumulator + `M ${cPoint.x}, ${cPoint.y - offsetY}\n`
+ } else {
+ return accumulator + `L ${cPoint.x}, ${cPoint.y - offsetY}\n`
+ }
+ }, '');
+}
+function uuid() {
+ return 'step-xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {
+ const r = Math.random()*16|0, v = c == 'x' ? r : (r&0x3|0x8);
+ return v.toString(16);
+ });
+}
+function getEdges(steps) {
+ const edges = [];
+ for (let prev, index = 0; index < steps.length; index++) {
+ const cStep = steps[index];
+ if(prev) {
+ edges.push({
+ source: prev.uuid,
+ target: cStep.uuid
+ });
+ }
+ prev = cStep;
+ if(Array.isArray(cStep.children)) {
+ cStep.children.forEach(cChild => {
+ if(cChild.steps.length === 0) {
+ return;
+ }
+ edges.push({
+ source: cStep.uuid,
+ target: cChild.steps[0].uuid
+ });
+ edges.push(...getEdges(cChild.steps));
+ });
+ }
+ }
+ return edges;
+}
+function getTransformed(steps) {
+ return steps.map(cStep => {
+ let cResStep = cStep;
+ cResStep = Object.assign({}, cResStep, {
+ uuid: uuid()
+ });
+ if(Array.isArray(cResStep.children)) {
+ const children = cResStep.children.map(cChild => Object.assign({}, cChild, {
+ steps: getTransformed(cChild.steps)
+ }));
+ cResStep = Object.assign({}, cResStep, {
+ children: children
+ });
+ }
+ return cResStep;
+ });
+}
+function createOrder(data) {
+ const stageDeps = data['STAGE DEPENDENCIES'];
+ const stagePlans = data['STAGE PLANS'];
+ const stageRootKey = Object.keys(stageDeps).find(cStageKey => stageDeps[cStageKey]['ROOT STAGE'] === 'TRUE');
+ const root = Object.assign({}, getStageData(stageRootKey, stagePlans), {
+ _stages: getDependentStageTreeInOrder(stageRootKey, stageDeps, stagePlans)
+ });
+ const expanded = doExpandChild(root);
+ return doClean(expanded);
+}
+function getDependentStageTreeInOrder(sourceStageKey, stageDeps, stagePlans) {
+ const stageKeys =
+ Object
+ .keys(stageDeps)
+ .filter(cStageKey => stageDeps[cStageKey] && stageDeps[cStageKey]['DEPENDENT STAGES'] === sourceStageKey);
+ const stages =
+ stageKeys.map(cStageKey => Object.assign({}, getStageData(cStageKey, stagePlans), {
+ _stages: getDependentStageTreeInOrder(cStageKey, stageDeps, stagePlans)
+ }));
+ return stages;
+}
+function getStageData(stageKey, stagePlans) {
+ const plan = stagePlans[stageKey];
+ const engineKeys = Object.keys(plan);
+ if(engineKeys.length !== 1) {
+ return plan;
+ }
+ const engineKey = engineKeys[0];
+ // returns a job
+ let step;
+ switch(engineKey) {
+ case 'Map Reduce':
+ step = buildForMR(plan[engineKey]);
+ break;
+ case 'Map Reduce Local Work':
+ step = buildForMRLocal(plan[engineKey]);
+ break;
+ case 'Tez':
+ step = buildForTez(plan[engineKey]);
+ break;
+ case 'Fetch Operator':
+ step = buildForFetch(plan[engineKey]);
+ break;
+ default:
+ step = {
+ type: 'placeholder',
+ _engine: 'not_found',
+ _plan: plan
+ };
+ }
+ return ({
+ steps: [
+ step
+ ]
+ });
+}
+function buildForMR(plan) {
+ return ({
+ type: 'stage',
+ _engine: 'mr',
+ _plan: plan
+ });
+}
+function buildForMRLocal(plan) {
+ return ({
+ type: 'stage',
+ _engine: 'mr-local',
+ _plan: plan
+ });
+}
+function buildForTez(plan) {
+ const edges = plan['Edges:'];
+ const vertices = plan['Vertices:'];
+ const fEdges =
+ Object
+ .keys(edges)
+ .reduce((accumulator, cTargetKey) => {
+ if(Array.isArray(edges[cTargetKey])) {
+ const edgesFromSourceKey = edges[cTargetKey];
+ accumulator.push(...edgesFromSourceKey.map(cEdgeFromSourceKey => ({
+ source: cEdgeFromSourceKey['parent'],
+ target: cTargetKey,
+ type: cEdgeFromSourceKey['type']
+ })));
+ } else {
+ const edgeFromSourceKey = edges[cTargetKey];
+ accumulator.push({
+ source: edgeFromSourceKey['parent'],
+ target: cTargetKey,
+ type: edgeFromSourceKey['type']
+ });
+ }
+ return accumulator;
+ }, []);
+ const rootKey = fEdges.find(cEdge => fEdges.some(iEdge => iEdge.source === cEdge.target) === false).target;
+ return Object.assign({}, doTezBuildTreeFromEdges(rootKey, fEdges, vertices), {
+ _engine: 'tez',
+ _plan: plan
+ });
+}
+function buildForFetch(plan) {
+ return ({
+ type: 'stage',
+ _engine: 'fetch',
+ _plan: plan
+ });
+}
+function doTezBuildTreeFromEdges(parentKey, edges, vertices) {
+ const jobs =
+ Object
+ .keys(vertices)
+ .map(cVertexKey => ({
+ type: 'job',
+ label: cVertexKey,
+ _data: vertices[cVertexKey],
+ }))
+ .reduce((accumulator, cVertex) => Object.assign(accumulator, {
+ [cVertex.label]: cVertex
+ }), {});
+ edges.forEach(cEdge => {
+ const job = jobs[cEdge.target];
+ if(!Array.isArray(job.children)) {
+ job.children = [];
+ }
+ const steps = [];
+ if(cEdge.type === 'BROADCAST_EDGE') {
+ steps.push({
+ type: 'broadcast',
+ _data: jobs[cEdge.target],
+ });
+ }
+ steps.push(jobs[cEdge.source]);
+ job.children.push({
+ steps
+ });
+ });
+ return jobs[parentKey];
+}
+function doExpandChild(node) {
+ return Object.assign({}, node, {
+ steps: node.steps.reduce((accumulator, cStep) => [...accumulator, ...doExpandStep(cStep, 'step')], [])
+ });
+}
+function doExpandStep(node) {
+ switch(node.type) {
+ case 'job':
+ const key = Object.keys(doOmit(node._data, ['Execution mode:']))[0];
+ let root = node._data[key];
+ if(!Array.isArray(root)) {
+ root = [root];
+ }
+ const steps = doGetOperators(root);
+ const children = Array.isArray(node.children) ? node.children.map(cChild => doExpandChild(cChild)) : [];
+ return ([
+ doOmit(node, ['children']),
+ ...steps.reverse().slice(0, steps.length - 1),
+ Object.assign({}, steps[steps.length - 1], {
+ children
+ })
+ ]);
+ default:
+ return [node];
+ }
+}
+function doClean(node) {
+ let cleaned =
+ Object
+ .keys(node)
+ .filter(cNodeKey => cNodeKey.startsWith('_') === false)
+ .reduce((accumulator, cNodeKey) => Object.assign(accumulator, {
+ [cNodeKey]: node[cNodeKey]
+ }), {});
+ if(cleaned.hasOwnProperty('children')) {
+ cleaned = Object.assign({}, cleaned, {
+ children: cleaned.children.map(cChild => doClean(cChild))
+ })
+ }
+ if(cleaned.hasOwnProperty('steps')) {
+ cleaned = Object.assign({}, cleaned, {
+ steps: cleaned.steps.map(cStep => doClean(cStep))
+ })
+ }
+ return cleaned;
+}
+function doGetOperators(node) {
+ let stepx = node;
+ if(!Array.isArray(stepx)) {
+ stepx = [stepx];
+ }
+ const steps =
+ stepx
+ .reduce((accumulator, cStep) => {
+ const key = Object.keys(cStep)[0];
+ const obj = cStep[key];
+ let children = [];
+ if(obj.children) {
+ children = doGetOperators(obj.children);
+ }
+ const filtered =
+ Object
+ .keys(obj)
+ .filter(cKey => cKey !== 'children')
+ .reduce((accumulator, cKey) => {
+ accumulator[cKey] = obj[cKey];
+ return accumulator;
+ }, {});
+ return [
+ ...accumulator,
+ Object.assign({
+ _data: cStep,
+ operator: key
+ }, filtered),
+ ...children
+ ];
+ }, []);
+ return steps;
+}
+function doGetStep(node) {
+ const key = Object.keys(node)[0];
+ const obj = node[key];
+ return {
+ operator: key,
+ _data: obj
+ };
+}
+function doOmit(object, keys) {
+ return Object
+ .keys(object)
+ .filter(cKey => keys.indexOf(cKey) === -1)
+ .reduce((accumulator, cKey) => {
+ accumulator[cKey] = object[cKey];
+ return accumulator;
+ }, {});
+}
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/bower.json
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/bower.json b/contrib/views/hive20/src/main/resources/ui/bower.json
index a4ce788..f4d9aa0 100644
--- a/contrib/views/hive20/src/main/resources/ui/bower.json
+++ b/contrib/views/hive20/src/main/resources/ui/bower.json
@@ -1,6 +1,7 @@
{
"name": "ui",
"dependencies": {
+ "d3": "~4.5.0",
"ember": "~2.7.0",
"ember-cli-shims": "~0.1.1",
"ember-qunit-notifications": "0.1.0",
http://git-wip-us.apache.org/repos/asf/ambari/blob/df50b6c0/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js
----------------------------------------------------------------------
diff --git a/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js b/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js
index e41c8e8..10e0402 100644
--- a/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js
+++ b/contrib/views/hive20/src/main/resources/ui/ember-cli-build.js
@@ -53,6 +53,7 @@ module.exports = function(defaults) {
app.import('bower_components/codemirror/lib/codemirror.js');
app.import('bower_components/codemirror/addon/hint/sql-hint.js');
app.import('bower_components/codemirror/addon/hint/show-hint.js');
+ app.import('bower_components/d3/d3.js');
app.import('bower_components/codemirror/lib/codemirror.css');
app.import('bower_components/codemirror/addon/hint/show-hint.css');