You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by sr...@apache.org on 2016/02/01 16:13:16 UTC
tez git commit: TEZ-3058. Tez UI 2: Add download data functionality
(sree)
Repository: tez
Updated Branches:
refs/heads/TEZ-2980 480ea0785 -> 1c731ab7a
TEZ-3058. Tez UI 2: Add download data functionality (sree)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/1c731ab7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/1c731ab7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/1c731ab7
Branch: refs/heads/TEZ-2980
Commit: 1c731ab7acf8c22c4d965c472cf8b36c485d7914
Parents: 480ea07
Author: Sreenath Somarajapuram <sr...@apache.org>
Authored: Mon Feb 1 20:42:24 2016 +0530
Committer: Sreenath Somarajapuram <sr...@apache.org>
Committed: Mon Feb 1 20:42:24 2016 +0530
----------------------------------------------------------------------
TEZ-2980-CHANGES.txt | 1 +
.../webapp/app/components/zip-download-modal.js | 43 ++
.../src/main/webapp/app/initializers/hosts.js | 1 +
.../src/main/webapp/app/routes/application.js | 3 +
.../src/main/webapp/app/routes/dag/graphical.js | 10 +-
tez-ui2/src/main/webapp/app/routes/dag/index.js | 22 +
tez-ui2/src/main/webapp/app/styles/app.less | 5 +-
.../webapp/app/styles/zip-download-modal.less | 30 ++
.../templates/components/zip-download-modal.hbs | 36 ++
.../src/main/webapp/app/templates/dag/index.hbs | 2 +-
.../main/webapp/app/utils/download-dag-zip.js | 407 +++++++++++++++++++
tez-ui2/src/main/webapp/bower.json | 5 +-
tez-ui2/src/main/webapp/config/environment.js | 1 +
tez-ui2/src/main/webapp/ember-cli-build.js | 13 +-
tez-ui2/src/main/webapp/package.json | 3 +
.../components/zip-download-modal-test.js | 46 +++
.../tests/unit/routes/application-test.js | 6 +
.../tests/unit/utils/download-dag-zip-test.js | 26 ++
18 files changed, 651 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/TEZ-2980-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-2980-CHANGES.txt b/TEZ-2980-CHANGES.txt
index 653899c..e4ab014 100644
--- a/TEZ-2980-CHANGES.txt
+++ b/TEZ-2980-CHANGES.txt
@@ -30,3 +30,4 @@ ALL CHANGES:
TEZ-3061. Tez UI 2: Display in-progress vertex table in DAG details
TEZ-3069. Tez UI 2: Make error bar fully functional
TEZ-3062. Tez UI 2: Integrate graphical view
+ TEZ-3058. Tez UI 2: Add download data functionality
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/components/zip-download-modal.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/components/zip-download-modal.js b/tez-ui2/src/main/webapp/app/components/zip-download-modal.js
new file mode 100644
index 0000000..c55b34e
--- /dev/null
+++ b/tez-ui2/src/main/webapp/app/components/zip-download-modal.js
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+
+export default Ember.Component.extend({
+ classNames: ['zip-download-modal'],
+ content: null,
+
+ _onSuccess: Ember.observer("content.downloader.succeeded", function () {
+ if(this.get("content.downloader.succeeded")) {
+ Ember.run.later(this, "close");
+ }
+ }),
+
+ close: function () {
+ Ember.$(".simple-modal").modal("hide");
+ },
+
+ actions: {
+ cancel: function () {
+ var downloader = this.get("content.downloader");
+ if(downloader) {
+ downloader.cancel();
+ }
+ }
+ }
+});
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/initializers/hosts.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/initializers/hosts.js b/tez-ui2/src/main/webapp/app/initializers/hosts.js
index 8791fc7..ec6f7a7 100644
--- a/tez-ui2/src/main/webapp/app/initializers/hosts.js
+++ b/tez-ui2/src/main/webapp/app/initializers/hosts.js
@@ -19,6 +19,7 @@
export function initialize(application) {
application.inject('controller', 'hosts', 'service:hosts');
application.inject('adapter', 'hosts', 'service:hosts');
+ application.inject('route', 'hosts', 'service:hosts');
}
export default {
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/routes/application.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/routes/application.js b/tez-ui2/src/main/webapp/app/routes/application.js
index 41ea3fb..99bed25 100644
--- a/tez-ui2/src/main/webapp/app/routes/application.js
+++ b/tez-ui2/src/main/webapp/app/routes/application.js
@@ -59,6 +59,9 @@ export default Ember.Route.extend({
Ember.$(".simple-modal").modal();
});
},
+ closeModal: function () {
+ Ember.$(".simple-modal").modal("hide");
+ },
destroyModal: function () {
Ember.run.later(this, function () {
this.disconnectOutlet({
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/routes/dag/graphical.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/routes/dag/graphical.js b/tez-ui2/src/main/webapp/app/routes/dag/graphical.js
index 3d9550f..69d2de4 100644
--- a/tez-ui2/src/main/webapp/app/routes/dag/graphical.js
+++ b/tez-ui2/src/main/webapp/app/routes/dag/graphical.js
@@ -39,11 +39,13 @@ export default MultiAmPollsterRoute.extend({
var loadedValue = this.get("loadedValue"),
records = [];
- loadedValue.forEach(function (record) {
- records.push(record);
- });
+ if(loadedValue) {
+ loadedValue.forEach(function (record) {
+ records.push(record);
+ });
- this.set("polledRecords", records);
+ this.set("polledRecords", records);
+ }
Ember.run.later(this, "setViewHeight", 100);
}),
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/routes/dag/index.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/routes/dag/index.js b/tez-ui2/src/main/webapp/app/routes/dag/index.js
index be60c1d..acabc58 100644
--- a/tez-ui2/src/main/webapp/app/routes/dag/index.js
+++ b/tez-ui2/src/main/webapp/app/routes/dag/index.js
@@ -19,6 +19,8 @@
import Ember from 'ember';
import SingleAmPollsterRoute from '../single-am-pollster';
+import downloadDAGZip from '../../utils/download-dag-zip';
+
export default SingleAmPollsterRoute.extend({
title: "DAG Details",
@@ -33,4 +35,24 @@ export default SingleAmPollsterRoute.extend({
return this.get("loader").queryRecord('dag', this.modelFor("dag").get("id"), options);
},
+ actions: {
+ downloadDagJson: function () {
+ var dag = this.get("loadedValue"),
+ downloader = downloadDAGZip(dag, {
+ batchSize: 500,
+ timelineHost: this.get("hosts.timeline"),
+ timelineNamespace: this.get("env.app.namespaces.webService.timeline")
+ }),
+ modalContent = Ember.Object.create({
+ dag: dag,
+ downloader: downloader
+ });
+
+ this.send("openModal", "zip-download-modal", {
+ title: "Download data",
+ content: modalContent
+ });
+ }
+ }
+
});
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/styles/app.less
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/styles/app.less b/tez-ui2/src/main/webapp/app/styles/app.less
index cdeccd1..11c611c 100644
--- a/tez-ui2/src/main/webapp/app/styles/app.less
+++ b/tez-ui2/src/main/webapp/app/styles/app.less
@@ -26,9 +26,12 @@
@import "tab-n-refresh";
@import "dags-page-search";
@import "table-controls";
-@import "column-selector";
@import "error-bar";
+// Modals
+@import "column-selector";
+@import "zip-download-modal";
+
// Pages
@import "page-layout";
@import "details-page";
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/styles/zip-download-modal.less
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/styles/zip-download-modal.less b/tez-ui2/src/main/webapp/app/styles/zip-download-modal.less
new file mode 100644
index 0000000..4ed0fd2
--- /dev/null
+++ b/tez-ui2/src/main/webapp/app/styles/zip-download-modal.less
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.zip-download-modal {
+ .message {
+ padding: 10px 15px;
+
+ .fa-spinner {
+ color: green;
+ }
+ .fa-exclamation-circle {
+ color: red;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/templates/components/zip-download-modal.hbs
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/templates/components/zip-download-modal.hbs b/tez-ui2/src/main/webapp/app/templates/components/zip-download-modal.hbs
new file mode 100644
index 0000000..03b820e
--- /dev/null
+++ b/tez-ui2/src/main/webapp/app/templates/components/zip-download-modal.hbs
@@ -0,0 +1,36 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="message">
+ {{#if content.downloader.failed}}
+ <i class="fa fa-lg fa-exclamation-circle"></i>
+ Error downloading data!
+ {{else}}
+ <i class="fa fa-lg fa-spinner fa-spin"></i>
+ Downloading data for dag: <b>{{content.dag.entityID}}</b>
+ {{/if}}
+</div>
+
+
+<div class="form-actions">
+ {{#if content.downloader.failed}}
+ <button type="button" class="btn btn-primary" data-dismiss="modal" aria-label="Close">Ok</button>
+ {{else}}
+ <button type="button" class="btn" data-dismiss="modal" aria-label="Close" {{action "cancel"}}>Cancel</button>
+ {{/if}}
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/templates/dag/index.hbs
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/templates/dag/index.hbs b/tez-ui2/src/main/webapp/app/templates/dag/index.hbs
index d7e23fc..59bb6a9 100644
--- a/tez-ui2/src/main/webapp/app/templates/dag/index.hbs
+++ b/tez-ui2/src/main/webapp/app/templates/dag/index.hbs
@@ -26,7 +26,7 @@
<tbody>
<tr>
<td colspan="2">
- {{bs-button icon="fa fa-download" title="Download data" defaultText="Download data" type="info" clicked="downloadDagJson"}}
+ {{bs-button icon="fa fa-download" title="Download data" defaultText="Download data" type="info" action="downloadDagJson"}}
</td>
</tr>
<tr>
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/app/utils/download-dag-zip.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/app/utils/download-dag-zip.js b/tez-ui2/src/main/webapp/app/utils/download-dag-zip.js
new file mode 100644
index 0000000..88ea9a4
--- /dev/null
+++ b/tez-ui2/src/main/webapp/app/utils/download-dag-zip.js
@@ -0,0 +1,407 @@
+/*global zip, saveAs*/
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import Ember from 'ember';
+import DS from 'ember-data';
+
+zip.workerScriptsPath = "/assets/zip/";
+
+var IO = {
+ /* Allow queuing of downloads and then get a callback once all the downloads are done.
+ * sample usage.
+ * var downloader = IO.fileDownloader();
+ * downloader.queueItem({
+ * url: 'http://....',
+ * onItemFetched: function(data, context) {...},
+ * context: {}, // context object gets passed back to the callback
+ * });
+ * downloader.queueItem({...}); //queue in other items
+ * downloader.finish(); // once all items are queued. items can be queued from
+ * // callbacks too. in that case the finish should be called
+ * // once all items are queued.
+ * downloader.then(successCallback).catch(failurecallback).finally(callback)
+ */
+ fileDownloader: function(options) {
+ var itemList = [],
+ opts = options || {},
+ numParallel = opts.numParallel || 5,
+ hasMoreInputs = true,
+ inProgress = 0,
+ hasFailed = false,
+ pendingRequests = {},
+ pendingRequestID = 0,
+ failureReason = 'Unknown',
+ deferredPromise = Ember.RSVP.defer();
+
+ function checkForCompletion() {
+ if (hasFailed) {
+ if (inProgress === 0) {
+ deferredPromise.reject("Unknown Error");
+ }
+ return;
+ }
+
+ if (hasMoreInputs || itemList.length > 0 || inProgress > 0) {
+ return;
+ }
+
+ deferredPromise.resolve();
+ }
+
+ function getRequestId() {
+ return "req_" + pendingRequestID++;
+ }
+
+ function abortPendingRequests() {
+ Ember.$.each(pendingRequests, function(idx, val) {
+ try {
+ val.abort("abort");
+ } catch(e) {}
+ });
+ }
+
+ function markFailed(reason) {
+ if (!hasFailed) {
+ hasFailed = true;
+ failureReason = reason;
+ abortPendingRequests();
+ }
+ }
+
+ function processNext() {
+ if (inProgress >= numParallel) {
+ Ember.Logger.debug(`delaying download as ${inProgress} of ${numParallel} is in progress`);
+ return;
+ }
+
+ if (itemList.length < 1) {
+ Ember.Logger.debug("no items to download");
+ checkForCompletion();
+ return;
+ }
+
+ inProgress++;
+ Ember.Logger.debug(`starting download ${inProgress}`);
+ var item = itemList.shift();
+
+ var xhr = Ember.$.ajax({
+ crossOrigin: true,
+ url: item.url,
+ dataType: 'json',
+ xhrFields: {
+ withCredentials: true
+ },
+ });
+ var reqID = getRequestId();
+ pendingRequests[reqID] = xhr;
+
+ xhr.done(function(data/*, statusText, xhr*/) {
+ delete pendingRequests[reqID];
+
+ if (Ember.$.isFunction(item.onItemFetched)) {
+ try {
+ item.onItemFetched(data, item.context);
+ } catch (e) {
+ markFailed(e || 'failed to process data');
+ inProgress--;
+ checkForCompletion();
+ return;
+ }
+ }
+
+ inProgress--;
+ processNext();
+ }).fail(function(xhr, statusText/*, errorObject*/) {
+ delete pendingRequests[reqID];
+ markFailed(statusText);
+ inProgress--;
+ checkForCompletion();
+ });
+ }
+
+ return DS.PromiseObject.create({
+ promise: deferredPromise.promise,
+
+ queueItems: function(options) {
+ options.forEach(this.queueItem);
+ },
+
+ queueItem: function(option) {
+ itemList.push(option);
+ processNext();
+ },
+
+ finish: function() {
+ hasMoreInputs = false;
+ checkForCompletion();
+ },
+
+ cancel: function() {
+ markFailed("User cancelled");
+ checkForCompletion();
+ }
+ });
+ },
+
+
+ /*
+ * allows to zip files and download that.
+ * usage:
+ * zipHelper = IO.zipHelper({
+ * onProgress: function(filename, current, total) { ...},
+ * onAdd: function(filename) {...}
+ * });
+ * zipHelper.addFile({name: filenameinsidezip, data: data);
+ * // add all files
+ * once all files are added call the close
+ * zipHelper.close(); // or .abort to abort zip
+ * zipHelper.then(function(zippedBlob) {
+ * saveAs(filename, zippedBlob);
+ * }).catch(failureCallback);
+ */
+ zipHelper: function(options) {
+ var opts = options || {},
+ zipWriter,
+ completion = Ember.RSVP.defer(),
+ fileList = [],
+ completed = 0,
+ currentIdx = -1,
+ numFiles = 0,
+ hasMoreInputs = true,
+ inProgress = false,
+ hasFailed = false;
+
+ zip.createWriter(new zip.BlobWriter("application/zip"), function(writer) {
+ zipWriter = writer;
+ checkForCompletion();
+ nextFile();
+ });
+
+ function checkForCompletion() {
+ if (hasFailed) {
+ if (zipWriter) {
+ Ember.Logger.debug("aborting zipping. closing file.");
+ zipWriter.close(completion.reject);
+ zipWriter = null;
+ }
+ } else {
+ if (!hasMoreInputs && numFiles === completed) {
+ Ember.Logger.debug("completed zipping. closing file.");
+ zipWriter.close(completion.resolve);
+ }
+ }
+ }
+
+ function onProgress(current, total) {
+ if (Ember.$.isFunction(opts.onProgress)) {
+ opts.onProgress(fileList[currentIdx].name, current, total);
+ }
+ }
+
+ function onAdd(filename) {
+ if (Ember.$.isFunction(opts.onAdd)) {
+ opts.onAdd(filename);
+ }
+ }
+
+ function nextFile() {
+ if (hasFailed || completed === numFiles || inProgress) {
+ return;
+ }
+
+ currentIdx++;
+ var file = fileList[currentIdx];
+ inProgress = true;
+ onAdd(file.name);
+ zipWriter.add(file.name, new zip.TextReader(file.data), function() {
+ completed++;
+ inProgress = false;
+ if (currentIdx < numFiles - 1) {
+ nextFile();
+ }
+ checkForCompletion();
+ }, onProgress);
+ }
+
+ return DS.PromiseObject.create({
+ addFiles: function(files) {
+ files.forEach(this.addFile);
+ },
+
+ addFile: function(file) {
+ if (hasFailed) {
+ Ember.Logger.debug(`Skipping add of file ${file.name} as zip has been aborted`);
+ return;
+ }
+ numFiles++;
+ fileList.push(file);
+ if (zipWriter) {
+ Ember.Logger.debug("adding file from addFile: " + file.name);
+ nextFile();
+ }
+ },
+
+ close: function() {
+ hasMoreInputs = false;
+ checkForCompletion();
+ },
+
+ promise: completion.promise,
+
+ abort: function() {
+ hasFailed = true;
+ this.close();
+ }
+ });
+ }
+};
+
+export default function downloadDagZip(dag, options) {
+ var opts = options || {},
+ batchSize = opts.batchSize || 1000,
+ dagID = dag.get("entityID"),
+ baseurl = `${options.timelineHost}/${options.timelineNamespace}`,
+ itemsToDownload = [
+ {
+ url: getUrl('TEZ_APPLICATION', 'tez_' + dag.get("appID")),
+ context: { name: 'application', type: 'TEZ_APPLICATION' },
+ onItemFetched: processSingleItem
+ },
+ {
+ url: getUrl('TEZ_DAG_ID', dagID),
+ context: { name: 'dag', type: 'TEZ_DAG_ID' },
+ onItemFetched: processSingleItem
+ },
+ {
+ url: getUrl('TEZ_VERTEX_ID', dagID),
+ context: { name: 'vertices', type: 'TEZ_VERTEX_ID', part: 0 },
+ onItemFetched: processMultipleItems
+ },
+ {
+ url: getUrl('TEZ_TASK_ID', dagID),
+ context: { name: 'tasks', type: 'TEZ_TASK_ID', part: 0 },
+ onItemFetched: processMultipleItems
+ },
+ {
+ url: getUrl('TEZ_TASK_ATTEMPT_ID', dagID),
+ context: { name: 'task_attempts', type: 'TEZ_TASK_ATTEMPT_ID', part: 0 },
+ onItemFetched: processMultipleItems
+ }
+ ],
+ totalItemsToDownload = itemsToDownload.length,
+ numItemTypesToDownload = totalItemsToDownload,
+ downloader = IO.fileDownloader(),
+ zipHelper = IO.zipHelper({
+ onProgress: function(filename, current, total) {
+ Ember.Logger.debug(`${filename}: ${current} of ${total}`);
+ },
+ onAdd: function(filename) {
+ Ember.Logger.debug(`adding ${filename} to Zip`);
+ }
+ }),
+ downloaderProxy = Ember.Object.create({
+ percent: 0,
+ succeeded: false,
+ failed: false,
+ cancel: function() {
+ downloader.cancel();
+ }
+ });
+
+ function getUrl(type, dagID, fromID) {
+ var url,
+ queryBatchSize = batchSize + 1;
+
+ if (type === 'TEZ_DAG_ID' || type === 'TEZ_APPLICATION') {
+ url = `${baseurl}/${type}/${dagID}`;
+ } else {
+ url = `${baseurl}/${type}?primaryFilter=TEZ_DAG_ID:${dagID}&limit=${queryBatchSize}`;
+ if (!!fromID) {
+ url = `${url}&fromId=${fromID}`;
+ }
+ }
+ return url;
+ }
+
+ function checkIfAllDownloaded() {
+ numItemTypesToDownload--;
+
+ var remainingItems = totalItemsToDownload - numItemTypesToDownload;
+ downloaderProxy.set("percent", remainingItems / totalItemsToDownload);
+
+ if (numItemTypesToDownload === 0) {
+ downloader.finish();
+ }
+ }
+
+ function processSingleItem(data, context) {
+ var obj = {};
+ obj[context.name] = data;
+
+ zipHelper.addFile({name: `${context.name}.json`, data: JSON.stringify(obj, null, 2)});
+ checkIfAllDownloaded();
+ }
+
+ function processMultipleItems(data, context) {
+ var obj = {};
+ var nextBatchStart;
+
+ if (!Ember.$.isArray(data.entities)) {
+ throw "invalid data";
+ }
+
+ // need to handle no more entries , zero entries
+ if (data.entities.length > batchSize) {
+ nextBatchStart = data.entities.pop().entity;
+ }
+ obj[context.name] = data.entities;
+
+ zipHelper.addFile({name: `${context.name}_part_${context.part}.json`, data: JSON.stringify(obj, null, 2)});
+
+ if (!!nextBatchStart) {
+ context.part++;
+ downloader.queueItem({
+ url: getUrl(context.type, dagID, nextBatchStart),
+ context: context,
+ onItemFetched: processMultipleItems
+ });
+ } else {
+ checkIfAllDownloaded();
+ }
+ }
+
+ downloader.queueItems(itemsToDownload);
+
+ downloader.then(function() {
+ Ember.Logger.info('Finished download');
+ zipHelper.close();
+ }).catch(function(e) {
+ Ember.Logger.error('Failed to download: ' + e);
+ zipHelper.abort();
+ });
+
+ zipHelper.then(function(zippedBlob) {
+ saveAs(zippedBlob, `${dagID}.zip`);
+ downloaderProxy.set("succeeded", true);
+ }, function() {
+ Ember.Logger.error('zip Failed');
+ downloaderProxy.set("failed", true);
+ });
+
+ return downloaderProxy;
+}
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/bower.json
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/bower.json b/tez-ui2/src/main/webapp/bower.json
index eb9569d..8d23c02 100644
--- a/tez-ui2/src/main/webapp/bower.json
+++ b/tez-ui2/src/main/webapp/bower.json
@@ -17,6 +17,9 @@
"moment": "^2.8.0",
"moment-timezone": "^0.5.0",
"numeral": "1.5.3",
- "snippet-ss": "~1.11.0"
+ "snippet-ss": "~1.11.0",
+ "jquery-mousewheel": "~3.1.13",
+ "FileSaver": "#230de7d",
+ "zip.js": "#1bead0a"
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/config/environment.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/config/environment.js b/tez-ui2/src/main/webapp/config/environment.js
index e5fc8ac..c08549f 100644
--- a/tez-ui2/src/main/webapp/config/environment.js
+++ b/tez-ui2/src/main/webapp/config/environment.js
@@ -37,6 +37,7 @@ module.exports = function(environment) {
contentSecurityPolicy: {
'connect-src': "* 'self'",
+ 'child-src': "'self' 'unsafe-inline'",
'style-src': "'self' 'unsafe-inline'",
'script-src': "'self' 'unsafe-inline'"
}
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/ember-cli-build.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/ember-cli-build.js b/tez-ui2/src/main/webapp/ember-cli-build.js
index d5cd31a..24c24fb 100644
--- a/tez-ui2/src/main/webapp/ember-cli-build.js
+++ b/tez-ui2/src/main/webapp/ember-cli-build.js
@@ -21,17 +21,23 @@
var Funnel = require("broccoli-funnel");
var EmberApp = require('ember-cli/lib/broccoli/ember-app');
+var MergeTrees = require('broccoli-merge-trees');
module.exports = function(defaults) {
var app = new EmberApp(defaults, {
storeConfigInMeta: false
});
- var extraAssets = new Funnel('config', {
+ var configEnv = new Funnel('config', {
srcDir: '/',
include: ['*.env'],
destDir: '/config'
});
+ var zipWorker = new Funnel('bower_components/zip.js', {
+ srcDir: '/WebContent',
+ include: ['z-worker.js', 'deflate.js', 'inflate.js'],
+ destDir: '/assets/zip'
+ });
app.import("bower_components/snippet-ss/less/force.less");
app.import("bower_components/snippet-ss/less/effects.less");
@@ -43,5 +49,8 @@ module.exports = function(defaults) {
app.import('bower_components/more-js/dist/more.js');
- return app.toTree(extraAssets);
+ app.import('bower_components/FileSaver/FileSaver.js');
+ app.import('bower_components/zip.js/WebContent/zip.js');
+
+ return app.toTree(new MergeTrees([configEnv, zipWorker]));
};
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/package.json
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/package.json b/tez-ui2/src/main/webapp/package.json
index e30d5bc..8fde220 100644
--- a/tez-ui2/src/main/webapp/package.json
+++ b/tez-ui2/src/main/webapp/package.json
@@ -23,6 +23,7 @@
"devDependencies": {
"bower": "^1.7.1",
"broccoli-asset-rev": "^2.2.0",
+ "broccoli-merge-trees": "^1.1.1",
"em-tgraph": "0.0.3",
"ember-bootstrap": "0.5.1",
"ember-cli": "1.13.13",
@@ -30,12 +31,14 @@
"ember-cli-auto-register": "^1.1.0",
"ember-cli-babel": "^5.1.5",
"ember-cli-content-security-policy": "0.4.0",
+ "ember-cli-d3": "1.1.2",
"ember-cli-dependency-checker": "^1.1.0",
"ember-cli-font-awesome": "1.4.0",
"ember-cli-htmlbars-inline-precompile": "^0.3.1",
"ember-cli-inject-live-reload": "^1.3.1",
"ember-cli-jquery-ui": "0.0.20",
"ember-cli-moment-shim": "0.7.3",
+ "ember-cli-mousewheel": "0.1.5",
"ember-cli-numeral": "0.1.2",
"ember-cli-qunit": "^1.0.4",
"ember-cli-release": "0.2.8",
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/tests/integration/components/zip-download-modal-test.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/tests/integration/components/zip-download-modal-test.js b/tez-ui2/src/main/webapp/tests/integration/components/zip-download-modal-test.js
new file mode 100644
index 0000000..cd9a61a
--- /dev/null
+++ b/tez-ui2/src/main/webapp/tests/integration/components/zip-download-modal-test.js
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { moduleForComponent, test } from 'ember-qunit';
+import hbs from 'htmlbars-inline-precompile';
+
+moduleForComponent('zip-download-modal', 'Integration | Component | zip download modal', {
+ integration: true
+});
+
+test('Basic creation test', function(assert) {
+ var testID = "dag_a",
+ expectedMessage = "Downloading data for dag: " + testID;
+
+ this.set("content", {
+ dag: {
+ entityID: testID
+ }
+ });
+
+ this.render(hbs`{{zip-download-modal content=content}}`);
+ assert.equal(this.$(".message").text().trim(), expectedMessage);
+
+ // Template block usage:" + EOL +
+ this.render(hbs`
+ {{#zip-download-modal content=content}}
+ template block text
+ {{/zip-download-modal}}
+ `);
+ assert.equal(this.$(".message").text().trim(), expectedMessage);
+});
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/tests/unit/routes/application-test.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/tests/unit/routes/application-test.js b/tez-ui2/src/main/webapp/tests/unit/routes/application-test.js
index 695b72f..e441e49 100644
--- a/tez-ui2/src/main/webapp/tests/unit/routes/application-test.js
+++ b/tez-ui2/src/main/webapp/tests/unit/routes/application-test.js
@@ -30,6 +30,12 @@ test('Basic creation test', function(assert) {
assert.ok(route.pageReset);
assert.ok(route.actions.didTransition);
assert.ok(route.actions.bubbleBreadcrumbs);
+
+ assert.ok(route.actions.error);
+
+ assert.ok(route.actions.openModal);
+ assert.ok(route.actions.closeModal);
+ assert.ok(route.actions.destroyModal);
});
test('Test didTransition action', function(assert) {
http://git-wip-us.apache.org/repos/asf/tez/blob/1c731ab7/tez-ui2/src/main/webapp/tests/unit/utils/download-dag-zip-test.js
----------------------------------------------------------------------
diff --git a/tez-ui2/src/main/webapp/tests/unit/utils/download-dag-zip-test.js b/tez-ui2/src/main/webapp/tests/unit/utils/download-dag-zip-test.js
new file mode 100644
index 0000000..0fe8c5f
--- /dev/null
+++ b/tez-ui2/src/main/webapp/tests/unit/utils/download-dag-zip-test.js
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import downloadDagZip from '../../../utils/download-dag-zip';
+import { module, test } from 'qunit';
+
+module('Unit | Utility | download dag zip');
+
+test('Basic creation test', function(assert) {
+ assert.ok(downloadDagZip);
+});