You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/08/28 16:11:36 UTC
[airavata-django-portal] branch master updated: AIRAVATA-3081
Integrated Uppy component for tus uploads with fallback to XHR uploads
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
The following commit(s) were added to refs/heads/master by this push:
new 5cac753 AIRAVATA-3081 Integrated Uppy component for tus uploads with fallback to XHR uploads
5cac753 is described below
commit 5cac753ce7719f2bbd6ce35263800771260c33cc
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Aug 14 13:01:20 2019 -0400
AIRAVATA-3081 Integrated Uppy component for tus uploads with fallback to XHR uploads
---
django_airavata/apps/api/data_products_helper.py | 4 +-
django_airavata/apps/api/serializers.py | 1 +
.../django_airavata_api/js/models/Settings.js | 2 +-
django_airavata/apps/api/urls.py | 2 +
django_airavata/apps/api/views.py | 27 ++++++-
django_airavata/apps/workspace/package.json | 5 ++
.../experiment/input-editors/FileInputEditor.vue | 36 ++++-----
.../components/experiment/input-editors/Uppy.vue | 93 ++++++++++++++++++++++
django_airavata/settings.py | 7 ++
9 files changed, 151 insertions(+), 26 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index e93131e..3a07699 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -27,10 +27,10 @@ def save(request, path, file):
return data_product
-def save_input_file_upload(request, file):
+def save_input_file_upload(request, file, name=None):
"""Save input file in staging area for input file uploads."""
username = request.user.username
- file_name = os.path.basename(file.name)
+ file_name = name if name is not None else os.path.basename(file.name)
full_path = datastore.save(username, TMP_INPUT_FILE_UPLOAD_DIR, file)
data_product = _save_data_product(request, full_path, name=file_name)
return data_product
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index d5adde2..84da946 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -975,3 +975,4 @@ class LogRecordSerializer(serializers.Serializer):
class SettingsSerializer(serializers.Serializer):
fileUploadMaxFileSize = serializers.IntegerField()
+ tusEndpoint = serializers.CharField()
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js b/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js
index 1a5c3f3..3d80d88 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/Settings.js
@@ -1,6 +1,6 @@
import BaseModel from "./BaseModel";
-const FIELDS = ["fileUploadMaxFileSize"];
+const FIELDS = ["fileUploadMaxFileSize", "tusEndpoint"];
export default class Settings extends BaseModel {
constructor(data = {}) {
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index 9548cca..70db566 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -52,6 +52,8 @@ app_name = 'django_airavata_api'
urlpatterns = [
url(r'^', include(router.urls)),
url(r'^upload$', views.upload_input_file, name='upload_input_file'),
+ url(r'^tus-upload-finish$', views.tus_upload_finish,
+ name='tus_upload_finish'),
url(r'^download', views.download_file, name='download_file'),
url(r'^delete-file$', views.delete_file, name='delete_file'),
url(r'^data-products', views.DataProductView.as_view(),
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index d0c8a07..866529a 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -2,6 +2,7 @@ import json
import logging
import os
from datetime import datetime, timedelta
+from urllib.parse import urlparse
from django.conf import settings
from django.contrib.auth.decorators import login_required
@@ -908,6 +909,29 @@ def upload_input_file(request):
@login_required
+def tus_upload_finish(request):
+ log.debug("POST={}".format(request.POST))
+ uploadURL = request.POST['uploadURL']
+ # file UUID is last path component in URL. For example:
+ # http://localhost:1080/files/2c44415fdb6259a22f425145b87d0840
+ upload_uuid = urlparse(uploadURL).path.split("/")[-1]
+ upload_bin_path = os.path.join(settings.TUS_DATA_DIR, f"{upload_uuid}.bin")
+ log.debug(f"upload_bin_path={upload_bin_path}")
+ upload_info_path = os.path.join(settings.TUS_DATA_DIR,
+ f"{upload_uuid}.info")
+ with open(upload_info_path) as upload_info_file, \
+ open(upload_bin_path, "rb") as upload_file:
+ upload_info = json.load(upload_info_file)
+ filename = upload_info['MetaData']['filename']
+ data_product = data_products_helper.save_input_file_upload(
+ request, upload_file, name=filename)
+ serializer = serializers.DataProductSerializer(
+ data_product, context={'request': request})
+ return JsonResponse({'uploaded': True,
+ 'data-product': serializer.data})
+
+
+@login_required
def download_file(request):
# TODO check that user has access to this file using sharing API
data_product_uri = request.GET.get('data-product-uri', '')
@@ -1708,7 +1732,8 @@ class SettingsAPIView(APIView):
def get(self, request, format=None):
data = {
- 'fileUploadMaxFileSize': settings.FILE_UPLOAD_MAX_FILE_SIZE
+ 'fileUploadMaxFileSize': settings.FILE_UPLOAD_MAX_FILE_SIZE,
+ 'tusEndpoint': settings.TUS_ENDPOINT,
}
serializer = self.serializer_class(
data, context={'request': request})
diff --git a/django_airavata/apps/workspace/package.json b/django_airavata/apps/workspace/package.json
index 4d4fc92..aadd26f 100644
--- a/django_airavata/apps/workspace/package.json
+++ b/django_airavata/apps/workspace/package.json
@@ -14,6 +14,11 @@
"test:unit:watch": "vue-cli-service test:unit --watch"
},
"dependencies": {
+ "@uppy/core": "^1.2.0",
+ "@uppy/file-input": "^1.2.0",
+ "@uppy/status-bar": "^1.2.0",
+ "@uppy/tus": "^1.3.0",
+ "@uppy/xhr-upload": "^1.2.0",
"bootstrap": "^4.3.1",
"bootstrap-vue": "2.0.0-rc.26",
"django-airavata-api": "file:../api",
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
index 3b47b34..dd386cf 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
@@ -62,13 +62,11 @@
:invalid-feedback="fileUploadInvalidFeedback"
class="input-file-option"
>
- <b-form-file
- :id="id"
- v-model="file"
- v-if="!isDataProductURI"
- placeholder="Upload file"
- @input="fileChanged"
- :state="fileUploadState"
+ <uppy
+ xhr-upload-endpoint="/api/upload"
+ tus-upload-finish-endpoint="/api/tus-upload-finish"
+ @upload-success="uploadSuccess"
+ @upload-started="$emit('uploadstart')"
/>
</b-form-group>
</div>
@@ -80,6 +78,7 @@ import { models, services, utils } from "django-airavata-api";
import { InputEditorMixin } from "django-airavata-workspace-plugin-api";
import { components } from "django-airavata-common-ui";
import UserStorageFileSelectionContainer from "../../storage/UserStorageFileSelectionContainer";
+import Uppy from "./Uppy";
export default {
name: "file-input-editor",
@@ -87,7 +86,8 @@ export default {
components: {
"data-product-viewer": components.DataProductViewer,
"delete-link": components.DeleteLink,
- UserStorageFileSelectionContainer
+ UserStorageFileSelectionContainer,
+ Uppy
},
computed: {
isDataProductURI() {
@@ -194,20 +194,6 @@ export default {
})
.catch(utils.FetchUtils.reportError);
},
- fileChanged() {
- if (this.file && !this.fileTooLarge) {
- let data = new FormData();
- data.append("file", this.file);
- this.$emit("uploadstart");
- utils.FetchUtils.post("/api/upload", data, "", { showSpinner: false })
- .then(result => {
- this.dataProduct = new models.DataProduct(result["data-product"]);
- this.data = this.dataProduct.productUri;
- this.valueChanged();
- })
- .finally(() => this.$emit("uploadend"));
- }
- },
unselect() {
this.file = null;
this.data = null;
@@ -233,6 +219,12 @@ export default {
this.fileContent = text;
this.$refs.modal.show();
});
+ },
+ uploadSuccess(result) {
+ this.dataProduct = new models.DataProduct(result["data-product"]);
+ this.data = this.dataProduct.productUri;
+ this.valueChanged();
+ this.$emit('uploadend');
}
}
};
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/Uppy.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/Uppy.vue
new file mode 100644
index 0000000..f77ca01
--- /dev/null
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/Uppy.vue
@@ -0,0 +1,93 @@
+<template>
+ <div>
+ <div ref="fileInput" />
+ <div ref="statusBar" />
+ </div>
+</template>
+
+<script>
+import { services, utils } from "django-airavata-api";
+
+import Uppy from "@uppy/core";
+import FileInput from "@uppy/file-input";
+import StatusBar from "@uppy/status-bar";
+import Tus from "@uppy/tus";
+import XHRUpload from "@uppy/xhr-upload";
+
+import "@uppy/core/dist/style.css";
+import "@uppy/status-bar/dist/style.css";
+import "@uppy/file-input/dist/style.css";
+
+// TODO: dispatch upload start
+// TODO: maybe use dragdrop UI?
+export default {
+ name: "uppy",
+ props: {
+ xhrUploadEndpoint: {
+ type: String,
+ required: true
+ },
+ // endpoint should accept POST request. Request will include form data with
+ // the key uploadURL.
+ tusUploadFinishEndpoint: {
+ type: String,
+ required: false
+ }
+ },
+ mounted() {
+ services.SettingsService.get().then(s => {
+ this.settings = s;
+ this.initUppy();
+ });
+ },
+ destroyed() {
+ // TODO: tear down the Uppy instance
+ },
+ methods: {
+ initUppy() {
+ const uppy = Uppy({
+ // TODO: set id
+ autoProceed: true,
+ // TODO: add maxFileSize restriction
+ debug: true
+ });
+ uppy.use(FileInput, { target: this.$refs.fileInput, pretty: false });
+ uppy.use(StatusBar, {
+ target: this.$refs.statusBar,
+ hideUploadButton: true,
+ hideAfterFinish: false
+ });
+ if (this.settings.tusEndpoint) {
+ uppy.use(Tus, { endpoint: this.settings.tusEndpoint });
+ uppy.on("upload-success", (file, response) => {
+ const data = new FormData();
+ data.append("uploadURL", response.uploadURL);
+ utils.FetchUtils.post(this.tusUploadFinishEndpoint, data, "", {
+ showSpinner: false
+ }).then(result => {
+ this.$emit("upload-success", result);
+ });
+ });
+ } else {
+ uppy.use(XHRUpload, {
+ endpoint: this.xhrUploadEndpoint,
+ withCredentials: true,
+ headers: {
+ 'X-CSRFToken': utils.FetchUtils.getCSRFToken()
+ },
+ fieldName: 'file'
+ });
+ uppy.on("upload-success", (file, response) => {
+ this.$emit("upload-success", response.body);
+ });
+ }
+ uppy.on("upload", () => {
+ this.$emit("upload-started");
+ });
+ uppy.on("complete", () => {
+ this.$emit("upload-finished");
+ });
+ }
+ }
+};
+</script>
diff --git a/django_airavata/settings.py b/django_airavata/settings.py
index e18c077..3e63a54 100644
--- a/django_airavata/settings.py
+++ b/django_airavata/settings.py
@@ -217,6 +217,13 @@ FILE_UPLOAD_HANDLERS = [
'django_airavata.uploadhandler.MaxFileSizeTemporaryFileUploadHandler',
]
+# Tus upload
+# Override and set to a valid tus endpoint, for example
+# "http://localhost:1080/files/"
+TUS_ENDPOINT = None
+# Override and set to the directory where tus uploads will be stored
+TUS_DATA_DIR = None
+
# Django REST Framework configuration
REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (