You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/10/16 19:15:32 UTC
[airavata-django-portal] branch master updated (e201154 -> daefdb4)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git.
from e201154 Fixing linting errors
new 4821097 AIRAVATA-3081 enforce max file size in tus upload finisher
new daefdb4 AIRAVATA-3081 docs: tusd installation notes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
django_airavata/apps/api/exceptions.py | 7 +++
django_airavata/apps/api/tus.py | 8 +++
django_airavata/apps/api/views.py | 15 +++---
django_airavata/settings_local.py.sample | 6 +++
docs/admin/tusd.md | 85 ++++++++++++++++++++++++++++++++
mkdocs.yml | 2 +
6 files changed, 117 insertions(+), 6 deletions(-)
create mode 100644 docs/admin/tusd.md
[airavata-django-portal] 01/02: AIRAVATA-3081 enforce max file size
in tus upload finisher
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 4821097487d60f1ec4b8887c52a8ca604fbca2d7
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Oct 16 12:56:26 2019 -0400
AIRAVATA-3081 enforce max file size in tus upload finisher
---
django_airavata/apps/api/exceptions.py | 7 +++++++
django_airavata/apps/api/tus.py | 8 ++++++++
django_airavata/apps/api/views.py | 15 +++++++++------
3 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/django_airavata/apps/api/exceptions.py b/django_airavata/apps/api/exceptions.py
index e95eee8..59d5271 100644
--- a/django_airavata/apps/api/exceptions.py
+++ b/django_airavata/apps/api/exceptions.py
@@ -1,6 +1,7 @@
import logging
from django.core.exceptions import ObjectDoesNotExist
+from django.http import JsonResponse
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import exception_handler
@@ -52,3 +53,9 @@ def custom_exception_handler(exc, context):
)
return response
+
+
+# For non-Django REST Framework error responses
+def generic_json_exception_response(
+ exc, status=status.HTTP_500_INTERNAL_SERVER_ERROR):
+ return JsonResponse({'detail': str(exc)}, status=status)
diff --git a/django_airavata/apps/api/tus.py b/django_airavata/apps/api/tus.py
index c9cf64a..41d7640 100644
--- a/django_airavata/apps/api/tus.py
+++ b/django_airavata/apps/api/tus.py
@@ -23,6 +23,14 @@ def move_tus_upload(upload_url, move_function):
logger.debug(f"upload_bin_path={upload_bin_path}")
upload_info_path = os.path.join(settings.TUS_DATA_DIR,
f"{upload_uuid}.info")
+ if os.path.getsize(upload_bin_path) > settings.FILE_UPLOAD_MAX_FILE_SIZE:
+ error_message = (f"File size of {upload_bin_path} is greater than "
+ f"the max of {settings.FILE_UPLOAD_MAX_FILE_SIZE} "
+ f"bytes")
+ logger.error(error_message)
+ os.remove(upload_bin_path)
+ os.remove(upload_info_path)
+ raise Exception(error_message)
with open(upload_info_path) as upload_info_file:
upload_info = json.load(upload_info_file)
filename = upload_info['MetaData']['filename']
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 4ba414c..656f620 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -46,6 +46,7 @@ from django_airavata.apps.auth.models import EmailVerification
from . import (
data_products_helper,
+ exceptions,
helpers,
models,
output_views,
@@ -910,17 +911,19 @@ def upload_input_file(request):
@login_required
def tus_upload_finish(request):
- log.debug("POST={}".format(request.POST))
uploadURL = request.POST['uploadURL']
def move_input_file(file_path, file_name):
return data_products_helper.move_input_file_upload_from_filepath(
request, file_path, name=file_name)
- data_product = tus.move_tus_upload(uploadURL, move_input_file)
- serializer = serializers.DataProductSerializer(
- data_product, context={'request': request})
- return JsonResponse({'uploaded': True,
- 'data-product': serializer.data})
+ try:
+ data_product = tus.move_tus_upload(uploadURL, move_input_file)
+ serializer = serializers.DataProductSerializer(
+ data_product, context={'request': request})
+ return JsonResponse({'uploaded': True,
+ 'data-product': serializer.data})
+ except Exception as e:
+ return exceptions.generic_json_exception_response(e, status=400)
@login_required
[airavata-django-portal] 02/02: AIRAVATA-3081 docs: tusd
installation notes
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit daefdb43a5849aa7388aa4ba2ebc5b0b1dc00dc7
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Oct 16 15:13:21 2019 -0400
AIRAVATA-3081 docs: tusd installation notes
---
django_airavata/settings_local.py.sample | 6 +++
docs/admin/tusd.md | 85 ++++++++++++++++++++++++++++++++
mkdocs.yml | 2 +
3 files changed, 93 insertions(+)
diff --git a/django_airavata/settings_local.py.sample b/django_airavata/settings_local.py.sample
index 8a15edd..d36b38c 100644
--- a/django_airavata/settings_local.py.sample
+++ b/django_airavata/settings_local.py.sample
@@ -97,6 +97,12 @@ SHARING_API_SECURE = False
# Portal settings
PORTAL_TITLE = 'Django Airavata Gateway'
+# Tus upload - uncomment the following to enable tus uploads
+# Override and set to a valid tus endpoint
+# TUS_ENDPOINT = "https://tus.domainname.org/files/"
+# Override and set to the directory where tus uploads will be stored.
+# TUS_DATA_DIR = "/path/to/tus-temp-dir"
+
# Logging configuration. Uncomment following to override default log configuration
# LOGGING = {
# 'version': 1,
diff --git a/docs/admin/tusd.md b/docs/admin/tusd.md
new file mode 100644
index 0000000..774ffdc
--- /dev/null
+++ b/docs/admin/tusd.md
@@ -0,0 +1,85 @@
+# tusd
+
+[tusd](https://github.com/tus/tusd) is the reference implementation of the
+[tus resumable upload protocol](https://tus.io/). The Airavata Django Portal
+integrates with tus to provide better support for large file uploads. Installing
+tusd and integrating with the Airavata Django Portal is entirely optional but is
+highly recommended if your users will be uploading files larger than, say, 1 GB.
+
+## Installation Notes
+
+!!! note
+
+ Version 1.0 of tusd introduces some breaking changes. The Airavata Django
+ Portal has been tested with version 0.13 of tusd.
+
+Install
+[version 0.13.3 of tusd](https://github.com/tus/tusd/releases/tag/0.13.3) from
+the GitHub releases page.
+
+How you install tusd is up to you, but here are some notes on a specific way to
+deploy it.
+
+### Reverse proxy behind Apache httpd
+
+Create a virtual host config file, for example in `/etc/httpd/conf.d/tus.conf`.
+(Actual location of config files is OS dependent.)
+
+Contents of tus.conf:
+
+```xml
+<VirtualHost *:443>
+ ServerName tus.domainname.org
+ SSLEngine on
+
+ SSLCertificateChainFile /etc/letsencrypt/live/tus.domainname.org/fullchain.pem
+ SSLCertificateFile /etc/letsencrypt/live/tus.domainname.org/cert.pem
+ SSLCertificateKeyFile /etc/letsencrypt/live/tus.domainname.org/privkey.pem
+
+ RequestHeader set X-Forwarded-Proto "https"
+ ProxyPreserveHost on
+
+ ProxyPass /files http://localhost:10080/files
+ ProxyPassReverse /files http://localhost:10080/files
+</VirtualHost>
+```
+
+Note this example assumes that the tus server willl be proxied behind an SSL
+connection and specifically that the SSL certificate will be generated with
+Let's Encrypt.
+
+With this configuration the **TUS_ENDPOINT** will be
+`https://tus.domainname.org/files/`
+
+### Systemd unit file
+
+Add the following `tus.service` file to `/etc/systemd/system/`.
+
+```ini
+[Unit]
+Description=Tusd
+
+[Service]
+ExecStart=/home/PORTALUSER/tusd_linux_amd64/tusd -dir /path/to/tus-temp-dir -port 10080 -behind-proxy
+User=PORTALUSER
+Group=PORTALUSER
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Replace `PORTALUSER` with the username of the user under which the Django portal
+runs.
+
+With this configuration, the **TUS_DATA_DIR** will be `/path/to/tus-temp-dir`
+
+## settings_local.py configuration
+
+Uncomment and set the following settings in settings_local.py, assuming you
+created the settings_local.py file by first copying settings_local.py.sample.
+
+- **TUS_ENDPOINT** - this should be the tus url. Keeping with the example
+ above this would be `https://tus.domainname.org/files/`
+- **TUS_DATA_DIR** - this is the directory where tus upload files and metadata
+ will be stored. This settings assumes that tusd the tus server used and so
+ it assumes that file naming conventions will follow the tusd implementation.
diff --git a/mkdocs.yml b/mkdocs.yml
index 5294472..af6dfd8 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -11,6 +11,8 @@ nav:
- Adding a Custom Django App: dev/custom_django_app.md
- Developing the Frontend: dev/developing_frontend.md
- Developing the Backend: dev/developing_backend.md
+ - Administrator Guide:
+ - Tusd Installation: admin/tusd.md
- Tutorials:
- Gateways 2019: tutorial/gateways2019_tutorial.md