You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/08/04 19:58:50 UTC
[airavata-django-portal] 02/02: AIRAVATA-3342 Update code to use
user_storage model from SDK
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3342-implement-sdk-function-for-managing-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit e8581d68c7a69f0211af46e8537363819932c9f4
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue Aug 4 15:56:14 2020 -0400
AIRAVATA-3342 Update code to use user_storage model from SDK
---
.../apps/api/migrations/0005_delete_user_files.py | 18 +++
django_airavata/apps/api/models.py | 13 --
django_airavata/apps/api/serializers.py | 2 +-
.../apps/api/tests/test_data_products_helper.py | 180 ---------------------
django_airavata/apps/api/views.py | 10 +-
django_airavata/apps/workspace/views.py | 3 +-
django_airavata/settings.py | 3 +
setup.py | 4 +-
8 files changed, 31 insertions(+), 202 deletions(-)
diff --git a/django_airavata/apps/api/migrations/0005_delete_user_files.py b/django_airavata/apps/api/migrations/0005_delete_user_files.py
new file mode 100644
index 0000000..ed22e12
--- /dev/null
+++ b/django_airavata/apps/api/migrations/0005_delete_user_files.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.21 on 2020-07-01 15:59
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_airavata_api', '0004_auto_20191112_2134'),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name='User_Files',
+ ),
+ ]
diff --git a/django_airavata/apps/api/models.py b/django_airavata/apps/api/models.py
index c8bd764..3826d9e 100644
--- a/django_airavata/apps/api/models.py
+++ b/django_airavata/apps/api/models.py
@@ -1,7 +1,5 @@
from django.db import models
-from airavata_django_portal_sdk import base
-
# Create your models here.
class WorkspacePreferences(models.Model):
@@ -35,14 +33,3 @@ class User_Notifications(models.Model):
username = models.CharField(max_length=64)
notification_id = models.CharField(max_length=255)
is_read = models.BooleanField(default=False)
-
-
-class User_Files(base.UserFiles):
-
- class Meta:
- indexes = [
- # FIXME: ideally we would include file_path in the index to make
- # lookups faster, but Django/MariaDB don't support key length on a
- # TEXT column which is required to create an index
- models.Index(fields=['username'], name='username_idx')
- ]
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 89e0211..b214124 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -496,7 +496,7 @@ class DataProductSerializer(
def get_isInputFileUpload(self, data_product):
"""Return True if this is an uploaded input file."""
request = self.context['request']
- return user_storage.is_input_file_upload(request, data_product)
+ return user_storage.is_input_file(request, data_product)
# TODO move this into airavata_sdk?
diff --git a/django_airavata/apps/api/tests/test_data_products_helper.py b/django_airavata/apps/api/tests/test_data_products_helper.py
deleted file mode 100644
index 839cb28..0000000
--- a/django_airavata/apps/api/tests/test_data_products_helper.py
+++ /dev/null
@@ -1,180 +0,0 @@
-import io
-import os
-import tempfile
-import uuid
-from unittest.mock import MagicMock
-from urllib.parse import urlparse
-
-from django.contrib.auth.models import User
-from django.test import RequestFactory, TestCase, override_settings
-
-from airavata.model.data.replica.ttypes import (
- DataProductModel,
- DataProductType,
- DataReplicaLocationModel,
- ReplicaLocationCategory
-)
-from django_airavata.apps.api import data_products_helper
-
-GATEWAY_ID = 'test-gateway'
-
-
-@override_settings(GATEWAY_ID=GATEWAY_ID)
-class BaseTestCase(TestCase):
-
- def setUp(self):
- self.user = User.objects.create_user('testuser')
- self.factory = RequestFactory()
- # Dummy POST request
- self.request = self.factory.post('/upload', {})
- self.request.user = self.user
- self.request.airavata_client = MagicMock(name="airavata_client")
- self.product_uri = f"airavata-dp://{uuid.uuid4()}"
- self.request.airavata_client.registerDataProduct.return_value = \
- self.product_uri
- self.request.authz_token = "dummy"
-
-
-class SaveTests(BaseTestCase):
-
- def test_save_with_defaults(self):
- "Test save with default name and content type"
- with tempfile.TemporaryDirectory() as tmpdirname, \
- self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
- GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
- # path is just the user directory in gateway storage
- path = os.path.join(tmpdirname, self.user.username)
- file = io.StringIO("Foo file")
- file.name = "foo.txt"
- data_product = data_products_helper.save(self.request, path, file)
-
- self.assertEqual(data_product.productUri, self.product_uri)
- self.request.airavata_client.registerDataProduct.\
- assert_called_once()
- args, kws = self.request.airavata_client.registerDataProduct.\
- call_args
- dp = args[1]
- self.assertEqual(self.user.username, dp.ownerName)
- self.assertEqual("foo.txt", dp.productName)
- self.assertEqual(DataProductType.FILE, dp.dataProductType)
- self.assertDictEqual({'mime-type': 'text/plain'},
- dp.productMetadata)
- self.assertEqual(1, len(dp.replicaLocations))
- self.assertEqual(f"file://gateway.com:{path}/{file.name}",
- dp.replicaLocations[0].filePath)
-
- def test_save_with_name_and_content_type(self):
- "Test save with specified name and content type"
- with tempfile.TemporaryDirectory() as tmpdirname, \
- self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
- GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
- # path is just the user directory in gateway storage
- path = os.path.join(tmpdirname, self.user.username)
- file = io.StringIO("Foo file")
- file.name = "foo.txt"
- data_product = data_products_helper.save(
- self.request, path, file, name="bar.txt",
- content_type="application/some-app")
-
- self.assertEqual(data_product.productUri, self.product_uri)
- self.request.airavata_client.registerDataProduct.\
- assert_called_once()
- args, kws = self.request.airavata_client.registerDataProduct.\
- call_args
- dp = args[1]
- self.assertEqual(self.user.username, dp.ownerName)
- self.assertEqual("bar.txt", dp.productName)
- self.assertEqual(DataProductType.FILE, dp.dataProductType)
- self.assertDictEqual({'mime-type': 'application/some-app'},
- dp.productMetadata)
- self.assertEqual(1, len(dp.replicaLocations))
- self.assertEqual(f"file://gateway.com:{path}/bar.txt",
- dp.replicaLocations[0].filePath)
-
- def test_save_with_unknown_text_file_type(self):
- "Test save with unknown file ext for text file"
- with tempfile.TemporaryDirectory() as tmpdirname, \
- self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
- GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
- path = os.path.join(
- tmpdirname, "foo.someext")
- os.makedirs(os.path.dirname(path), exist_ok=True)
- with open(path, 'w') as f:
- f.write("Some Unicode text")
- with open(path, 'r') as f:
- dp = data_products_helper.save(
- self.request, "some/path", f,
- content_type="application/octet-stream")
- # Make sure that the file contents are tested to see if text
- self.assertDictEqual({'mime-type': 'text/plain'},
- dp.productMetadata)
-
- def test_save_with_unknown_binary_file_type(self):
- "Test save with unknown file ext for binary file"
- with tempfile.TemporaryDirectory() as tmpdirname, \
- self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
- GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
- path = os.path.join(
- tmpdirname, "foo.someext")
- os.makedirs(os.path.dirname(path), exist_ok=True)
- with open(path, 'wb') as f:
- f.write(bytes(range(256)))
- with open(path, 'rb') as f:
- dp = data_products_helper.save(
- self.request, "some/path", f,
- content_type="application/octet-stream")
- # Make sure that DID NOT determine file contents are text
- self.assertDictEqual({'mime-type': 'application/octet-stream'},
- dp.productMetadata)
-
-
-class CopyInputFileUploadTests(BaseTestCase):
- def test_copy_input_file_upload(self):
- "Test copy input file upload copies data product"
- with tempfile.TemporaryDirectory() as tmpdirname, \
- self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
- GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
- # path is just the user directory in gateway storage
- source_path = os.path.join(
- tmpdirname, self.user.username, "foo.ext")
- os.makedirs(os.path.dirname(source_path))
- with open(source_path, 'wb') as f:
- f.write(b"123")
-
- data_product = DataProductModel()
- data_product.productUri = f"airavata-dp://{uuid.uuid4()}"
- data_product.gatewayId = GATEWAY_ID
- data_product.ownerName = self.user.username
- data_product.productName = "foo.ext"
- data_product.dataProductType = DataProductType.FILE
- data_product.productMetadata = {
- 'mime-type': 'application/some-app'
- }
- replica_category = ReplicaLocationCategory.GATEWAY_DATA_STORE
- replica_path = f"file://gateway.com:{source_path}"
- data_product.replicaLocations = [
- DataReplicaLocationModel(
- filePath=replica_path,
- replicaLocationCategory=replica_category)]
-
- data_product_copy = data_products_helper.copy_input_file_upload(
- self.request, data_product)
-
- self.request.airavata_client.registerDataProduct.\
- assert_called_once()
- self.assertIsNot(data_product_copy, data_product)
- self.assertNotEqual(data_product_copy.productUri,
- data_product.productUri)
- self.assertDictEqual(data_product_copy.productMetadata,
- data_product.productMetadata)
- self.assertEqual(data_product_copy.productName,
- data_product.productName)
- self.assertEqual(data_product_copy.dataProductType,
- data_product.dataProductType)
- replica_copy_path = data_product_copy.replicaLocations[0].filePath
- self.assertNotEqual(replica_copy_path, replica_path)
- replica_copy_filepath = urlparse(replica_copy_path).path
- self.assertEqual(
- os.path.dirname(replica_copy_filepath),
- os.path.join(tmpdirname, self.user.username, "tmp"),
- msg="Verify input file copied to user's tmp dir")
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 83d61ab..ada9b1a 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -262,10 +262,10 @@ class ExperimentViewSet(APIBackedViewSet):
"""
data_product = self.request.airavata_client.getDataProduct(
self.authz_token, data_product_uri)
- if user_storage.is_input_file_upload(
+ if user_storage.is_input_file(
self.request, data_product):
moved_data_product = \
- user_storage.move_input_file_upload(
+ user_storage.move_input_file(
self.request,
data_product,
experiment_data_dir)
@@ -398,7 +398,7 @@ class ExperimentViewSet(APIBackedViewSet):
source_data_product = self.request.airavata_client.getDataProduct(
self.authz_token, data_product_uri)
if user_storage.exists(self.request, source_data_product):
- return user_storage.copy_input_file_upload(
+ return user_storage.copy_input_file(
self.request, source_data_product)
else:
log.warning("Could not find file for source data "
@@ -942,7 +942,7 @@ class DataProductView(APIView):
def upload_input_file(request):
try:
input_file = request.FILES['file']
- data_product = user_storage.save_input_file_upload(
+ data_product = user_storage.save_input_file(
request, input_file, content_type=input_file.content_type)
serializer = serializers.DataProductSerializer(
data_product, context={'request': request})
@@ -960,7 +960,7 @@ def tus_upload_finish(request):
uploadURL = request.POST['uploadURL']
def move_input_file(file_path, file_name, file_type):
- return user_storage.move_input_file_upload_from_filepath(
+ return user_storage.move_input_file_from_filepath(
request, file_path, name=file_name, content_type=file_type)
try:
data_product = tus.move_tus_upload(uploadURL, move_input_file)
diff --git a/django_airavata/apps/workspace/views.py b/django_airavata/apps/workspace/views.py
index 3959702..b358c3c 100644
--- a/django_airavata/apps/workspace/views.py
+++ b/django_airavata/apps/workspace/views.py
@@ -98,8 +98,9 @@ def create_experiment(request, app_module_id):
user_input_values[app_input['name']] = dp_uri
except Exception as e:
logger.exception(
- "Failed checking data product uri: {dp_uri}")
+ f"Failed checking data product uri: {dp_uri}")
else:
+ # TODO: remove this functionality, data product URI should be passed instead
data_product_uri = user_storage_sdk.user_file_exists(
request, user_file_url.path)
if data_product_uri is not None:
diff --git a/django_airavata/settings.py b/django_airavata/settings.py
index 43b77b3..7fd2c87 100644
--- a/django_airavata/settings.py
+++ b/django_airavata/settings.py
@@ -74,6 +74,9 @@ INSTALLED_APPS = [
# django-webpack-loader
'webpack_loader',
+
+ # Airavata Django Portal SDK
+ 'airavata_django_portal_sdk',
]
# List of app labels for Airavata apps that should be hidden from menus
diff --git a/setup.py b/setup.py
index 2100663..c688283 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,6 @@ setup(
long_description=read('README.md'),
license='Apache License 2.0',
packages=find_packages(),
- # TODO: add dependency on airavata-django-portal-sdk
install_requires=[
'Django',
'djangorestframework',
@@ -31,7 +30,8 @@ setup(
'wagtail',
'wagtailfontawesome',
'jupyter',
- 'papermill'
+ 'papermill',
+ "airavata-django-portal-sdk @ git+https://github.com/machristie/airavata-django-portal-sdk.git@master#egg=airavata-django-portal-sdk",
],
extras_require={
'dev': [