You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/01/03 21:49:52 UTC
[airavata-django-portal] branch develop updated (5a60ce5 -> f3a927e)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git.
from 5a60ce5 Merge branch 'master' into develop
new 594f851 AIRAVATA-3281 Capture content type of uploaded file
new a8add0b AIRAVATA-3281 Tests for data_products_helper
new f3a927e AIRAVATA-3281 Check if file is text
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
django_airavata/apps/api/data_products_helper.py | 93 ++++++++---
django_airavata/apps/api/output_views.py | 2 +-
.../django_airavata_api/js/models/DataProduct.js | 9 ++
.../base/migrations => apps/api/tests}/__init__.py | 0
.../apps/api/tests/test_data_products_helper.py | 180 +++++++++++++++++++++
.../apps/api/{tests.py => tests/test_views.py} | 3 +-
django_airavata/apps/api/tus.py | 5 +-
django_airavata/apps/api/views.py | 17 +-
.../experiment/input-editors/FileInputEditor.vue | 6 +-
.../components/storage/UserStoragePathViewer.vue | 2 +-
.../common/js/components/DataProductViewer.vue | 10 +-
11 files changed, 293 insertions(+), 34 deletions(-)
copy django_airavata/{wagtailapps/base/migrations => apps/api/tests}/__init__.py (100%)
create mode 100644 django_airavata/apps/api/tests/test_data_products_helper.py
rename django_airavata/apps/api/{tests.py => tests/test_views.py} (99%)
[airavata-django-portal] 03/03: AIRAVATA-3281 Check if file is text
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit f3a927e1f8c1bcef7d8e7b4791d8fb3bacf60401
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jan 3 15:39:52 2020 -0500
AIRAVATA-3281 Check if file is text
---
django_airavata/apps/api/data_products_helper.py | 28 ++++++++++++-----
django_airavata/apps/api/output_views.py | 2 +-
.../django_airavata_api/js/models/DataProduct.js | 9 ++++++
.../apps/api/tests/test_data_products_helper.py | 36 ++++++++++++++++++++++
django_airavata/apps/api/views.py | 4 ++-
.../experiment/input-editors/FileInputEditor.vue | 6 +++-
.../components/storage/UserStoragePathViewer.vue | 2 +-
.../common/js/components/DataProductViewer.vue | 10 ++++--
8 files changed, 83 insertions(+), 14 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 1748370..6dcdd84 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -99,7 +99,7 @@ def move_input_file_upload_from_filepath(request, source_path, name=None,
return data_product
-def open(request, data_product):
+def open_file(request, data_product):
"Return file object for replica if it exists in user storage."
path = _get_replica_filepath(data_product)
return datastore.open(data_product.ownerName, path)
@@ -263,18 +263,30 @@ def _create_data_product(username, full_path, name=None,
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
- if content_type is not None:
- data_product.productMetadata = {'mime-type': content_type}
- else:
- # Try to guess the content-type from file extension
- guessed_type, encoding = mimetypes.guess_type(file_name)
- if guessed_type is not None:
- data_product.productMetadata = {'mime-type': guessed_type}
+ final_content_type = _determine_content_type(full_path, content_type)
+ if final_content_type is not None:
+ data_product.productMetadata = {'mime-type': final_content_type}
data_replica_location = _create_replica_location(full_path, file_name)
data_product.replicaLocations = [data_replica_location]
return data_product
+def _determine_content_type(full_path, content_type=None):
+ result = content_type
+ if result is None:
+ # Try to guess the content-type from file extension
+ guessed_type, encoding = mimetypes.guess_type(full_path)
+ result = guessed_type
+ if result is None or result == 'application/octet-stream':
+ # Check if file is Unicode text by trying to read some of it
+ try:
+ open(full_path, 'r').read(1024)
+ result = 'text/plain'
+ except UnicodeDecodeError:
+ logger.debug(f"Failed to read as Unicode text: {full_path}")
+ return result
+
+
def _create_replica_location(full_path, file_name):
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
diff --git a/django_airavata/apps/api/output_views.py b/django_airavata/apps/api/output_views.py
index 2375bd2..0614e56 100644
--- a/django_airavata/apps/api/output_views.py
+++ b/django_airavata/apps/api/output_views.py
@@ -193,7 +193,7 @@ def _generate_data(request,
data_product = request.airavata_client.getDataProduct(
request.authz_token, experiment_output.value)
if data_products_helper.exists(request, data_product):
- output_file = data_products_helper.open(request, data_product)
+ output_file = data_products_helper.open_file(request, data_product)
elif settings.DEBUG and test_output_file is not None:
output_file = open(test_output_file, 'rb')
# TODO: change interface to provide output_file as a path
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js b/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
index 2634807..da6f6d9 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
@@ -32,6 +32,7 @@ const FIELDS = [
];
const FILENAME_REGEX = /[^/]+$/;
+const TEXT_MIME_TYPE_REGEX = /^text\/.+/;
export default class DataProduct extends BaseModel {
constructor(data = {}) {
@@ -49,4 +50,12 @@ export default class DataProduct extends BaseModel {
}
return null;
}
+
+ get isText() {
+ return this.mimeType && TEXT_MIME_TYPE_REGEX.test(this.mimeType);
+ }
+
+ get mimeType() {
+ return this.productMetadata && this.productMetadata['mime-type'] ? this.productMetadata['mime-type'] : null;
+ }
}
diff --git a/django_airavata/apps/api/tests/test_data_products_helper.py b/django_airavata/apps/api/tests/test_data_products_helper.py
index 317e28c..839cb28 100644
--- a/django_airavata/apps/api/tests/test_data_products_helper.py
+++ b/django_airavata/apps/api/tests/test_data_products_helper.py
@@ -91,6 +91,42 @@ class SaveTests(BaseTestCase):
self.assertEqual(f"file://gateway.com:{path}/bar.txt",
dp.replicaLocations[0].filePath)
+ def test_save_with_unknown_text_file_type(self):
+ "Test save with unknown file ext for text file"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ path = os.path.join(
+ tmpdirname, "foo.someext")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'w') as f:
+ f.write("Some Unicode text")
+ with open(path, 'r') as f:
+ dp = data_products_helper.save(
+ self.request, "some/path", f,
+ content_type="application/octet-stream")
+ # Make sure that the file contents are tested to see if text
+ self.assertDictEqual({'mime-type': 'text/plain'},
+ dp.productMetadata)
+
+ def test_save_with_unknown_binary_file_type(self):
+ "Test save with unknown file ext for binary file"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ path = os.path.join(
+ tmpdirname, "foo.someext")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'wb') as f:
+ f.write(bytes(range(256)))
+ with open(path, 'rb') as f:
+ dp = data_products_helper.save(
+ self.request, "some/path", f,
+ content_type="application/octet-stream")
+ # Make sure that DID NOT determine file contents are text
+ self.assertDictEqual({'mime-type': 'application/octet-stream'},
+ dp.productMetadata)
+
class CopyInputFileUploadTests(BaseTestCase):
def test_copy_input_file_upload(self):
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 2cf629b..007c4ba 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -975,12 +975,14 @@ def download_file(request):
.format(data_product_uri), exc_info=True)
raise Http404("data product does not exist") from e
try:
- data_file = data_products_helper.open(request, data_product)
+ data_file = data_products_helper.open_file(request, data_product)
response = FileResponse(data_file, content_type=mime_type)
file_name = os.path.basename(data_file.name)
if mime_type == 'application/octet-stream' or force_download:
response['Content-Disposition'] = ('attachment; filename="{}"'
.format(file_name))
+ else:
+ response['Content-Disposition'] = f'filename="{file_name}"'
return response
except ObjectDoesNotExist as e:
raise Http404(str(e)) from e
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
index 52c0c39..3dcf631 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
@@ -8,8 +8,9 @@
class="mr-auto"
:data-product="dataProduct"
:input-file="true"
+ :open-in-new-window="true"
/>
- <b-link @click="viewFile">
+ <b-link @click="viewFile" v-if="isViewable">
View File <i class="fa fa-eye"></i>
<span class="sr-only">View file</span>
</b-link>
@@ -85,6 +86,9 @@ export default {
} else {
return [];
}
+ },
+ isViewable() {
+ return this.dataProduct.isText;
}
},
data() {
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
index 3351fd1..627935f 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
@@ -22,7 +22,7 @@
v-else
:href="data.item.downloadURL"
:target="downloadTarget"
- > <i class="fa fa-download"></i> {{ data.item.name }}</b-link>
+ > {{ data.item.name }}</b-link>
</template>
<template
slot="createdTimestamp"
diff --git a/django_airavata/static/common/js/components/DataProductViewer.vue b/django_airavata/static/common/js/components/DataProductViewer.vue
index 52f35f7..788ca79 100644
--- a/django_airavata/static/common/js/components/DataProductViewer.vue
+++ b/django_airavata/static/common/js/components/DataProductViewer.vue
@@ -1,8 +1,7 @@
<template>
<span v-if="downloadURL">
- <a :href="downloadURL" class="action-link">
- <i class="fa fa-download"></i>
+ <a :href="downloadURL" class="action-link" :target="linkTarget">
{{ filename }}
</a>
</span>
@@ -24,6 +23,10 @@ export default {
},
mimeType: {
type: String
+ },
+ openInNewWindow: {
+ type: Boolean,
+ default: false
}
},
computed: {
@@ -45,6 +48,9 @@ export default {
} else {
return this.dataProduct.downloadURL;
}
+ },
+ linkTarget() {
+ return this.openInNewWindow ? "_blank": "_self";
}
}
};
[airavata-django-portal] 01/03: AIRAVATA-3281 Capture content type
of uploaded file
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 594f851ff97cc208386c71c97a71468a5067b482
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Thu Jan 2 14:57:56 2020 -0500
AIRAVATA-3281 Capture content type of uploaded file
---
django_airavata/apps/api/data_products_helper.py | 79 +++++++++++++++++++-----
django_airavata/apps/api/tus.py | 5 +-
django_airavata/apps/api/views.py | 13 ++--
3 files changed, 72 insertions(+), 25 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 30ab128..1748370 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -1,4 +1,6 @@
+import copy
import logging
+import mimetypes
import os
from urllib.parse import urlparse
@@ -19,30 +21,34 @@ logger = logging.getLogger(__name__)
TMP_INPUT_FILE_UPLOAD_DIR = "tmp"
-def save(request, path, file, name=None):
+def save(request, path, file, name=None, content_type=None):
"Save file in path in the user's storage."
username = request.user.username
full_path = datastore.save(username, path, file, name=name)
- data_product = _save_data_product(request, full_path, name=name)
+ data_product = _save_data_product(request, full_path, name=name,
+ content_type=content_type)
return data_product
-def move_from_filepath(request, source_path, target_path, name=None):
+def move_from_filepath(request, source_path, target_path, name=None,
+ content_type=None):
"Move a file from filesystem into user's storage."
username = request.user.username
file_name = name if name is not None else os.path.basename(source_path)
full_path = datastore.move_external(
source_path, username, target_path, file_name)
- data_product = _save_data_product(request, full_path, name=file_name)
+ data_product = _save_data_product(request, full_path, name=file_name,
+ content_type=content_type)
return data_product
-def save_input_file_upload(request, file, name=None):
+def save_input_file_upload(request, file, name=None, content_type=None):
"""Save input file in staging area for input file uploads."""
username = request.user.username
file_name = name if name is not None else os.path.basename(file.name)
full_path = datastore.save(username, TMP_INPUT_FILE_UPLOAD_DIR, file)
- data_product = _save_data_product(request, full_path, name=file_name)
+ data_product = _save_data_product(request, full_path, name=file_name,
+ content_type=content_type)
return data_product
@@ -54,7 +60,7 @@ def copy_input_file_upload(request, data_product):
request.user.username,
TMP_INPUT_FILE_UPLOAD_DIR,
name=name)
- return _save_data_product(request, full_path, name=name)
+ return _save_copy_of_data_product(request, full_path, data_product)
def is_input_file_upload(request, data_product):
@@ -77,17 +83,19 @@ def move_input_file_upload(request, data_product, path):
path,
file_name)
_delete_data_product(data_product.ownerName, source_path)
- data_product = _save_data_product(request, full_path, name=file_name)
+ data_product = _save_copy_of_data_product(request, full_path, data_product)
return data_product
-def move_input_file_upload_from_filepath(request, source_path, name=None):
+def move_input_file_upload_from_filepath(request, source_path, name=None,
+ content_type=None):
"Move a file from filesystem into user's input file staging area."
username = request.user.username
file_name = name if name is not None else os.path.basename(source_path)
full_path = datastore.move_external(
source_path, username, TMP_INPUT_FILE_UPLOAD_DIR, file_name)
- data_product = _save_data_product(request, full_path, name=file_name)
+ data_product = _save_data_product(request, full_path, name=file_name,
+ content_type=content_type)
return data_product
@@ -196,19 +204,43 @@ def _get_data_product_uri(request, full_path):
return product_uri
-def _save_data_product(request, full_path, name=None):
+def _save_data_product(request, full_path, name=None, content_type=None):
"Create, register and record in DB a data product for full_path."
data_product = _create_data_product(
- request.user.username, full_path, name=name)
+ request.user.username, full_path, name=name, content_type=content_type)
+ product_uri = _register_data_product(request, full_path, data_product)
+ data_product.productUri = product_uri
+ return data_product
+
+
+def _register_data_product(request, full_path, data_product):
product_uri = request.airavata_client.registerDataProduct(
request.authz_token, data_product)
- data_product.productUri = product_uri
user_file_instance = models.User_Files(
username=request.user.username,
file_path=full_path,
file_dpu=product_uri)
user_file_instance.save()
- return data_product
+ return product_uri
+
+
+def _save_copy_of_data_product(request, full_path, data_product):
+ """Save copy of a data product with a different path."""
+ data_product_copy = _copy_data_product(request, data_product, full_path)
+ product_uri = _register_data_product(request, full_path, data_product_copy)
+ data_product_copy.productUri = product_uri
+ return data_product_copy
+
+
+def _copy_data_product(request, data_product, full_path):
+ """Create an unsaved copy of a data product with different path."""
+ data_product_copy = copy.copy(data_product)
+ data_product_copy.productUri = None
+ data_product_copy.ownerName = request.user.username
+ data_replica_location = _create_replica_location(
+ full_path, data_product_copy.productName)
+ data_product_copy.replicaLocations = [data_replica_location]
+ return data_product_copy
def _delete_data_product(username, full_path):
@@ -220,7 +252,8 @@ def _delete_data_product(username, full_path):
user_file.delete()
-def _create_data_product(username, full_path, name=None):
+def _create_data_product(username, full_path, name=None,
+ content_type=None):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
data_product.ownerName = username
@@ -230,6 +263,19 @@ def _create_data_product(username, full_path, name=None):
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
+ if content_type is not None:
+ data_product.productMetadata = {'mime-type': content_type}
+ else:
+ # Try to guess the content-type from file extension
+ guessed_type, encoding = mimetypes.guess_type(file_name)
+ if guessed_type is not None:
+ data_product.productMetadata = {'mime-type': guessed_type}
+ data_replica_location = _create_replica_location(full_path, file_name)
+ data_product.replicaLocations = [data_replica_location]
+ return data_product
+
+
+def _create_replica_location(full_path, file_name):
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
settings.GATEWAY_DATA_STORE_RESOURCE_ID
@@ -242,8 +288,7 @@ def _create_data_product(username, full_path, name=None):
data_replica_location.filePath = \
"file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
full_path)
- data_product.replicaLocations = [data_replica_location]
- return data_product
+ return data_replica_location
def _get_replica_filepath(data_product):
diff --git a/django_airavata/apps/api/tus.py b/django_airavata/apps/api/tus.py
index 41d7640..8f6fea1 100644
--- a/django_airavata/apps/api/tus.py
+++ b/django_airavata/apps/api/tus.py
@@ -12,7 +12,7 @@ def move_tus_upload(upload_url, move_function):
"""
Move upload identified by upload_url using the provided move_function.
- move_function signature should be (file_path, file_name). It's
+ move_function signature should be (file_path, file_name, file_type). It's
return value will be returned.
"""
# file UUID is last path component in URL. For example:
@@ -34,6 +34,7 @@ def move_tus_upload(upload_url, move_function):
with open(upload_info_path) as upload_info_file:
upload_info = json.load(upload_info_file)
filename = upload_info['MetaData']['filename']
- result = move_function(upload_bin_path, filename)
+ filetype = upload_info['MetaData']['filetype']
+ result = move_function(upload_bin_path, filename, filetype)
os.remove(upload_info_path)
return result
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index c168a4f..2cf629b 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -926,7 +926,7 @@ def upload_input_file(request):
try:
input_file = request.FILES['file']
data_product = data_products_helper.save_input_file_upload(
- request, input_file)
+ request, input_file, content_type=input_file.content_type)
serializer = serializers.DataProductSerializer(
data_product, context={'request': request})
return JsonResponse({'uploaded': True,
@@ -942,9 +942,9 @@ def upload_input_file(request):
def tus_upload_finish(request):
uploadURL = request.POST['uploadURL']
- def move_input_file(file_path, file_name):
+ def move_input_file(file_path, file_name, file_type):
return data_products_helper.move_input_file_upload_from_filepath(
- request, file_path, name=file_name)
+ request, file_path, name=file_name, content_type=file_type)
try:
data_product = tus.move_tus_upload(uploadURL, move_input_file)
serializer = serializers.DataProductSerializer(
@@ -1489,14 +1489,15 @@ class UserStoragePathView(APIView):
if 'file' in request.FILES:
user_file = request.FILES['file']
data_product = data_products_helper.save(
- request, path, user_file)
+ request, path, user_file, content_type=user_file.content_type)
# Handle a tus upload
elif 'uploadURL' in request.POST:
uploadURL = request.POST['uploadURL']
- def move_file(file_path, file_name):
+ def move_file(file_path, file_name, file_type):
return data_products_helper.move_from_filepath(
- request, file_path, path, name=file_name)
+ request, file_path, path, name=file_name,
+ content_type=file_type)
data_product = tus.move_tus_upload(uploadURL, move_file)
return self._create_response(request, path, uploaded=data_product)
[airavata-django-portal] 02/03: AIRAVATA-3281 Tests for
data_products_helper
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit a8add0bfb52bb42bf5c016e738242774af1fcca1
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jan 3 12:09:35 2020 -0500
AIRAVATA-3281 Tests for data_products_helper
---
django_airavata/apps/api/tests/__init__.py | 0
.../apps/api/tests/test_data_products_helper.py | 144 +++++++++++++++++++++
.../apps/api/{tests.py => tests/test_views.py} | 3 +-
3 files changed, 145 insertions(+), 2 deletions(-)
diff --git a/django_airavata/apps/api/tests/__init__.py b/django_airavata/apps/api/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/django_airavata/apps/api/tests/test_data_products_helper.py b/django_airavata/apps/api/tests/test_data_products_helper.py
new file mode 100644
index 0000000..317e28c
--- /dev/null
+++ b/django_airavata/apps/api/tests/test_data_products_helper.py
@@ -0,0 +1,144 @@
+import io
+import os
+import tempfile
+import uuid
+from unittest.mock import MagicMock
+from urllib.parse import urlparse
+
+from django.contrib.auth.models import User
+from django.test import RequestFactory, TestCase, override_settings
+
+from airavata.model.data.replica.ttypes import (
+ DataProductModel,
+ DataProductType,
+ DataReplicaLocationModel,
+ ReplicaLocationCategory
+)
+from django_airavata.apps.api import data_products_helper
+
+GATEWAY_ID = 'test-gateway'
+
+
+@override_settings(GATEWAY_ID=GATEWAY_ID)
+class BaseTestCase(TestCase):
+
+ def setUp(self):
+ self.user = User.objects.create_user('testuser')
+ self.factory = RequestFactory()
+ # Dummy POST request
+ self.request = self.factory.post('/upload', {})
+ self.request.user = self.user
+ self.request.airavata_client = MagicMock(name="airavata_client")
+ self.product_uri = f"airavata-dp://{uuid.uuid4()}"
+ self.request.airavata_client.registerDataProduct.return_value = \
+ self.product_uri
+ self.request.authz_token = "dummy"
+
+
+class SaveTests(BaseTestCase):
+
+ def test_save_with_defaults(self):
+ "Test save with default name and content type"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ # path is just the user directory in gateway storage
+ path = os.path.join(tmpdirname, self.user.username)
+ file = io.StringIO("Foo file")
+ file.name = "foo.txt"
+ data_product = data_products_helper.save(self.request, path, file)
+
+ self.assertEqual(data_product.productUri, self.product_uri)
+ self.request.airavata_client.registerDataProduct.\
+ assert_called_once()
+ args, kws = self.request.airavata_client.registerDataProduct.\
+ call_args
+ dp = args[1]
+ self.assertEqual(self.user.username, dp.ownerName)
+ self.assertEqual("foo.txt", dp.productName)
+ self.assertEqual(DataProductType.FILE, dp.dataProductType)
+ self.assertDictEqual({'mime-type': 'text/plain'},
+ dp.productMetadata)
+ self.assertEqual(1, len(dp.replicaLocations))
+ self.assertEqual(f"file://gateway.com:{path}/{file.name}",
+ dp.replicaLocations[0].filePath)
+
+ def test_save_with_name_and_content_type(self):
+ "Test save with specified name and content type"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ # path is just the user directory in gateway storage
+ path = os.path.join(tmpdirname, self.user.username)
+ file = io.StringIO("Foo file")
+ file.name = "foo.txt"
+ data_product = data_products_helper.save(
+ self.request, path, file, name="bar.txt",
+ content_type="application/some-app")
+
+ self.assertEqual(data_product.productUri, self.product_uri)
+ self.request.airavata_client.registerDataProduct.\
+ assert_called_once()
+ args, kws = self.request.airavata_client.registerDataProduct.\
+ call_args
+ dp = args[1]
+ self.assertEqual(self.user.username, dp.ownerName)
+ self.assertEqual("bar.txt", dp.productName)
+ self.assertEqual(DataProductType.FILE, dp.dataProductType)
+ self.assertDictEqual({'mime-type': 'application/some-app'},
+ dp.productMetadata)
+ self.assertEqual(1, len(dp.replicaLocations))
+ self.assertEqual(f"file://gateway.com:{path}/bar.txt",
+ dp.replicaLocations[0].filePath)
+
+
+class CopyInputFileUploadTests(BaseTestCase):
+ def test_copy_input_file_upload(self):
+ "Test copy input file upload copies data product"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ # path is just the user directory in gateway storage
+ source_path = os.path.join(
+ tmpdirname, self.user.username, "foo.ext")
+ os.makedirs(os.path.dirname(source_path))
+ with open(source_path, 'wb') as f:
+ f.write(b"123")
+
+ data_product = DataProductModel()
+ data_product.productUri = f"airavata-dp://{uuid.uuid4()}"
+ data_product.gatewayId = GATEWAY_ID
+ data_product.ownerName = self.user.username
+ data_product.productName = "foo.ext"
+ data_product.dataProductType = DataProductType.FILE
+ data_product.productMetadata = {
+ 'mime-type': 'application/some-app'
+ }
+ replica_category = ReplicaLocationCategory.GATEWAY_DATA_STORE
+ replica_path = f"file://gateway.com:{source_path}"
+ data_product.replicaLocations = [
+ DataReplicaLocationModel(
+ filePath=replica_path,
+ replicaLocationCategory=replica_category)]
+
+ data_product_copy = data_products_helper.copy_input_file_upload(
+ self.request, data_product)
+
+ self.request.airavata_client.registerDataProduct.\
+ assert_called_once()
+ self.assertIsNot(data_product_copy, data_product)
+ self.assertNotEqual(data_product_copy.productUri,
+ data_product.productUri)
+ self.assertDictEqual(data_product_copy.productMetadata,
+ data_product.productMetadata)
+ self.assertEqual(data_product_copy.productName,
+ data_product.productName)
+ self.assertEqual(data_product_copy.dataProductType,
+ data_product.dataProductType)
+ replica_copy_path = data_product_copy.replicaLocations[0].filePath
+ self.assertNotEqual(replica_copy_path, replica_path)
+ replica_copy_filepath = urlparse(replica_copy_path).path
+ self.assertEqual(
+ os.path.dirname(replica_copy_filepath),
+ os.path.join(tmpdirname, self.user.username, "tmp"),
+ msg="Verify input file copied to user's tmp dir")
diff --git a/django_airavata/apps/api/tests.py b/django_airavata/apps/api/tests/test_views.py
similarity index 99%
rename from django_airavata/apps/api/tests.py
rename to django_airavata/apps/api/tests/test_views.py
index 8d87d97..a70703f 100644
--- a/django_airavata/apps/api/tests.py
+++ b/django_airavata/apps/api/tests/test_views.py
@@ -9,8 +9,7 @@ from rest_framework.test import APIRequestFactory, force_authenticate
from airavata.model.appcatalog.gatewaygroups.ttypes import GatewayGroups
from airavata.model.group.ttypes import GroupModel
from airavata.model.user.ttypes import UserProfile
-
-from . import signals, views
+from django_airavata.apps.api import signals, views
GATEWAY_ID = "test-gateway"
PORTAL_ADMINS = [('Admin Name', 'admin@example.com')]