You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/01/03 21:49:55 UTC
[airavata-django-portal] 03/03: AIRAVATA-3281 Check if file is text
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit f3a927e1f8c1bcef7d8e7b4791d8fb3bacf60401
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jan 3 15:39:52 2020 -0500
AIRAVATA-3281 Check if file is text
---
django_airavata/apps/api/data_products_helper.py | 28 ++++++++++++-----
django_airavata/apps/api/output_views.py | 2 +-
.../django_airavata_api/js/models/DataProduct.js | 9 ++++++
.../apps/api/tests/test_data_products_helper.py | 36 ++++++++++++++++++++++
django_airavata/apps/api/views.py | 4 ++-
.../experiment/input-editors/FileInputEditor.vue | 6 +++-
.../components/storage/UserStoragePathViewer.vue | 2 +-
.../common/js/components/DataProductViewer.vue | 10 ++++--
8 files changed, 83 insertions(+), 14 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 1748370..6dcdd84 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -99,7 +99,7 @@ def move_input_file_upload_from_filepath(request, source_path, name=None,
return data_product
-def open(request, data_product):
+def open_file(request, data_product):
"Return file object for replica if it exists in user storage."
path = _get_replica_filepath(data_product)
return datastore.open(data_product.ownerName, path)
@@ -263,18 +263,30 @@ def _create_data_product(username, full_path, name=None,
file_name = os.path.basename(full_path)
data_product.productName = file_name
data_product.dataProductType = DataProductType.FILE
- if content_type is not None:
- data_product.productMetadata = {'mime-type': content_type}
- else:
- # Try to guess the content-type from file extension
- guessed_type, encoding = mimetypes.guess_type(file_name)
- if guessed_type is not None:
- data_product.productMetadata = {'mime-type': guessed_type}
+ final_content_type = _determine_content_type(full_path, content_type)
+ if final_content_type is not None:
+ data_product.productMetadata = {'mime-type': final_content_type}
data_replica_location = _create_replica_location(full_path, file_name)
data_product.replicaLocations = [data_replica_location]
return data_product
+def _determine_content_type(full_path, content_type=None):
+ result = content_type
+ if result is None:
+ # Try to guess the content-type from file extension
+ guessed_type, encoding = mimetypes.guess_type(full_path)
+ result = guessed_type
+ if result is None or result == 'application/octet-stream':
+ # Check if file is Unicode text by trying to read some of it
+ try:
+ open(full_path, 'r').read(1024)
+ result = 'text/plain'
+ except UnicodeDecodeError:
+ logger.debug(f"Failed to read as Unicode text: {full_path}")
+ return result
+
+
def _create_replica_location(full_path, file_name):
data_replica_location = DataReplicaLocationModel()
data_replica_location.storageResourceId = \
diff --git a/django_airavata/apps/api/output_views.py b/django_airavata/apps/api/output_views.py
index 2375bd2..0614e56 100644
--- a/django_airavata/apps/api/output_views.py
+++ b/django_airavata/apps/api/output_views.py
@@ -193,7 +193,7 @@ def _generate_data(request,
data_product = request.airavata_client.getDataProduct(
request.authz_token, experiment_output.value)
if data_products_helper.exists(request, data_product):
- output_file = data_products_helper.open(request, data_product)
+ output_file = data_products_helper.open_file(request, data_product)
elif settings.DEBUG and test_output_file is not None:
output_file = open(test_output_file, 'rb')
# TODO: change interface to provide output_file as a path
diff --git a/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js b/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
index 2634807..da6f6d9 100644
--- a/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
+++ b/django_airavata/apps/api/static/django_airavata_api/js/models/DataProduct.js
@@ -32,6 +32,7 @@ const FIELDS = [
];
const FILENAME_REGEX = /[^/]+$/;
+const TEXT_MIME_TYPE_REGEX = /^text\/.+/;
export default class DataProduct extends BaseModel {
constructor(data = {}) {
@@ -49,4 +50,12 @@ export default class DataProduct extends BaseModel {
}
return null;
}
+
+ get isText() {
+ return this.mimeType && TEXT_MIME_TYPE_REGEX.test(this.mimeType);
+ }
+
+ get mimeType() {
+ return this.productMetadata && this.productMetadata['mime-type'] ? this.productMetadata['mime-type'] : null;
+ }
}
diff --git a/django_airavata/apps/api/tests/test_data_products_helper.py b/django_airavata/apps/api/tests/test_data_products_helper.py
index 317e28c..839cb28 100644
--- a/django_airavata/apps/api/tests/test_data_products_helper.py
+++ b/django_airavata/apps/api/tests/test_data_products_helper.py
@@ -91,6 +91,42 @@ class SaveTests(BaseTestCase):
self.assertEqual(f"file://gateway.com:{path}/bar.txt",
dp.replicaLocations[0].filePath)
+ def test_save_with_unknown_text_file_type(self):
+ "Test save with unknown file ext for text file"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ path = os.path.join(
+ tmpdirname, "foo.someext")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'w') as f:
+ f.write("Some Unicode text")
+ with open(path, 'r') as f:
+ dp = data_products_helper.save(
+ self.request, "some/path", f,
+ content_type="application/octet-stream")
+ # Make sure that the file contents are tested to see if text
+ self.assertDictEqual({'mime-type': 'text/plain'},
+ dp.productMetadata)
+
+ def test_save_with_unknown_binary_file_type(self):
+ "Test save with unknown file ext for binary file"
+ with tempfile.TemporaryDirectory() as tmpdirname, \
+ self.settings(GATEWAY_DATA_STORE_DIR=tmpdirname,
+ GATEWAY_DATA_STORE_HOSTNAME="gateway.com"):
+ path = os.path.join(
+ tmpdirname, "foo.someext")
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'wb') as f:
+ f.write(bytes(range(256)))
+ with open(path, 'rb') as f:
+ dp = data_products_helper.save(
+ self.request, "some/path", f,
+ content_type="application/octet-stream")
+ # Make sure that DID NOT determine file contents are text
+ self.assertDictEqual({'mime-type': 'application/octet-stream'},
+ dp.productMetadata)
+
class CopyInputFileUploadTests(BaseTestCase):
def test_copy_input_file_upload(self):
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 2cf629b..007c4ba 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -975,12 +975,14 @@ def download_file(request):
.format(data_product_uri), exc_info=True)
raise Http404("data product does not exist") from e
try:
- data_file = data_products_helper.open(request, data_product)
+ data_file = data_products_helper.open_file(request, data_product)
response = FileResponse(data_file, content_type=mime_type)
file_name = os.path.basename(data_file.name)
if mime_type == 'application/octet-stream' or force_download:
response['Content-Disposition'] = ('attachment; filename="{}"'
.format(file_name))
+ else:
+ response['Content-Disposition'] = f'filename="{file_name}"'
return response
except ObjectDoesNotExist as e:
raise Http404(str(e)) from e
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
index 52c0c39..3dcf631 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/experiment/input-editors/FileInputEditor.vue
@@ -8,8 +8,9 @@
class="mr-auto"
:data-product="dataProduct"
:input-file="true"
+ :open-in-new-window="true"
/>
- <b-link @click="viewFile">
+ <b-link @click="viewFile" v-if="isViewable">
View File <i class="fa fa-eye"></i>
<span class="sr-only">View file</span>
</b-link>
@@ -85,6 +86,9 @@ export default {
} else {
return [];
}
+ },
+ isViewable() {
+ return this.dataProduct.isText;
}
},
data() {
diff --git a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
index 3351fd1..627935f 100644
--- a/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
+++ b/django_airavata/apps/workspace/static/django_airavata_workspace/js/components/storage/UserStoragePathViewer.vue
@@ -22,7 +22,7 @@
v-else
:href="data.item.downloadURL"
:target="downloadTarget"
- > <i class="fa fa-download"></i> {{ data.item.name }}</b-link>
+ > {{ data.item.name }}</b-link>
</template>
<template
slot="createdTimestamp"
diff --git a/django_airavata/static/common/js/components/DataProductViewer.vue b/django_airavata/static/common/js/components/DataProductViewer.vue
index 52f35f7..788ca79 100644
--- a/django_airavata/static/common/js/components/DataProductViewer.vue
+++ b/django_airavata/static/common/js/components/DataProductViewer.vue
@@ -1,8 +1,7 @@
<template>
<span v-if="downloadURL">
- <a :href="downloadURL" class="action-link">
- <i class="fa fa-download"></i>
+ <a :href="downloadURL" class="action-link" :target="linkTarget">
{{ filename }}
</a>
</span>
@@ -24,6 +23,10 @@ export default {
},
mimeType: {
type: String
+ },
+ openInNewWindow: {
+ type: Boolean,
+ default: false
}
},
computed: {
@@ -45,6 +48,9 @@ export default {
} else {
return this.dataProduct.downloadURL;
}
+ },
+ linkTarget() {
+ return this.openInNewWindow ? "_blank": "_self";
}
}
};