You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2020/09/17 13:43:50 UTC
[airavata-django-portal-sdk] branch
AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage updated
(2dd9556 -> 8c6b231)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git.
from 2dd9556 AIRAVATA-3346: AIRAVATA-3346 Implementing remote API mode for user_storage functions to support user storage view
new d3bf173 AIRAVATA-3346: AIRAVATA-3346 Fixing preservation of content type in uploads
new dacf233 AIRAVATA-3346: AIRAVATA-3346: AIRAVATA-3346 Implement remote save_input_file
new a705626 AIRAVATA-3346: AIRAVATA-3346 Prevent accidentally using local data store API when configured for remote data store API
new b5f8260 AIRAVATA-3346: AIRAVATA-3346 removing deprecated user_file_exists
new 8c6b231 AIRAVATA-3346: AIRAVATA-3346 Enable remote API for update_file_content
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
airavata_django_portal_sdk/user_storage.py | 100 +++++++++++++++++++++--------
1 file changed, 75 insertions(+), 25 deletions(-)
[airavata-django-portal-sdk] 01/05: AIRAVATA-3346: AIRAVATA-3346
Fixing preservation of content type in uploads
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit d3bf173e033a14aa48c2208808dd6ac924ee0ed1
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Sep 16 10:49:14 2020 -0400
AIRAVATA-3346: AIRAVATA-3346 Fixing preservation of content type in uploads
---
airavata_django_portal_sdk/user_storage.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py
index fcc761f..2570e86 100644
--- a/airavata_django_portal_sdk/user_storage.py
+++ b/airavata_django_portal_sdk/user_storage.py
@@ -33,9 +33,10 @@ def save(request, path, file, name=None, content_type=None):
if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
headers = {
'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ if name is None and hasattr(file, 'name'):
+ name = os.path.basename(file.name)
files = {
- # 'file': (name, file, content_type)
- 'file': file
+ 'file': (name, file, content_type) if content_type is not None else file,
}
r = requests.post(
f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/user-storage/~/{path}',
[airavata-django-portal-sdk] 05/05: AIRAVATA-3346: AIRAVATA-3346
Enable remote API for update_file_content
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 8c6b2312897e67e460bed85b2cb4a3d51fc40c7d
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Sep 16 14:52:20 2020 -0400
AIRAVATA-3346: AIRAVATA-3346 Enable remote API for update_file_content
---
airavata_django_portal_sdk/user_storage.py | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py
index 9c1cf57..9f793fe 100644
--- a/airavata_django_portal_sdk/user_storage.py
+++ b/airavata_django_portal_sdk/user_storage.py
@@ -254,10 +254,21 @@ def delete_user_file(request, path):
def update_file_content(request, path, fileContentText):
- full_path = _Datastore().path(request.user.username, path)
- with open(full_path, 'w') as f:
- myfile = File(f)
- myfile.write(fileContentText)
+ if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.put(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/user-storage/~/{path}',
+ headers=headers,
+ data={"fileContentText": fileContentText}
+ )
+ r.raise_for_status()
+ return
+ else:
+ full_path = _Datastore().path(request.user.username, path)
+ with open(full_path, 'w') as f:
+ myfile = File(f)
+ myfile.write(fileContentText)
def get_file(request, path):
[airavata-django-portal-sdk] 03/05: AIRAVATA-3346: AIRAVATA-3346
Prevent accidentally using local data store API when configured for remote
data store API
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit a705626931a09872291a10fe6e1a471c274f451b
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Sep 16 14:23:07 2020 -0400
AIRAVATA-3346: AIRAVATA-3346 Prevent accidentally using local data store API when configured for remote data store API
---
airavata_django_portal_sdk/user_storage.py | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py
index 3d66764..2a48bb8 100644
--- a/airavata_django_portal_sdk/user_storage.py
+++ b/airavata_django_portal_sdk/user_storage.py
@@ -118,6 +118,16 @@ def copy_input_file(request, data_product):
def is_input_file(request, data_product):
+ if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.get(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/data-products/',
+ headers=headers,
+ params={'product-uri': data_product.productUri})
+ r.raise_for_status()
+ data = r.json()
+ return data['isInputFileUpload']
# Check if file is one of user's files and in TMP_INPUT_FILE_UPLOAD_DIR
path = _get_replica_filepath(data_product)
if _Datastore().exists(request.user.username, path):
@@ -555,6 +565,15 @@ class _Datastore:
"""Internal datastore abstraction."""
def __init__(self, directory=None):
+ if getattr(
+ settings,
+ 'GATEWAY_DATA_STORE_REMOTE_API',
+ None) is not None:
+ raise Exception(
+ f"This Django portal instance is configured to connect to a "
+ f"remote data store via API (settings.GATEWAY_DATA_STORE_REMOTE_API="
+ f"{settings.GATEWAY_DATA_STORE_REMOTE_API}). This local "
+ f"Datastore instance is not available in remote data store mode.")
if directory:
self.directory = directory
else:
[airavata-django-portal-sdk] 02/05: AIRAVATA-3346: AIRAVATA-3346:
AIRAVATA-3346 Implement remote save_input_file
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit dacf2332b8265b389ea688067c3d7b3679714e37
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Sep 16 12:30:05 2020 -0400
AIRAVATA-3346: AIRAVATA-3346: AIRAVATA-3346 Implement remote save_input_file
---
airavata_django_portal_sdk/user_storage.py | 38 ++++++++++++++++++++++--------
1 file changed, 28 insertions(+), 10 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py
index 2570e86..3d66764 100644
--- a/airavata_django_portal_sdk/user_storage.py
+++ b/airavata_django_portal_sdk/user_storage.py
@@ -35,9 +35,8 @@ def save(request, path, file, name=None, content_type=None):
'Authorization': f'Bearer {request.authz_token.accessToken}'}
if name is None and hasattr(file, 'name'):
name = os.path.basename(file.name)
- files = {
- 'file': (name, file, content_type) if content_type is not None else file,
- }
+ files = {'file': (name, file, content_type)
+ if content_type is not None else file, }
r = requests.post(
f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/user-storage/~/{path}',
headers=headers,
@@ -77,13 +76,32 @@ def move_from_filepath(
def save_input_file(request, file, name=None, content_type=None):
"""Save input file in staging area for input files."""
- username = request.user.username
- file_name = name if name is not None else os.path.basename(file.name)
- full_path = _Datastore().save(username, TMP_INPUT_FILE_UPLOAD_DIR, file)
- data_product = _save_data_product(
- request, full_path, name=file_name, content_type=content_type
- )
- return data_product
+ if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ if name is None and hasattr(file, 'name'):
+ name = os.path.basename(file.name)
+ files = {'file': (name, file, content_type)
+ if content_type is not None else file, }
+ r = requests.post(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/upload',
+ headers=headers,
+ files=files
+ )
+ r.raise_for_status()
+ data = r.json()
+ product_uri = data['data-product']['productUri']
+ data_product = request.airavata_client.getDataProduct(
+ request.authz_token, product_uri)
+ return data_product
+ else:
+ username = request.user.username
+ file_name = name if name is not None else os.path.basename(file.name)
+ full_path = _Datastore().save(username, TMP_INPUT_FILE_UPLOAD_DIR, file)
+ data_product = _save_data_product(
+ request, full_path, name=file_name, content_type=content_type
+ )
+ return data_product
def copy_input_file(request, data_product):
[airavata-django-portal-sdk] 04/05: AIRAVATA-3346: AIRAVATA-3346
removing deprecated user_file_exists
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch AIRAVATA-3346-implement-remote-fs-abstraction-of-user-storage
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit b5f8260cd10869486a29bfe555a3fab76385a212
Author: Marcus Christie <ma...@iu.edu>
AuthorDate: Wed Sep 16 14:44:07 2020 -0400
AIRAVATA-3346: AIRAVATA-3346 removing deprecated user_file_exists
---
airavata_django_portal_sdk/user_storage.py | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage.py b/airavata_django_portal_sdk/user_storage.py
index 2a48bb8..9c1cf57 100644
--- a/airavata_django_portal_sdk/user_storage.py
+++ b/airavata_django_portal_sdk/user_storage.py
@@ -221,16 +221,6 @@ def dir_exists(request, path):
return _Datastore().dir_exists(request.user.username, path)
-def user_file_exists(request, path):
- """If file exists, return data product URI, else None."""
- if _Datastore().exists(request.user.username, path):
- full_path = _Datastore().path(request.user.username, path)
- data_product_uri = _get_data_product_uri(request, full_path)
- return data_product_uri
- else:
- return None
-
-
def delete_dir(request, path):
"""Delete path in user's data store, if it exists."""
if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
@@ -249,6 +239,17 @@ def delete_dir(request, path):
def delete_user_file(request, path):
"""Delete file in user's data store, if it exists."""
+ if getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None:
+ headers = {
+ 'Authorization': f'Bearer {request.authz_token.accessToken}'}
+ r = requests.delete(
+ f'{settings.GATEWAY_DATA_STORE_REMOTE_API}/user-storage/~/{path}',
+ headers=headers,
+ )
+ if r.status_code == 404:
+ raise ObjectDoesNotExist(f"File path does not exist {path}")
+ r.raise_for_status()
+ return
return _Datastore().delete(request.user.username, path)