You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/07/09 13:50:20 UTC
[airavata-django-portal-sdk] branch master updated (b33bcf0 ->
46c15e6)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git.
from b33bcf0 AIRAVATA-3431 Copy fix to the SDK download_file view
new f532edc AIRAVATA-3485 Accept experiment data dir relative paths in user_storage functions
new a5a8369 AIRAVATA-3485 Also adding experiment_id to listdir and get_file_metadata, for testing purposes mostly
new f5f04d1 AIRAVATA-3485 Support for experiment-id in remote API
new 46c15e6 AIRAVATA-3485 Bump version to 1.1.dev.4
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 1 +
airavata_django_portal_sdk/user_storage/api.py | 88 +++++++++++++++++++-------
setup.py | 2 +-
3 files changed, 66 insertions(+), 25 deletions(-)
[airavata-django-portal-sdk] 01/04: AIRAVATA-3485 Accept experiment
data dir relative paths in user_storage functions
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit f532edc20663eed5522ed80e0c7bc2ac05ba31ec
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Thu Jul 8 09:58:55 2021 -0400
AIRAVATA-3485 Accept experiment data dir relative paths in user_storage functions
---
airavata_django_portal_sdk/user_storage/api.py | 70 +++++++++++++++++++-------
1 file changed, 53 insertions(+), 17 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index b015af7..3de2655 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -74,13 +74,14 @@ def get_user_storage_provider(request, owner_username=None, storage_resource_id=
return instance
-def save(request, path, file, name=None, content_type=None, storage_resource_id=None):
+def save(request, path, file, name=None, content_type=None, storage_resource_id=None, experiment_id=None):
"Save file in path in the user's storage and return DataProduct."
if _is_remote_api():
if name is None and hasattr(file, 'name'):
name = os.path.basename(file.name)
files = {'file': (name, file, content_type)
if content_type is not None else file, }
+ # TODO: add experiment data directory relative paths support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -91,8 +92,9 @@ def save(request, path, file, name=None, content_type=None, storage_resource_id=
data_product = request.airavata_client.getDataProduct(
request.authz_token, product_uri)
return data_product
+ final_path = _get_final_path(request, path, experiment_id)
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- storage_resource_id, resource_path = backend.save(path, file, name=name, content_type=content_type)
+ storage_resource_id, resource_path = backend.save(final_path, file, name=name, content_type=content_type)
data_product = _save_data_product(
request, resource_path, storage_resource_id, name=name, content_type=content_type, backend=backend
)
@@ -270,9 +272,10 @@ def exists(request, data_product=None, data_product_uri=None):
return backend.exists(path)
-def dir_exists(request, path, storage_resource_id=None):
+def dir_exists(request, path, storage_resource_id=None, experiment_id=None):
"Return True if path exists in user's data store."
if _is_remote_api():
+ # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -282,13 +285,16 @@ def dir_exists(request, path, storage_resource_id=None):
resp.raise_for_status()
return resp.json()['isDir']
else:
- backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- return backend.is_dir(path)
+ final_path, owner_username = _get_final_path_and_owner_username(request, path, experiment_id)
+ backend = get_user_storage_provider(
+ request, storage_resource_id=storage_resource_id, owner_username=owner_username)
+ return backend.is_dir(final_path)
-def user_file_exists(request, path, storage_resource_id=None):
+def user_file_exists(request, path, storage_resource_id=None, experiment_id=None):
"""If file exists, return data product URI, else None."""
if _is_remote_api():
+ # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -297,9 +303,11 @@ def user_file_exists(request, path, storage_resource_id=None):
return None
resp.raise_for_status()
return resp.json()['files'][0]['dataProductURI']
- backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- if backend.is_file(path):
- _, files = backend.get_metadata(path)
+ final_path, owner_username = _get_final_path_and_owner_username(request, path, experiment_id)
+ backend = get_user_storage_provider(
+ request, storage_resource_id=storage_resource_id, owner_username=owner_username)
+ if backend.is_file(final_path):
+ _, files = backend.get_metadata(final_path)
full_path = files[0]['resource_path']
data_product_uri = _get_data_product_uri(request, full_path, backend.resource_id)
return data_product_uri
@@ -307,9 +315,10 @@ def user_file_exists(request, path, storage_resource_id=None):
return None
-def delete_dir(request, path, storage_resource_id=None):
+def delete_dir(request, path, storage_resource_id=None, experiment_id=None):
"""Delete path in user's data store, if it exists."""
if _is_remote_api():
+ # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -319,12 +328,14 @@ def delete_dir(request, path, storage_resource_id=None):
resp.raise_for_status()
return
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- backend.delete(path)
+ final_path = _get_final_path(request, path, experiment_id)
+ backend.delete(final_path)
-def delete_user_file(request, path, storage_resource_id=None):
+def delete_user_file(request, path, storage_resource_id=None, experiment_id=None):
"""Delete file in user's data store, if it exists."""
if _is_remote_api():
+ # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -334,7 +345,8 @@ def delete_user_file(request, path, storage_resource_id=None):
resp.raise_for_status()
return
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- backend.delete(path)
+ final_path = _get_final_path(request, path, experiment_id)
+ backend.delete(final_path)
def update_file_content(request, path, fileContentText, storage_resource_id=None):
@@ -620,7 +632,7 @@ def get_experiment_dir(request, project_name=None, experiment_name=None, path=No
return resource_path
-def create_user_dir(request, path="", dir_names=(), create_unique=False, storage_resource_id=None):
+def create_user_dir(request, path="", dir_names=(), create_unique=False, storage_resource_id=None, experiment_id=None):
"""
Creates a directory, and intermediate directories if given, at the given
path in the user's storage. `dir_names` should be either a list or tuple of
@@ -637,6 +649,7 @@ def create_user_dir(request, path="", dir_names=(), create_unique=False, storage
"""
if _is_remote_api():
logger.debug(f"path={path}")
+ # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
@@ -649,14 +662,15 @@ def create_user_dir(request, path="", dir_names=(), create_unique=False, storage
return storage_resource_id, path
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
# For backwards compatibility, manufacture the dir_names array as needed
+ final_path = _get_final_path(request, path, experiment_id)
if len(dir_names) == 0:
dir_names = []
- while not backend.exists(path):
- path, dir_name = os.path.split(path)
+ while not backend.exists(final_path):
+ final_path, dir_name = os.path.split(final_path)
if dir_name == '':
break
dir_names.insert(0, dir_name)
- storage_resource_id, resource_path = backend.create_dirs(path, dir_names=dir_names, create_unique=create_unique)
+ storage_resource_id, resource_path = backend.create_dirs(final_path, dir_names=dir_names, create_unique=create_unique)
return storage_resource_id, resource_path
@@ -856,6 +870,28 @@ def _get_replica_resource_id_and_filepath(data_product):
return None, None
+def _get_final_path(request, path, experiment_id):
+ "If experiment_id is given, join the path to it's data directory"
+ final_path = path
+ if experiment_id is not None:
+ experiment = request.airavata_client.getExperiment(
+ request.authz_token, experiment_id)
+ exp_data_dir = experiment.userConfigurationData.experimentDataDir
+ final_path = os.path.join(exp_data_dir, path)
+ return final_path
+
+
+def _get_final_path_and_owner_username(request, path, experiment_id):
+ "If experiment_id is given, join the path to it's data directory and also return owner username"
+ final_path = path
+ if experiment_id is not None:
+ experiment = request.airavata_client.getExperiment(
+ request.authz_token, experiment_id)
+ exp_data_dir = experiment.userConfigurationData.experimentDataDir
+ return os.path.join(exp_data_dir, path), experiment.userName
+ return final_path, None
+
+
def _is_remote_api():
return getattr(settings, 'GATEWAY_DATA_STORE_REMOTE_API', None) is not None
[airavata-django-portal-sdk] 04/04: AIRAVATA-3485 Bump version to
1.1.dev.4
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit 46c15e603ace907da275c55c35e50b188b12527d
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jul 9 09:48:55 2021 -0400
AIRAVATA-3485 Bump version to 1.1.dev.4
---
README.md | 1 +
setup.py | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index b6021bd..4b989e3 100644
--- a/README.md
+++ b/README.md
@@ -91,6 +91,7 @@ isort .
```
cd /tmp/
git clone /path/to/airavata-django-portal-sdk/ -b VERSION
+ cd airavata-django-portal-sdk
python3 -m venv venv
source venv/bin/activate
python3 -m pip install --upgrade build
diff --git a/setup.py b/setup.py
index 638d365..15e9836 100644
--- a/setup.py
+++ b/setup.py
@@ -10,7 +10,7 @@ def read(fname):
setup(
name="airavata-django-portal-sdk",
- version="1.1.dev3",
+ version="1.1.dev4",
url="https://github.com/apache/airavata-django-portal-sdk",
author="Apache Software Foundation",
author_email="dev@airavata.apache.org",
[airavata-django-portal-sdk] 02/04: AIRAVATA-3485 Also adding
experiment_id to listdir and get_file_metadata, for testing purposes mostly
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit a5a8369d6e7012132704aca560cdd650d30b6e49
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jul 9 09:31:58 2021 -0400
AIRAVATA-3485 Also adding experiment_id to listdir and get_file_metadata, for testing purposes mostly
---
airavata_django_portal_sdk/user_storage/api.py | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index 3de2655..ebed712 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -379,7 +379,7 @@ def update_data_product_content(request, data_product=None, fileContentText="",
update_file_content(request, path, fileContentText, storage_resource_id=storage_resource_id)
-def get_file_metadata(request, path, storage_resource_id=None):
+def get_file_metadata(request, path, storage_resource_id=None, experiment_id=None):
if _is_remote_api():
resp = _call_remote_api(request,
"/user-storage/~/{path}",
@@ -397,8 +397,9 @@ def get_file_metadata(request, path, storage_resource_id=None):
return file
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- if backend.is_file(path):
- _, files = backend.get_metadata(path)
+ final_path = _get_final_path(request, path, experiment_id)
+ if backend.is_file(final_path):
+ _, files = backend.get_metadata(final_path)
file = files[0]
data_product_uri = _get_data_product_uri(request, file['resource_path'],
storage_resource_id=backend.resource_id)
@@ -461,9 +462,9 @@ def get_data_product_metadata(request, data_product=None, data_product_uri=None)
raise ObjectDoesNotExist("File does not exist at that path.")
-def get_file(request, path, storage_resource_id=None):
+def get_file(request, path, storage_resource_id=None, experiment_id=None):
warnings.warn("Use 'get_file_metadata' instead.", DeprecationWarning)
- return get_file_metadata(request, path, storage_resource_id)
+ return get_file_metadata(request, path, storage_resource_id, experiment_id=experiment_id)
def delete(request, data_product=None, data_product_uri=None):
@@ -495,7 +496,7 @@ def delete(request, data_product=None, data_product_uri=None):
raise
-def listdir(request, path, storage_resource_id=None):
+def listdir(request, path, storage_resource_id=None, experiment_id=None):
"""Return a tuple of two lists, one for directories, the second for files."""
if _is_remote_api():
@@ -517,7 +518,8 @@ def listdir(request, path, storage_resource_id=None):
return data['directories'], data['files']
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- directories, files = backend.get_metadata(path)
+ final_path = _get_final_path(request, path, experiment_id)
+ directories, files = backend.get_metadata(final_path)
# Mark the TMP_INPUT_FILE_UPLOAD_DIR directory as hidden in the UI
for directory in directories:
directory['hidden'] = directory['path'] == TMP_INPUT_FILE_UPLOAD_DIR
[airavata-django-portal-sdk] 03/04: AIRAVATA-3485 Support for
experiment-id in remote API
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
commit f5f04d107af980901f3ab56dae8d6260bd4266cb
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Fri Jul 9 09:45:09 2021 -0400
AIRAVATA-3485 Support for experiment-id in remote API
---
airavata_django_portal_sdk/user_storage/api.py | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index ebed712..6ee7d69 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -81,10 +81,10 @@ def save(request, path, file, name=None, content_type=None, storage_resource_id=
name = os.path.basename(file.name)
files = {'file': (name, file, content_type)
if content_type is not None else file, }
- # TODO: add experiment data directory relative paths support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
method="post",
files=files)
data = resp.json()
@@ -275,10 +275,10 @@ def exists(request, data_product=None, data_product_uri=None):
def dir_exists(request, path, storage_resource_id=None, experiment_id=None):
"Return True if path exists in user's data store."
if _is_remote_api():
- # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
raise_for_status=False)
if resp.status_code == HTTPStatus.NOT_FOUND:
return False
@@ -294,10 +294,10 @@ def dir_exists(request, path, storage_resource_id=None, experiment_id=None):
def user_file_exists(request, path, storage_resource_id=None, experiment_id=None):
"""If file exists, return data product URI, else None."""
if _is_remote_api():
- # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
raise_for_status=False)
if resp.status_code == HTTPStatus.NOT_FOUND or resp.json()['isDir']:
return None
@@ -318,10 +318,10 @@ def user_file_exists(request, path, storage_resource_id=None, experiment_id=None
def delete_dir(request, path, storage_resource_id=None, experiment_id=None):
"""Delete path in user's data store, if it exists."""
if _is_remote_api():
- # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
method="delete",
raise_for_status=False)
_raise_404(resp, f"File path does not exist {path}")
@@ -335,10 +335,10 @@ def delete_dir(request, path, storage_resource_id=None, experiment_id=None):
def delete_user_file(request, path, storage_resource_id=None, experiment_id=None):
"""Delete file in user's data store, if it exists."""
if _is_remote_api():
- # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
method="delete",
raise_for_status=False)
_raise_404(resp, f"File path does not exist {path}")
@@ -384,6 +384,7 @@ def get_file_metadata(request, path, storage_resource_id=None, experiment_id=Non
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
raise_for_status=False
)
_raise_404(resp, "User storage file path does not exist")
@@ -503,6 +504,7 @@ def listdir(request, path, storage_resource_id=None, experiment_id=None):
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
)
data = resp.json()
for directory in data['directories']:
@@ -651,10 +653,10 @@ def create_user_dir(request, path="", dir_names=(), create_unique=False, storage
"""
if _is_remote_api():
logger.debug(f"path={path}")
- # TODO: add experiment data directory relative path support to remote API
resp = _call_remote_api(request,
"/user-storage/~/{path}",
path_params={"path": path},
+ params={"experiment-id": experiment_id},
method="post")
json = resp.json()
# 'path' is a new response attribute, for backwards compatibility check if it exists first