You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/05/28 13:22:33 UTC
[airavata-django-portal] 03/06: AIRAVATA-3032 Move tmp input file
uploads to data dir at launch
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit eeb471b5cbcb6189eec758425a961344d450b565
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Sun May 26 15:06:09 2019 -0400
AIRAVATA-3032 Move tmp input file uploads to data dir at launch
---
django_airavata/apps/api/data_products_helper.py | 16 +++++++++
django_airavata/apps/api/datastore.py | 26 +++++++++++---
django_airavata/apps/api/views.py | 43 ++++++++++++++++++++++--
3 files changed, 78 insertions(+), 7 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index eb7f0ad..b085e87 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -36,6 +36,22 @@ def save_input_file_upload(request, file):
return data_product
+def is_input_file_upload(request, data_product):
+ path = _get_replica_filepath(data_product)
+ rel_path = datastore.rel_path(request.user.username, path)
+ return os.path.dirname(rel_path) == TMP_INPUT_FILE_UPLOAD_DIR
+
+
+def move_input_file_upload(request, data_product, path):
+ source_path = _get_replica_filepath(data_product)
+ file_name = data_product.productName
+ target_path = os.path.join(path, file_name)
+ full_path = datastore.move(request.user.username, source_path, target_path)
+ _delete_data_product(request, source_path)
+ data_product = _save_data_product(request, full_path, name=file_name)
+ return data_product
+
+
def open(request, data_product):
"Return file object for replica if it exists in user storage."
path = _get_replica_filepath(data_product)
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index 124949f..2fc0965 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -4,6 +4,7 @@ import shutil
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation
+from django.core.files.move import file_move_safe
from django.core.files.storage import FileSystemStorage
experiment_data_storage = FileSystemStorage(
@@ -40,6 +41,13 @@ def save(username, path, file):
return input_file_fullpath
+def move(username, source_path, target_path):
+ source_full_path = path_(username, source_path)
+ target_full_path = path_(username, target_path)
+ file_move_safe(source_full_path, target_full_path)
+ return target_full_path
+
+
def create_user_dir(username, path):
user_data_storage = _user_data_storage(username)
if not user_data_storage.exists(path):
@@ -73,7 +81,6 @@ def delete_dir(username, path):
raise ObjectDoesNotExist("File path does not exist: {}".format(path))
-# TODO: update this to just return an available experiment directory name
def get_experiment_dir(
username,
project_name=None,
@@ -85,16 +92,22 @@ def get_experiment_dir(
experiment_dir_name = os.path.join(
user_experiment_data_storage.get_valid_name(project_name),
user_experiment_data_storage.get_valid_name(experiment_name))
+ # Since there may already be another experiment with the same name in
+ # this project, we need to check for available name
+ experiment_dir_name = user_experiment_data_storage.get_available_name(
+ experiment_dir_name)
experiment_dir = user_experiment_data_storage.path(experiment_dir_name)
else:
# path can be relative to the user's storage space or absolute (as long
# as it is still inside the user's storage space)
+ # if path is passed in, assumption is that it has already been created
user_experiment_data_storage = _user_data_storage(username)
experiment_dir = user_experiment_data_storage.path(path)
if not user_experiment_data_storage.exists(experiment_dir):
- os.makedirs(experiment_dir,
- mode=user_experiment_data_storage.directory_permissions_mode)
- # os.mkdir mode isn't always respected so need to chmod to be sure
+ os.makedirs(
+ experiment_dir,
+ mode=user_experiment_data_storage.directory_permissions_mode)
+ # os.makedirs mode isn't always respected so need to chmod to be sure
os.chmod(experiment_dir,
mode=user_experiment_data_storage.directory_permissions_mode)
return experiment_dir
@@ -137,6 +150,11 @@ def path(username, file_path):
return path_(username, file_path)
+def rel_path(username, file_path):
+ full_path = path_(username, file_path)
+ return os.path.relpath(full_path, path_(username, ""))
+
+
def path_(username, file_path):
user_data_storage = _user_data_storage(username)
return user_data_storage.path(file_path)
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 819b16b..4ec398e 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -179,7 +179,6 @@ class ExperimentViewSet(APIBackedViewSet):
experiment = serializer.save(
gatewayId=self.gateway_id,
userName=self.username)
- self._set_storage_id_and_data_dir(experiment)
experiment_id = self.request.airavata_client.createExperiment(
self.authz_token, self.gateway_id, experiment)
self._update_most_recent_project(experiment.projectId)
@@ -189,8 +188,6 @@ class ExperimentViewSet(APIBackedViewSet):
experiment = serializer.save(
gatewayId=self.gateway_id,
userName=self.username)
- # The project or exp name may have changed, so update the exp data dir
- # self._set_storage_id_and_data_dir(experiment)
self.request.airavata_client.updateExperiment(
self.authz_token, experiment.experimentId, experiment)
self._update_most_recent_project(experiment.projectId)
@@ -215,9 +212,49 @@ class ExperimentViewSet(APIBackedViewSet):
path=experiment.userConfigurationData.experimentDataDir)
experiment.userConfigurationData.experimentDataDir = exp_dir
+ def _move_tmp_input_file_uploads_to_data_dir(self, experiment):
+ exp_data_dir = experiment.userConfigurationData.experimentDataDir
+ for experiment_input in experiment.experimentInputs:
+ if experiment_input.type == DataType.URI:
+ experiment_input.value = self._move_if_tmp_input_file_upload(
+ experiment_input.value, exp_data_dir)
+ elif experiment_input.type == DataType.URI_COLLECTION:
+ data_product_uris = experiment_input.value.split(
+ ",") if experiment_input.value else []
+ moved_data_product_uris = []
+ for data_product_uri in data_product_uris:
+ moved_data_product_uris.append(
+ self._move_if_tmp_input_file_upload(data_product_uri,
+ exp_data_dir))
+ experiment_input.value = ",".join(moved_data_product_uris)
+
+ def _move_if_tmp_input_file_upload(
+ self, data_product_uri, experiment_data_dir):
+ """
+ Conditionally moves tmp input file to data dir and returns new dp URI.
+ """
+ data_product = self.request.airavata_client.getDataProduct(
+ self.authz_token, data_product_uri)
+ if data_products_helper.is_input_file_upload(
+ self.request, data_product):
+ moved_data_product = \
+ data_products_helper.move_input_file_upload(
+ self.request,
+ data_product,
+ experiment_data_dir)
+ return moved_data_product.productUri
+ else:
+ return data_product_uri
+
@detail_route(methods=['post'])
def launch(self, request, experiment_id=None):
try:
+ experiment = request.airavata_client.getExperiment(
+ self.authz_token, experiment_id)
+ self._set_storage_id_and_data_dir(experiment)
+ self._move_tmp_input_file_uploads_to_data_dir(experiment)
+ request.airavata_client.updateExperiment(
+ self.authz_token, experiment_id, experiment)
request.airavata_client.launchExperiment(
request.authz_token, experiment_id, self.gateway_id)
return Response({'success': True})