You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/05/15 02:13:02 UTC
[airavata-django-portal] 02/02: WIP
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit b71f79567d7e0645f2d873c85949ea0e492a0413
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue May 14 22:12:31 2019 -0400
WIP
---
django_airavata/apps/api/data_products_helper.py | 69 +++++++++++----
django_airavata/apps/api/datastore.py | 106 ++++++-----------------
django_airavata/apps/api/serializers.py | 6 +-
django_airavata/apps/api/views.py | 36 ++------
4 files changed, 86 insertions(+), 131 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 65b8024..e301d78 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -1,4 +1,5 @@
import os
+from urllib.parse import urlparse
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
@@ -15,21 +16,23 @@ from . import datastore, models
def save(request, path, file):
- # return data_product
- # TODO
- pass
+ "Save file in path in the user's storage."
+ username = request.user.username
+ full_path = datastore.save(username, path, file)
+ data_product = _save_data_product(request, full_path)
+ return data_product
def open(request, data_product):
- # return file object
- # TODO
- pass
+ "Return file object for replica if it exists in user storage."
+ path = _get_replica_filepath(data_product)
+ return datastore.open(request.user.username, path)
def exists(request, data_product):
- # return boolean
- # TODO
- pass
+ "Return True if replica for data_product exists in user storage."
+ path = _get_replica_filepath(data_product)
+ return datastore.exists(request.user.username, path)
def delete(request, data_product):
@@ -58,6 +61,18 @@ def listdir(request, path):
raise ObjectDoesNotExist("User storage path does not exist")
+def get_experiment_dir(request,
+ project_name=None,
+ experiment_name=None,
+ path=None):
+ return datastore.get_experiment_dir(
+ request.user.username, project_name, experiment_name, path)
+
+
+def create_user_dir(request, path):
+ return datastore.create_user_dir(request.user.username, path)
+
+
def _get_data_product_uri(request, full_path):
user_file = models.User_Files.objects.filter(
@@ -65,17 +80,25 @@ def _get_data_product_uri(request, full_path):
if user_file.exists():
product_uri = user_file[0].file_dpu
else:
- data_product = _create_data_product(request.user.username, full_path)
- product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- user_file_instance = models.User_Files(
- username=request.user.username,
- file_path=full_path,
- file_dpu=product_uri)
- user_file_instance.save()
+ data_product = _save_data_product(request, full_path)
+ product_uri = data_product.productUri
return product_uri
+def _save_data_product(request, full_path):
+ "Create, register and record in DB a data product for full_path."
+ data_product = _create_data_product(request.user.username, full_path)
+ product_uri = request.airavata_client.registerDataProduct(
+ request.authz_token, data_product)
+ data_product.productUri = product_uri
+ user_file_instance = models.User_Files(
+ username=request.user.username,
+ file_path=full_path,
+ file_dpu=product_uri)
+ user_file_instance.save()
+ return data_product
+
+
def _create_data_product(username, full_path):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
@@ -97,3 +120,15 @@ def _create_data_product(username, full_path):
full_path)
data_product.replicaLocations = [data_replica_location]
return data_product
+
+
+def _get_replica_filepath(data_product):
+ replica_filepaths = [rep.filePath
+ for rep in data_product.replicaLocations
+ if rep.replicaLocationCategory ==
+ ReplicaLocationCategory.GATEWAY_DATA_STORE]
+ replica_filepath = (replica_filepaths[0]
+ if len(replica_filepaths) > 0 else None)
+ if replica_filepath:
+ return urlparse(replica_filepath).path
+ return None
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index 2b5f9b5..387bcb1 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,106 +19,49 @@ experiment_data_storage = FileSystemStorage(
logger = logging.getLogger(__name__)
-# TODO: exists(username, path)
-def exists(data_product):
+def exists(username, path):
"""Check if replica for data product exists in this data store."""
- filepath = _get_replica_filepath(data_product)
try:
- return experiment_data_storage.exists(filepath) if filepath else False
+ return _user_data_storage(username).exists(path)
except SuspiciousFileOperation as e:
- logger.warning("Unable to find file at {} for data product uri {}"
- .format(filepath, data_product.productUri))
+ logger.warning("Invalid path for user {}: {}".format(username, str(e)))
return False
-# TODO: open(username, path)
-def open(data_product):
- """Open replica for data product if it exists in this data store."""
- if exists(data_product):
- filepath = _get_replica_filepath(data_product)
- return experiment_data_storage.open(filepath)
+def open(username, path):
+ """Open path for user if it exists in this data store."""
+ if exists(username, path):
+ return _user_data_storage(username).open(path)
else:
- raise ObjectDoesNotExist("Replica file does not exist")
-
-
-def save_user(username, file):
- """Save file to username/project name/experiment_name in data store."""
- user_dir = os.path.join(
- experiment_data_storage.get_valid_name(username),
- 'MyFiles')
- # file.name may be full path, so get just the name of the file
- file_name = os.path.basename(file.name)
- file_path = os.path.join(
- user_dir,
- experiment_data_storage.get_valid_name(file_name))
- input_file_name = experiment_data_storage.save(file_path, file)
- input_file_fullpath = experiment_data_storage.path(input_file_name)
- # Create DataProductModel instance with DataReplicaLocationModel
- data_product = DataProductModel()
- data_product.gatewayId = settings.GATEWAY_ID
- data_product.ownerName = username
- data_product.productName = file_name
- data_product.dataProductType = DataProductType.FILE
- data_replica_location = DataReplicaLocationModel()
- data_replica_location.storageResourceId = \
- settings.GATEWAY_DATA_STORE_RESOURCE_ID
- data_replica_location.replicaName = \
- "{} gateway data store copy".format(file_name)
- data_replica_location.replicaLocationCategory = \
- ReplicaLocationCategory.GATEWAY_DATA_STORE
- data_replica_location.replicaPersistentType = \
- ReplicaPersistentType.TRANSIENT
- data_replica_location.filePath = \
- "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
- input_file_fullpath)
- data_product.replicaLocations = [data_replica_location]
- return data_product
+ raise ObjectDoesNotExist("File path does not exist: {}".format(path))
-# TODO: save(username, path, file)
-def save(username, project_name, experiment_name, file):
+def save(username, path, file):
"""Save file to username/project name/experiment_name in data store."""
- exp_dir = os.path.join(
- _user_dir_name(username),
- experiment_data_storage.get_valid_name(project_name),
- experiment_data_storage.get_valid_name(experiment_name))
# file.name may be full path, so get just the name of the file
file_name = os.path.basename(file.name)
file_path = os.path.join(
- exp_dir,
- experiment_data_storage.get_valid_name(file_name))
- input_file_name = experiment_data_storage.save(file_path, file)
- input_file_fullpath = experiment_data_storage.path(input_file_name)
- # Create DataProductModel instance with DataReplicaLocationModel
- data_product = _create_data_product(username, input_file_fullpath)
- return data_product
-
-
-def save_user_file(username, path, file):
- experiment_data_storage.save(os.path.join(
- _user_dir_name(username),
- experiment_data_storage.get_valid_name(path),
- experiment_data_storage.get_valid_name(file.name)
- ), file)
+ path, experiment_data_storage.get_valid_name(file_name))
+ user_data_storage = _user_data_storage(username)
+ input_file_name = user_data_storage.save(file_path, file)
+ input_file_fullpath = user_data_storage.path(input_file_name)
+ return input_file_fullpath
-def create_user_dir(username, path, dir_name):
- user_dir = os.path.join(
- _user_dir_name(username),
- path,
- experiment_data_storage.get_valid_name(dir_name))
- if not experiment_data_storage.exists(user_dir):
- os.mkdir(experiment_data_storage.path(user_dir))
+def create_user_dir(username, path):
+ user_data_storage = _user_data_storage(username)
+ user_dir = user_data_storage.get_valid_name(path)
+ if not user_data_storage.exists(user_dir):
+ os.mkdir(user_data_storage.path(user_dir))
else:
raise Exception(
- "Directory {} already exists at that path".format(dir_name))
+ "Directory {} already exists".format(path))
-# TODO: copy(username, source_path, target_path)
-def copy(username, project_name, experiment_name, data_product):
- """Copy a data product into username/project_name/experiment_name dir."""
- f = open(data_product)
- return save(username, project_name, experiment_name, f)
+def copy(username, source_path, target_path):
+ """Copy a user file into target_path dir."""
+ f = open(username, source_path)
+ return save(username, target_path, f)
# TODO: delete(username, path)
@@ -138,6 +81,7 @@ def delete(data_product):
raise ObjectDoesNotExist("Replica file does not exist")
+# TODO: update this to just return an available experiment directory name
def get_experiment_dir(
username,
project_name=None,
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 86e235e..9c9fa08 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -52,7 +52,7 @@ from airavata.model.status.ttypes import ExperimentStatus
from airavata.model.user.ttypes import UserProfile
from airavata.model.workspace.ttypes import Project
-from . import datastore, models, thrift_utils
+from . import data_products_helper, datastore, models, thrift_utils
log = logging.getLogger(__name__)
@@ -435,8 +435,8 @@ class DataProductSerializer(
def get_downloadURL(self, data_product):
"""Getter for downloadURL field."""
- if datastore.exists(data_product):
- request = self.context['request']
+ request = self.context['request']
+ if data_products_helper.exists(request, data_product):
return (request.build_absolute_uri(
reverse('django_airavata_api:download_file')) +
'?' +
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 21c7b6c..a7572a8 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -865,11 +865,11 @@ def upload_input_file(request):
request.authz_token, project_id)
exp_name = request.POST['experiment-name']
input_file = request.FILES['file']
- data_product = datastore.save(username, project.name, exp_name,
- input_file)
- data_product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- data_product.productUri = data_product_uri
+ # TODO: experiment_data_dir should be passed in
+ experiment_data_dir = data_products_helper.get_experiment_dir(
+ request, project_name=project.name, experiment_name=exp_name)
+ data_product = data_products_helper.save(request, experiment_data_dir,
+ input_file)
serializer = serializers.DataProductSerializer(
data_product, context={'request': request})
return JsonResponse({'uploaded': True,
@@ -894,7 +894,7 @@ def download_file(request):
.format(data_product_uri), exc_info=True)
raise Http404("data product does not exist") from e
try:
- data_file = datastore.open(data_product)
+ data_file = data_products_helper.open(request, data_product)
response = FileResponse(data_file,
content_type="application/octet-stream")
file_name = os.path.basename(data_file.name)
@@ -1383,30 +1383,6 @@ class UserStoragePathView(APIView):
context={'request': request})
return Response(serializer.data)
- def post(self, request, path="/", format=None):
- # TODO: this needs to be fixed or rethought
- username = request.user.username
- user_storage_path = path
- if user_storage_path.startswith("/"):
- user_storage_path = "." + user_storage_path
- serializer = self.serializer_class(
- data=request.data, context={
- 'request': request})
- serializer.is_valid(raise_exception=True)
- if serializer.validated_data['type'] == 'file':
- upload_file = request.FILES['file']
- datastore.save_user_file(username, user_storage_path, upload_file)
- elif serializer.validated_data['type'] == 'dir':
- datastore.create_user_dir(
- username, user_storage_path, serializer.validated_data['name'])
-
- # TODO return representation of created item
- listing = datastore.list_user_dir(
- request.user.username, user_storage_path)
- serializer = self.serializer_class(
- listing, many=True, context={'request': request})
- return Response(serializer.data)
-
class WorkspacePreferencesView(APIView):
serializer_class = serializers.WorkspacePreferencesSerializer