You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/05/15 02:13:00 UTC
[airavata-django-portal] branch airavata-3016 created (now b71f795)
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a change to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git.
at b71f795 WIP
This branch includes the following new commits:
new 8808990 AIRAVATA-3033 REST API for browsing, downloading user storage
new b71f795 WIP
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airavata-django-portal] 02/02: WIP
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit b71f79567d7e0645f2d873c85949ea0e492a0413
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue May 14 22:12:31 2019 -0400
WIP
---
django_airavata/apps/api/data_products_helper.py | 69 +++++++++++----
django_airavata/apps/api/datastore.py | 106 ++++++-----------------
django_airavata/apps/api/serializers.py | 6 +-
django_airavata/apps/api/views.py | 36 ++------
4 files changed, 86 insertions(+), 131 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
index 65b8024..e301d78 100644
--- a/django_airavata/apps/api/data_products_helper.py
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -1,4 +1,5 @@
import os
+from urllib.parse import urlparse
from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
@@ -15,21 +16,23 @@ from . import datastore, models
def save(request, path, file):
- # return data_product
- # TODO
- pass
+ "Save file in path in the user's storage."
+ username = request.user.username
+ full_path = datastore.save(username, path, file)
+ data_product = _save_data_product(request, full_path)
+ return data_product
def open(request, data_product):
- # return file object
- # TODO
- pass
+ "Return file object for replica if it exists in user storage."
+ path = _get_replica_filepath(data_product)
+ return datastore.open(request.user.username, path)
def exists(request, data_product):
- # return boolean
- # TODO
- pass
+ "Return True if replica for data_product exists in user storage."
+ path = _get_replica_filepath(data_product)
+ return datastore.exists(request.user.username, path)
def delete(request, data_product):
@@ -58,6 +61,18 @@ def listdir(request, path):
raise ObjectDoesNotExist("User storage path does not exist")
+def get_experiment_dir(request,
+ project_name=None,
+ experiment_name=None,
+ path=None):
+ return datastore.get_experiment_dir(
+ request.user.username, project_name, experiment_name, path)
+
+
+def create_user_dir(request, path):
+ return datastore.create_user_dir(request.user.username, path)
+
+
def _get_data_product_uri(request, full_path):
user_file = models.User_Files.objects.filter(
@@ -65,17 +80,25 @@ def _get_data_product_uri(request, full_path):
if user_file.exists():
product_uri = user_file[0].file_dpu
else:
- data_product = _create_data_product(request.user.username, full_path)
- product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- user_file_instance = models.User_Files(
- username=request.user.username,
- file_path=full_path,
- file_dpu=product_uri)
- user_file_instance.save()
+ data_product = _save_data_product(request, full_path)
+ product_uri = data_product.productUri
return product_uri
+def _save_data_product(request, full_path):
+ "Create, register and record in DB a data product for full_path."
+ data_product = _create_data_product(request.user.username, full_path)
+ product_uri = request.airavata_client.registerDataProduct(
+ request.authz_token, data_product)
+ data_product.productUri = product_uri
+ user_file_instance = models.User_Files(
+ username=request.user.username,
+ file_path=full_path,
+ file_dpu=product_uri)
+ user_file_instance.save()
+ return data_product
+
+
def _create_data_product(username, full_path):
data_product = DataProductModel()
data_product.gatewayId = settings.GATEWAY_ID
@@ -97,3 +120,15 @@ def _create_data_product(username, full_path):
full_path)
data_product.replicaLocations = [data_replica_location]
return data_product
+
+
+def _get_replica_filepath(data_product):
+ replica_filepaths = [rep.filePath
+ for rep in data_product.replicaLocations
+ if rep.replicaLocationCategory ==
+ ReplicaLocationCategory.GATEWAY_DATA_STORE]
+ replica_filepath = (replica_filepaths[0]
+ if len(replica_filepaths) > 0 else None)
+ if replica_filepath:
+ return urlparse(replica_filepath).path
+ return None
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index 2b5f9b5..387bcb1 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,106 +19,49 @@ experiment_data_storage = FileSystemStorage(
logger = logging.getLogger(__name__)
-# TODO: exists(username, path)
-def exists(data_product):
+def exists(username, path):
"""Check if replica for data product exists in this data store."""
- filepath = _get_replica_filepath(data_product)
try:
- return experiment_data_storage.exists(filepath) if filepath else False
+ return _user_data_storage(username).exists(path)
except SuspiciousFileOperation as e:
- logger.warning("Unable to find file at {} for data product uri {}"
- .format(filepath, data_product.productUri))
+ logger.warning("Invalid path for user {}: {}".format(username, str(e)))
return False
-# TODO: open(username, path)
-def open(data_product):
- """Open replica for data product if it exists in this data store."""
- if exists(data_product):
- filepath = _get_replica_filepath(data_product)
- return experiment_data_storage.open(filepath)
+def open(username, path):
+ """Open path for user if it exists in this data store."""
+ if exists(username, path):
+ return _user_data_storage(username).open(path)
else:
- raise ObjectDoesNotExist("Replica file does not exist")
-
-
-def save_user(username, file):
- """Save file to username/project name/experiment_name in data store."""
- user_dir = os.path.join(
- experiment_data_storage.get_valid_name(username),
- 'MyFiles')
- # file.name may be full path, so get just the name of the file
- file_name = os.path.basename(file.name)
- file_path = os.path.join(
- user_dir,
- experiment_data_storage.get_valid_name(file_name))
- input_file_name = experiment_data_storage.save(file_path, file)
- input_file_fullpath = experiment_data_storage.path(input_file_name)
- # Create DataProductModel instance with DataReplicaLocationModel
- data_product = DataProductModel()
- data_product.gatewayId = settings.GATEWAY_ID
- data_product.ownerName = username
- data_product.productName = file_name
- data_product.dataProductType = DataProductType.FILE
- data_replica_location = DataReplicaLocationModel()
- data_replica_location.storageResourceId = \
- settings.GATEWAY_DATA_STORE_RESOURCE_ID
- data_replica_location.replicaName = \
- "{} gateway data store copy".format(file_name)
- data_replica_location.replicaLocationCategory = \
- ReplicaLocationCategory.GATEWAY_DATA_STORE
- data_replica_location.replicaPersistentType = \
- ReplicaPersistentType.TRANSIENT
- data_replica_location.filePath = \
- "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
- input_file_fullpath)
- data_product.replicaLocations = [data_replica_location]
- return data_product
+ raise ObjectDoesNotExist("File path does not exist: {}".format(path))
-# TODO: save(username, path, file)
-def save(username, project_name, experiment_name, file):
+def save(username, path, file):
"""Save file to username/project name/experiment_name in data store."""
- exp_dir = os.path.join(
- _user_dir_name(username),
- experiment_data_storage.get_valid_name(project_name),
- experiment_data_storage.get_valid_name(experiment_name))
# file.name may be full path, so get just the name of the file
file_name = os.path.basename(file.name)
file_path = os.path.join(
- exp_dir,
- experiment_data_storage.get_valid_name(file_name))
- input_file_name = experiment_data_storage.save(file_path, file)
- input_file_fullpath = experiment_data_storage.path(input_file_name)
- # Create DataProductModel instance with DataReplicaLocationModel
- data_product = _create_data_product(username, input_file_fullpath)
- return data_product
-
-
-def save_user_file(username, path, file):
- experiment_data_storage.save(os.path.join(
- _user_dir_name(username),
- experiment_data_storage.get_valid_name(path),
- experiment_data_storage.get_valid_name(file.name)
- ), file)
+ path, experiment_data_storage.get_valid_name(file_name))
+ user_data_storage = _user_data_storage(username)
+ input_file_name = user_data_storage.save(file_path, file)
+ input_file_fullpath = user_data_storage.path(input_file_name)
+ return input_file_fullpath
-def create_user_dir(username, path, dir_name):
- user_dir = os.path.join(
- _user_dir_name(username),
- path,
- experiment_data_storage.get_valid_name(dir_name))
- if not experiment_data_storage.exists(user_dir):
- os.mkdir(experiment_data_storage.path(user_dir))
+def create_user_dir(username, path):
+ user_data_storage = _user_data_storage(username)
+ user_dir = user_data_storage.get_valid_name(path)
+ if not user_data_storage.exists(user_dir):
+ os.mkdir(user_data_storage.path(user_dir))
else:
raise Exception(
- "Directory {} already exists at that path".format(dir_name))
+ "Directory {} already exists".format(path))
-# TODO: copy(username, source_path, target_path)
-def copy(username, project_name, experiment_name, data_product):
- """Copy a data product into username/project_name/experiment_name dir."""
- f = open(data_product)
- return save(username, project_name, experiment_name, f)
+def copy(username, source_path, target_path):
+ """Copy a user file into target_path dir."""
+ f = open(username, source_path)
+ return save(username, target_path, f)
# TODO: delete(username, path)
@@ -138,6 +81,7 @@ def delete(data_product):
raise ObjectDoesNotExist("Replica file does not exist")
+# TODO: update this to just return an available experiment directory name
def get_experiment_dir(
username,
project_name=None,
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 86e235e..9c9fa08 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -52,7 +52,7 @@ from airavata.model.status.ttypes import ExperimentStatus
from airavata.model.user.ttypes import UserProfile
from airavata.model.workspace.ttypes import Project
-from . import datastore, models, thrift_utils
+from . import data_products_helper, datastore, models, thrift_utils
log = logging.getLogger(__name__)
@@ -435,8 +435,8 @@ class DataProductSerializer(
def get_downloadURL(self, data_product):
"""Getter for downloadURL field."""
- if datastore.exists(data_product):
- request = self.context['request']
+ request = self.context['request']
+ if data_products_helper.exists(request, data_product):
return (request.build_absolute_uri(
reverse('django_airavata_api:download_file')) +
'?' +
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 21c7b6c..a7572a8 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -865,11 +865,11 @@ def upload_input_file(request):
request.authz_token, project_id)
exp_name = request.POST['experiment-name']
input_file = request.FILES['file']
- data_product = datastore.save(username, project.name, exp_name,
- input_file)
- data_product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- data_product.productUri = data_product_uri
+ # TODO: experiment_data_dir should be passed in
+ experiment_data_dir = data_products_helper.get_experiment_dir(
+ request, project_name=project.name, experiment_name=exp_name)
+ data_product = data_products_helper.save(request, experiment_data_dir,
+ input_file)
serializer = serializers.DataProductSerializer(
data_product, context={'request': request})
return JsonResponse({'uploaded': True,
@@ -894,7 +894,7 @@ def download_file(request):
.format(data_product_uri), exc_info=True)
raise Http404("data product does not exist") from e
try:
- data_file = datastore.open(data_product)
+ data_file = data_products_helper.open(request, data_product)
response = FileResponse(data_file,
content_type="application/octet-stream")
file_name = os.path.basename(data_file.name)
@@ -1383,30 +1383,6 @@ class UserStoragePathView(APIView):
context={'request': request})
return Response(serializer.data)
- def post(self, request, path="/", format=None):
- # TODO: this needs to be fixed or rethought
- username = request.user.username
- user_storage_path = path
- if user_storage_path.startswith("/"):
- user_storage_path = "." + user_storage_path
- serializer = self.serializer_class(
- data=request.data, context={
- 'request': request})
- serializer.is_valid(raise_exception=True)
- if serializer.validated_data['type'] == 'file':
- upload_file = request.FILES['file']
- datastore.save_user_file(username, user_storage_path, upload_file)
- elif serializer.validated_data['type'] == 'dir':
- datastore.create_user_dir(
- username, user_storage_path, serializer.validated_data['name'])
-
- # TODO return representation of created item
- listing = datastore.list_user_dir(
- request.user.username, user_storage_path)
- serializer = self.serializer_class(
- listing, many=True, context={'request': request})
- return Response(serializer.data)
-
class WorkspacePreferencesView(APIView):
serializer_class = serializers.WorkspacePreferencesSerializer
[airavata-django-portal] 01/02: AIRAVATA-3033 REST API for browsing,
downloading user storage
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 88089904f303826f97b188cab4cdb0cc9c6007f6
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue May 14 09:40:14 2019 -0400
AIRAVATA-3033 REST API for browsing, downloading user storage
---
django_airavata/apps/api/data_products_helper.py | 99 ++++++++++++++
django_airavata/apps/api/datastore.py | 36 +++++
.../apps/api/migrations/0002_auto_20190513_2037.py | 27 ++++
django_airavata/apps/api/models.py | 12 ++
django_airavata/apps/api/serializers.py | 26 ++++
django_airavata/apps/api/urls.py | 11 +-
django_airavata/apps/api/views.py | 150 ++++++++++++++-------
.../workspace/migrations/0002_delete_user_files.py | 18 +++
django_airavata/apps/workspace/models.py | 8 --
9 files changed, 329 insertions(+), 58 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
new file mode 100644
index 0000000..65b8024
--- /dev/null
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -0,0 +1,99 @@
+import os
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+
+from airavata.model.data.replica.ttypes import (
+ DataProductModel,
+ DataProductType,
+ DataReplicaLocationModel,
+ ReplicaLocationCategory,
+ ReplicaPersistentType
+)
+
+from . import datastore, models
+
+
+def save(request, path, file):
+ # return data_product
+ # TODO
+ pass
+
+
+def open(request, data_product):
+ # return file object
+ # TODO
+ pass
+
+
+def exists(request, data_product):
+ # return boolean
+ # TODO
+ pass
+
+
+def delete(request, data_product):
+ # TODO
+ pass
+
+
+def listdir(request, path):
+ if datastore.user_file_exists(request.user.username, path):
+ directories, files = datastore.list_user_dir(
+ request.user.username, path)
+ directories_data = []
+ for d in directories:
+ directories_data.append({'name': d,
+ 'path': os.path.join(path, d)})
+ files_data = []
+ for f in files:
+ user_rel_path = os.path.join(path, f)
+ full_path = datastore.path(request.user.username, user_rel_path)
+ data_product_uri = _get_data_product_uri(request, full_path)
+ files_data.append({'name': f,
+ 'path': user_rel_path,
+ 'data-product-uri': data_product_uri})
+ return directories_data, files_data
+ else:
+ raise ObjectDoesNotExist("User storage path does not exist")
+
+
+def _get_data_product_uri(request, full_path):
+
+ user_file = models.User_Files.objects.filter(
+ username=request.user.username, file_path=full_path)
+ if user_file.exists():
+ product_uri = user_file[0].file_dpu
+ else:
+ data_product = _create_data_product(request.user.username, full_path)
+ product_uri = request.airavata_client.registerDataProduct(
+ request.authz_token, data_product)
+ user_file_instance = models.User_Files(
+ username=request.user.username,
+ file_path=full_path,
+ file_dpu=product_uri)
+ user_file_instance.save()
+ return product_uri
+
+
+def _create_data_product(username, full_path):
+ data_product = DataProductModel()
+ data_product.gatewayId = settings.GATEWAY_ID
+ data_product.ownerName = username
+ file_name = os.path.basename(full_path)
+ data_product.productName = file_name
+ data_product.dataProductType = DataProductType.FILE
+ data_replica_location = DataReplicaLocationModel()
+ data_replica_location.storageResourceId = \
+ settings.GATEWAY_DATA_STORE_RESOURCE_ID
+ data_replica_location.replicaName = \
+ "{} gateway data store copy".format(file_name)
+ data_replica_location.replicaLocationCategory = \
+ ReplicaLocationCategory.GATEWAY_DATA_STORE
+ data_replica_location.replicaPersistentType = \
+ ReplicaPersistentType.TRANSIENT
+ data_replica_location.filePath = \
+ "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
+ full_path)
+ data_product.replicaLocations = [data_replica_location]
+ return data_product
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index eb0de66..2b5f9b5 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,6 +19,7 @@ experiment_data_storage = FileSystemStorage(
logger = logging.getLogger(__name__)
+# TODO: exists(username, path)
def exists(data_product):
"""Check if replica for data product exists in this data store."""
filepath = _get_replica_filepath(data_product)
@@ -30,6 +31,7 @@ def exists(data_product):
return False
+# TODO: open(username, path)
def open(data_product):
"""Open replica for data product if it exists in this data store."""
if exists(data_product):
@@ -73,6 +75,7 @@ def save_user(username, file):
return data_product
+# TODO: save(username, path, file)
def save(username, project_name, experiment_name, file):
"""Save file to username/project name/experiment_name in data store."""
exp_dir = os.path.join(
@@ -91,12 +94,34 @@ def save(username, project_name, experiment_name, file):
return data_product
+def save_user_file(username, path, file):
+ experiment_data_storage.save(os.path.join(
+ _user_dir_name(username),
+ experiment_data_storage.get_valid_name(path),
+ experiment_data_storage.get_valid_name(file.name)
+ ), file)
+
+
+def create_user_dir(username, path, dir_name):
+ user_dir = os.path.join(
+ _user_dir_name(username),
+ path,
+ experiment_data_storage.get_valid_name(dir_name))
+ if not experiment_data_storage.exists(user_dir):
+ os.mkdir(experiment_data_storage.path(user_dir))
+ else:
+ raise Exception(
+ "Directory {} already exists at that path".format(dir_name))
+
+
+# TODO: copy(username, source_path, target_path)
def copy(username, project_name, experiment_name, data_product):
"""Copy a data product into username/project_name/experiment_name dir."""
f = open(data_product)
return save(username, project_name, experiment_name, f)
+# TODO: delete(username, path)
def delete(data_product):
"""Delete replica for data product in this data store."""
@@ -162,6 +187,17 @@ def get_data_product(username, file_path):
raise ObjectDoesNotExist("User file does not exist")
+def list_user_dir(username, file_path):
+ logger.debug("file_path={}".format(file_path))
+ user_data_storage = _user_data_storage(username)
+ return user_data_storage.listdir(file_path)
+
+
+def path(username, file_path):
+ user_data_storage = _user_data_storage(username)
+ return user_data_storage.path(file_path)
+
+
def _get_replica_filepath(data_product):
replica_filepaths = [rep.filePath
for rep in data_product.replicaLocations
diff --git a/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
new file mode 100644
index 0000000..4bd4988
--- /dev/null
+++ b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-13 20:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_airavata_api', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='User_Files',
+ fields=[
+ ('username', models.CharField(max_length=64)),
+ ('file_path', models.TextField()),
+ ('file_dpu', models.CharField(max_length=500, primary_key=True, serialize=False)),
+ ],
+ ),
+ migrations.AddIndex(
+ model_name='user_files',
+ index=models.Index(fields=['username', 'file_path'], name='username_file_path_idx'),
+ ),
+ ]
diff --git a/django_airavata/apps/api/models.py b/django_airavata/apps/api/models.py
index 92344fd..5242208 100644
--- a/django_airavata/apps/api/models.py
+++ b/django_airavata/apps/api/models.py
@@ -9,3 +9,15 @@ class WorkspacePreferences(models.Model):
@classmethod
def create(self, username):
return WorkspacePreferences(username=username)
+
+
+class User_Files(models.Model):
+ username = models.CharField(max_length=64)
+ file_path = models.TextField()
+ file_dpu = models.CharField(max_length=500, primary_key=True)
+
+ class Meta:
+ indexes = [
+ models.Index(fields=['username', 'file_path'],
+ name='username_file_path_idx')
+ ]
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 5f0685c..86e235e 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -748,6 +748,32 @@ class ParserSerializer(thrift_utils.create_serializer_class(Parser)):
lookup_url_kwarg='parser_id')
+class UserStorageFileSerializer(serializers.Serializer):
+ name = serializers.CharField()
+ downloadURL = serializers.SerializerMethodField()
+
+ def get_downloadURL(self, file):
+ """Getter for downloadURL field."""
+ request = self.context['request']
+ return (request.build_absolute_uri(
+ reverse('django_airavata_api:download_file')) +
+ '?' +
+ urlencode({'data-product-uri': file['data-product-uri']}))
+
+
+class UserStorageDirectorySerializer(serializers.Serializer):
+ name = serializers.CharField()
+ url = FullyEncodedHyperlinkedIdentityField(
+ view_name='django_airavata_api:user-storage-items',
+ lookup_field='path',
+ lookup_url_kwarg='path')
+
+
+class UserStoragePathSerializer(serializers.Serializer):
+ directories = UserStorageDirectorySerializer(many=True)
+ files = UserStorageFileSerializer(many=True)
+
+
# ModelSerializers
class WorkspacePreferencesSerializer(serializers.ModelSerializer):
class Meta:
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index d835967..6b80b19 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -45,9 +45,10 @@ router.register(r'parsers', views.ParserViewSet, base_name='parser')
app_name = 'django_airavata_api'
urlpatterns = [
url(r'^', include(router.urls)),
- url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
- url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
- url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
+ # TODO: remove these
+ # url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
+ # url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
+ # url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
url(r'^upload$', views.upload_input_file, name='upload_input_file'),
url(r'^download', views.download_file, name='download_file'),
url(r'^delete-file$', views.delete_file, name='delete_file'),
@@ -77,6 +78,10 @@ urlpatterns = [
url(r'^workspace-preferences',
views.WorkspacePreferencesView.as_view(),
name="workspace-preferences"),
+ # url(r'^user-storage/~/(?P<path>.*)/$',
+ url(r'^user-storage/~/(?P<path>.*)$',
+ views.UserStoragePathView.as_view(),
+ name="user-storage-items")
]
if logger.isEnabledFor(logging.DEBUG):
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 989e4ae..21c7b6c 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -37,9 +37,18 @@ from django_airavata.apps.api.view_utils import (
APIResultPagination,
GenericAPIBackedViewSet
)
-from django_airavata.apps.workspace.models import User_Files
-from . import datastore, helpers, models, serializers, thrift_utils
+from . import (
+ data_products_helper,
+ datastore,
+ helpers,
+ models,
+ serializers,
+ thrift_utils
+)
+
+# from django_airavata.apps.workspace.models import User_Files
+
READ_PERMISSION_TYPE = '{}:READ'
@@ -788,63 +797,63 @@ class DataProductView(APIView):
data_product, context={'request': request})
return Response(serializer.data)
+# TODO: remove
+# @login_required
+# def get_user_files(request):
-@login_required
-def get_user_files(request):
+# dirs = [] # a list with file_name and file_dpu for each file
+# for o in User_Files.objects.values_list('file_name', 'file_dpu'):
+# file_details = {}
+# file_details['file_name'] = o[0]
+# file_details['file_dpu'] = o[1]
+# dirs.append(file_details)
- dirs = [] # a list with file_name and file_dpu for each file
- for o in User_Files.objects.values_list('file_name', 'file_dpu'):
- file_details = {}
- file_details['file_name'] = o[0]
- file_details['file_dpu'] = o[1]
- dirs.append(file_details)
+# return JsonResponse({'uploaded': True, 'user-files': dirs})
- return JsonResponse({'uploaded': True, 'user-files': dirs})
+# @login_required
+# def upload_user_file(request):
+# username = request.user.username
+# input_file = request.FILES['file']
+# file_details = {}
-@login_required
-def upload_user_file(request):
- username = request.user.username
- input_file = request.FILES['file']
- file_details = {}
-
- # To avoid duplicate file names
+# # To avoid duplicate file names
- if User_Files.objects.filter(file_name=input_file.name).exists():
- resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
- resp.status_code = 400
- return resp
+# if User_Files.objects.filter(file_name=input_file.name).exists():
+# resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
+# resp.status_code = 400
+# return resp
- else:
+# else:
- data_product = datastore.save_user(username, input_file)
- data_product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
- d.save()
- file_details['file_name'] = d.file_name
- file_details['file_dpu'] = d.file_dpu
- return JsonResponse({'uploaded': True,
- 'upload-file': file_details})
+# data_product = datastore.save_user(username, input_file)
+# data_product_uri = request.airavata_client.registerDataProduct(
+# request.authz_token, data_product)
+# d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
+# d.save()
+# file_details['file_name'] = d.file_name
+# file_details['file_dpu'] = d.file_dpu
+# return JsonResponse({'uploaded': True,
+# 'upload-file': file_details})
-@login_required
-def delete_user_file(request):
- data_product_uri = request.body.decode('utf-8')
- try:
- data_product = request.airavata_client.getDataProduct(
- request.authz_token, data_product_uri)
+# @login_required
+# def delete_user_file(request):
+# data_product_uri = request.body.decode('utf-8')
+# try:
+# data_product = request.airavata_client.getDataProduct(
+# request.authz_token, data_product_uri)
- except Exception as e:
- log.warning("Failed to load DataProduct for {}"
- .format(data_product_uri), exc_info=True)
- raise Http404("data product does not exist")(e)
+# except Exception as e:
+# log.warning("Failed to load DataProduct for {}"
+# .format(data_product_uri), exc_info=True)
+# raise Http404("data product does not exist")(e)
- # remove file_details entry from database and delete from datastore
- User_Files.objects.filter(file_dpu=data_product_uri).delete()
- datastore.delete(data_product)
+# # remove file_details entry from database and delete from datastore
+# User_Files.objects.filter(file_dpu=data_product_uri).delete()
+# datastore.delete(data_product)
- return JsonResponse({'deleted': True})
+# return JsonResponse({'deleted': True})
@login_required
@@ -897,6 +906,13 @@ def download_file(request):
@login_required
+def user_storage_download_file(request, path):
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+
+
+@login_required
def delete_file(request):
# TODO check that user has write access to this file using sharing API
data_product_uri = request.GET.get('data-product-uri', '')
@@ -1337,7 +1353,8 @@ class ParserViewSet(mixins.CreateModelMixin,
lookup_field = 'parser_id'
def get_list(self):
- return self.request.airavata_client.listAllParsers(self.authz_token, settings.GATEWAY_ID)
+ return self.request.airavata_client.listAllParsers(
+ self.authz_token, settings.GATEWAY_ID)
def get_instance(self, lookup_value):
return self.request.airavata_client.getParser(
@@ -1352,6 +1369,45 @@ class ParserViewSet(mixins.CreateModelMixin,
self.request.airavata_client.saveParser(self.authz_token, parser)
+class UserStoragePathView(APIView):
+ serializer_class = serializers.UserStoragePathSerializer
+
+ def get(self, request, path="/", format=None):
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+ # TODO: check if path is directory or file
+ directories, files = data_products_helper.listdir(request, path)
+ serializer = self.serializer_class(
+ {'directories': directories, 'files': files},
+ context={'request': request})
+ return Response(serializer.data)
+
+ def post(self, request, path="/", format=None):
+ # TODO: this needs to be fixed or rethought
+ username = request.user.username
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+ serializer = self.serializer_class(
+ data=request.data, context={
+ 'request': request})
+ serializer.is_valid(raise_exception=True)
+ if serializer.validated_data['type'] == 'file':
+ upload_file = request.FILES['file']
+ datastore.save_user_file(username, user_storage_path, upload_file)
+ elif serializer.validated_data['type'] == 'dir':
+ datastore.create_user_dir(
+ username, user_storage_path, serializer.validated_data['name'])
+
+ # TODO return representation of created item
+ listing = datastore.list_user_dir(
+ request.user.username, user_storage_path)
+ serializer = self.serializer_class(
+ listing, many=True, context={'request': request})
+ return Response(serializer.data)
+
+
class WorkspacePreferencesView(APIView):
serializer_class = serializers.WorkspacePreferencesSerializer
diff --git a/django_airavata/apps/workspace/migrations/0002_delete_user_files.py b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
new file mode 100644
index 0000000..9ed44e7
--- /dev/null
+++ b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-10 16:35
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_airavata_workspace', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name='User_Files',
+ ),
+ ]
diff --git a/django_airavata/apps/workspace/models.py b/django_airavata/apps/workspace/models.py
index a17940b..35e0d64 100644
--- a/django_airavata/apps/workspace/models.py
+++ b/django_airavata/apps/workspace/models.py
@@ -1,10 +1,2 @@
# Create your models here.
-
-from django.db import models
-
-
-class User_Files(models.Model):
- file_name = models.CharField(max_length=50)
- file_dpu = models.CharField(max_length=500)
- # username = models.CharField(primary_key=True,max_length=20) not required