You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2019/05/15 02:13:01 UTC
[airavata-django-portal] 01/02: AIRAVATA-3033 REST API for browsing,
downloading user storage
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch airavata-3016
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal.git
commit 88089904f303826f97b188cab4cdb0cc9c6007f6
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Tue May 14 09:40:14 2019 -0400
AIRAVATA-3033 REST API for browsing, downloading user storage
---
django_airavata/apps/api/data_products_helper.py | 99 ++++++++++++++
django_airavata/apps/api/datastore.py | 36 +++++
.../apps/api/migrations/0002_auto_20190513_2037.py | 27 ++++
django_airavata/apps/api/models.py | 12 ++
django_airavata/apps/api/serializers.py | 26 ++++
django_airavata/apps/api/urls.py | 11 +-
django_airavata/apps/api/views.py | 150 ++++++++++++++-------
.../workspace/migrations/0002_delete_user_files.py | 18 +++
django_airavata/apps/workspace/models.py | 8 --
9 files changed, 329 insertions(+), 58 deletions(-)
diff --git a/django_airavata/apps/api/data_products_helper.py b/django_airavata/apps/api/data_products_helper.py
new file mode 100644
index 0000000..65b8024
--- /dev/null
+++ b/django_airavata/apps/api/data_products_helper.py
@@ -0,0 +1,99 @@
+import os
+
+from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+
+from airavata.model.data.replica.ttypes import (
+ DataProductModel,
+ DataProductType,
+ DataReplicaLocationModel,
+ ReplicaLocationCategory,
+ ReplicaPersistentType
+)
+
+from . import datastore, models
+
+
+def save(request, path, file):
+ # return data_product
+ # TODO
+ pass
+
+
+def open(request, data_product):
+ # return file object
+ # TODO
+ pass
+
+
+def exists(request, data_product):
+ # return boolean
+ # TODO
+ pass
+
+
+def delete(request, data_product):
+ # TODO
+ pass
+
+
+def listdir(request, path):
+ if datastore.user_file_exists(request.user.username, path):
+ directories, files = datastore.list_user_dir(
+ request.user.username, path)
+ directories_data = []
+ for d in directories:
+ directories_data.append({'name': d,
+ 'path': os.path.join(path, d)})
+ files_data = []
+ for f in files:
+ user_rel_path = os.path.join(path, f)
+ full_path = datastore.path(request.user.username, user_rel_path)
+ data_product_uri = _get_data_product_uri(request, full_path)
+ files_data.append({'name': f,
+ 'path': user_rel_path,
+ 'data-product-uri': data_product_uri})
+ return directories_data, files_data
+ else:
+ raise ObjectDoesNotExist("User storage path does not exist")
+
+
+def _get_data_product_uri(request, full_path):
+
+ user_file = models.User_Files.objects.filter(
+ username=request.user.username, file_path=full_path)
+ if user_file.exists():
+ product_uri = user_file[0].file_dpu
+ else:
+ data_product = _create_data_product(request.user.username, full_path)
+ product_uri = request.airavata_client.registerDataProduct(
+ request.authz_token, data_product)
+ user_file_instance = models.User_Files(
+ username=request.user.username,
+ file_path=full_path,
+ file_dpu=product_uri)
+ user_file_instance.save()
+ return product_uri
+
+
+def _create_data_product(username, full_path):
+ data_product = DataProductModel()
+ data_product.gatewayId = settings.GATEWAY_ID
+ data_product.ownerName = username
+ file_name = os.path.basename(full_path)
+ data_product.productName = file_name
+ data_product.dataProductType = DataProductType.FILE
+ data_replica_location = DataReplicaLocationModel()
+ data_replica_location.storageResourceId = \
+ settings.GATEWAY_DATA_STORE_RESOURCE_ID
+ data_replica_location.replicaName = \
+ "{} gateway data store copy".format(file_name)
+ data_replica_location.replicaLocationCategory = \
+ ReplicaLocationCategory.GATEWAY_DATA_STORE
+ data_replica_location.replicaPersistentType = \
+ ReplicaPersistentType.TRANSIENT
+ data_replica_location.filePath = \
+ "file://{}:{}".format(settings.GATEWAY_DATA_STORE_HOSTNAME,
+ full_path)
+ data_product.replicaLocations = [data_replica_location]
+ return data_product
diff --git a/django_airavata/apps/api/datastore.py b/django_airavata/apps/api/datastore.py
index eb0de66..2b5f9b5 100644
--- a/django_airavata/apps/api/datastore.py
+++ b/django_airavata/apps/api/datastore.py
@@ -19,6 +19,7 @@ experiment_data_storage = FileSystemStorage(
logger = logging.getLogger(__name__)
+# TODO: exists(username, path)
def exists(data_product):
"""Check if replica for data product exists in this data store."""
filepath = _get_replica_filepath(data_product)
@@ -30,6 +31,7 @@ def exists(data_product):
return False
+# TODO: open(username, path)
def open(data_product):
"""Open replica for data product if it exists in this data store."""
if exists(data_product):
@@ -73,6 +75,7 @@ def save_user(username, file):
return data_product
+# TODO: save(username, path, file)
def save(username, project_name, experiment_name, file):
"""Save file to username/project name/experiment_name in data store."""
exp_dir = os.path.join(
@@ -91,12 +94,34 @@ def save(username, project_name, experiment_name, file):
return data_product
+def save_user_file(username, path, file):
+ experiment_data_storage.save(os.path.join(
+ _user_dir_name(username),
+ experiment_data_storage.get_valid_name(path),
+ experiment_data_storage.get_valid_name(file.name)
+ ), file)
+
+
+def create_user_dir(username, path, dir_name):
+ user_dir = os.path.join(
+ _user_dir_name(username),
+ path,
+ experiment_data_storage.get_valid_name(dir_name))
+ if not experiment_data_storage.exists(user_dir):
+ os.mkdir(experiment_data_storage.path(user_dir))
+ else:
+ raise Exception(
+ "Directory {} already exists at that path".format(dir_name))
+
+
+# TODO: copy(username, source_path, target_path)
def copy(username, project_name, experiment_name, data_product):
"""Copy a data product into username/project_name/experiment_name dir."""
f = open(data_product)
return save(username, project_name, experiment_name, f)
+# TODO: delete(username, path)
def delete(data_product):
"""Delete replica for data product in this data store."""
@@ -162,6 +187,17 @@ def get_data_product(username, file_path):
raise ObjectDoesNotExist("User file does not exist")
+def list_user_dir(username, file_path):
+ logger.debug("file_path={}".format(file_path))
+ user_data_storage = _user_data_storage(username)
+ return user_data_storage.listdir(file_path)
+
+
+def path(username, file_path):
+ user_data_storage = _user_data_storage(username)
+ return user_data_storage.path(file_path)
+
+
def _get_replica_filepath(data_product):
replica_filepaths = [rep.filePath
for rep in data_product.replicaLocations
diff --git a/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
new file mode 100644
index 0000000..4bd4988
--- /dev/null
+++ b/django_airavata/apps/api/migrations/0002_auto_20190513_2037.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-13 20:37
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_airavata_api', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='User_Files',
+ fields=[
+ ('username', models.CharField(max_length=64)),
+ ('file_path', models.TextField()),
+ ('file_dpu', models.CharField(max_length=500, primary_key=True, serialize=False)),
+ ],
+ ),
+ migrations.AddIndex(
+ model_name='user_files',
+ index=models.Index(fields=['username', 'file_path'], name='username_file_path_idx'),
+ ),
+ ]
diff --git a/django_airavata/apps/api/models.py b/django_airavata/apps/api/models.py
index 92344fd..5242208 100644
--- a/django_airavata/apps/api/models.py
+++ b/django_airavata/apps/api/models.py
@@ -9,3 +9,15 @@ class WorkspacePreferences(models.Model):
@classmethod
def create(self, username):
return WorkspacePreferences(username=username)
+
+
+class User_Files(models.Model):
+ username = models.CharField(max_length=64)
+ file_path = models.TextField()
+ file_dpu = models.CharField(max_length=500, primary_key=True)
+
+ class Meta:
+ indexes = [
+ models.Index(fields=['username', 'file_path'],
+ name='username_file_path_idx')
+ ]
diff --git a/django_airavata/apps/api/serializers.py b/django_airavata/apps/api/serializers.py
index 5f0685c..86e235e 100644
--- a/django_airavata/apps/api/serializers.py
+++ b/django_airavata/apps/api/serializers.py
@@ -748,6 +748,32 @@ class ParserSerializer(thrift_utils.create_serializer_class(Parser)):
lookup_url_kwarg='parser_id')
+class UserStorageFileSerializer(serializers.Serializer):
+ name = serializers.CharField()
+ downloadURL = serializers.SerializerMethodField()
+
+ def get_downloadURL(self, file):
+ """Getter for downloadURL field."""
+ request = self.context['request']
+ return (request.build_absolute_uri(
+ reverse('django_airavata_api:download_file')) +
+ '?' +
+ urlencode({'data-product-uri': file['data-product-uri']}))
+
+
+class UserStorageDirectorySerializer(serializers.Serializer):
+ name = serializers.CharField()
+ url = FullyEncodedHyperlinkedIdentityField(
+ view_name='django_airavata_api:user-storage-items',
+ lookup_field='path',
+ lookup_url_kwarg='path')
+
+
+class UserStoragePathSerializer(serializers.Serializer):
+ directories = UserStorageDirectorySerializer(many=True)
+ files = UserStorageFileSerializer(many=True)
+
+
# ModelSerializers
class WorkspacePreferencesSerializer(serializers.ModelSerializer):
class Meta:
diff --git a/django_airavata/apps/api/urls.py b/django_airavata/apps/api/urls.py
index d835967..6b80b19 100644
--- a/django_airavata/apps/api/urls.py
+++ b/django_airavata/apps/api/urls.py
@@ -45,9 +45,10 @@ router.register(r'parsers', views.ParserViewSet, base_name='parser')
app_name = 'django_airavata_api'
urlpatterns = [
url(r'^', include(router.urls)),
- url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
- url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
- url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
+ # TODO: remove these
+ # url(r'^get-ufiles$', views.get_user_files, name='get_user_files'),
+ # url(r'^upload-ufiles$', views.upload_user_file, name='upload_user_file'),
+ # url(r'^delete-ufiles$', views.delete_user_file, name='delete_user_file'),
url(r'^upload$', views.upload_input_file, name='upload_input_file'),
url(r'^download', views.download_file, name='download_file'),
url(r'^delete-file$', views.delete_file, name='delete_file'),
@@ -77,6 +78,10 @@ urlpatterns = [
url(r'^workspace-preferences',
views.WorkspacePreferencesView.as_view(),
name="workspace-preferences"),
+ # url(r'^user-storage/~/(?P<path>.*)/$',
+ url(r'^user-storage/~/(?P<path>.*)$',
+ views.UserStoragePathView.as_view(),
+ name="user-storage-items")
]
if logger.isEnabledFor(logging.DEBUG):
diff --git a/django_airavata/apps/api/views.py b/django_airavata/apps/api/views.py
index 989e4ae..21c7b6c 100644
--- a/django_airavata/apps/api/views.py
+++ b/django_airavata/apps/api/views.py
@@ -37,9 +37,18 @@ from django_airavata.apps.api.view_utils import (
APIResultPagination,
GenericAPIBackedViewSet
)
-from django_airavata.apps.workspace.models import User_Files
-from . import datastore, helpers, models, serializers, thrift_utils
+from . import (
+ data_products_helper,
+ datastore,
+ helpers,
+ models,
+ serializers,
+ thrift_utils
+)
+
+# from django_airavata.apps.workspace.models import User_Files
+
READ_PERMISSION_TYPE = '{}:READ'
@@ -788,63 +797,63 @@ class DataProductView(APIView):
data_product, context={'request': request})
return Response(serializer.data)
+# TODO: remove
+# @login_required
+# def get_user_files(request):
-@login_required
-def get_user_files(request):
+# dirs = [] # a list with file_name and file_dpu for each file
+# for o in User_Files.objects.values_list('file_name', 'file_dpu'):
+# file_details = {}
+# file_details['file_name'] = o[0]
+# file_details['file_dpu'] = o[1]
+# dirs.append(file_details)
- dirs = [] # a list with file_name and file_dpu for each file
- for o in User_Files.objects.values_list('file_name', 'file_dpu'):
- file_details = {}
- file_details['file_name'] = o[0]
- file_details['file_dpu'] = o[1]
- dirs.append(file_details)
+# return JsonResponse({'uploaded': True, 'user-files': dirs})
- return JsonResponse({'uploaded': True, 'user-files': dirs})
+# @login_required
+# def upload_user_file(request):
+# username = request.user.username
+# input_file = request.FILES['file']
+# file_details = {}
-@login_required
-def upload_user_file(request):
- username = request.user.username
- input_file = request.FILES['file']
- file_details = {}
-
- # To avoid duplicate file names
+# # To avoid duplicate file names
- if User_Files.objects.filter(file_name=input_file.name).exists():
- resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
- resp.status_code = 400
- return resp
+# if User_Files.objects.filter(file_name=input_file.name).exists():
+# resp = JsonResponse({'uploaded': False, 'error': "File already exists"})
+# resp.status_code = 400
+# return resp
- else:
+# else:
- data_product = datastore.save_user(username, input_file)
- data_product_uri = request.airavata_client.registerDataProduct(
- request.authz_token, data_product)
- d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
- d.save()
- file_details['file_name'] = d.file_name
- file_details['file_dpu'] = d.file_dpu
- return JsonResponse({'uploaded': True,
- 'upload-file': file_details})
+# data_product = datastore.save_user(username, input_file)
+# data_product_uri = request.airavata_client.registerDataProduct(
+# request.authz_token, data_product)
+# d = User_Files(file_name=input_file.name, file_dpu=data_product_uri)
+# d.save()
+# file_details['file_name'] = d.file_name
+# file_details['file_dpu'] = d.file_dpu
+# return JsonResponse({'uploaded': True,
+# 'upload-file': file_details})
-@login_required
-def delete_user_file(request):
- data_product_uri = request.body.decode('utf-8')
- try:
- data_product = request.airavata_client.getDataProduct(
- request.authz_token, data_product_uri)
+# @login_required
+# def delete_user_file(request):
+# data_product_uri = request.body.decode('utf-8')
+# try:
+# data_product = request.airavata_client.getDataProduct(
+# request.authz_token, data_product_uri)
- except Exception as e:
- log.warning("Failed to load DataProduct for {}"
- .format(data_product_uri), exc_info=True)
- raise Http404("data product does not exist")(e)
+# except Exception as e:
+# log.warning("Failed to load DataProduct for {}"
+# .format(data_product_uri), exc_info=True)
+# raise Http404("data product does not exist")(e)
- # remove file_details entry from database and delete from datastore
- User_Files.objects.filter(file_dpu=data_product_uri).delete()
- datastore.delete(data_product)
+# # remove file_details entry from database and delete from datastore
+# User_Files.objects.filter(file_dpu=data_product_uri).delete()
+# datastore.delete(data_product)
- return JsonResponse({'deleted': True})
+# return JsonResponse({'deleted': True})
@login_required
@@ -897,6 +906,13 @@ def download_file(request):
@login_required
+def user_storage_download_file(request, path):
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+
+
+@login_required
def delete_file(request):
# TODO check that user has write access to this file using sharing API
data_product_uri = request.GET.get('data-product-uri', '')
@@ -1337,7 +1353,8 @@ class ParserViewSet(mixins.CreateModelMixin,
lookup_field = 'parser_id'
def get_list(self):
- return self.request.airavata_client.listAllParsers(self.authz_token, settings.GATEWAY_ID)
+ return self.request.airavata_client.listAllParsers(
+ self.authz_token, settings.GATEWAY_ID)
def get_instance(self, lookup_value):
return self.request.airavata_client.getParser(
@@ -1352,6 +1369,45 @@ class ParserViewSet(mixins.CreateModelMixin,
self.request.airavata_client.saveParser(self.authz_token, parser)
+class UserStoragePathView(APIView):
+ serializer_class = serializers.UserStoragePathSerializer
+
+ def get(self, request, path="/", format=None):
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+ # TODO: check if path is directory or file
+ directories, files = data_products_helper.listdir(request, path)
+ serializer = self.serializer_class(
+ {'directories': directories, 'files': files},
+ context={'request': request})
+ return Response(serializer.data)
+
+ def post(self, request, path="/", format=None):
+ # TODO: this needs to be fixed or rethought
+ username = request.user.username
+ user_storage_path = path
+ if user_storage_path.startswith("/"):
+ user_storage_path = "." + user_storage_path
+ serializer = self.serializer_class(
+ data=request.data, context={
+ 'request': request})
+ serializer.is_valid(raise_exception=True)
+ if serializer.validated_data['type'] == 'file':
+ upload_file = request.FILES['file']
+ datastore.save_user_file(username, user_storage_path, upload_file)
+ elif serializer.validated_data['type'] == 'dir':
+ datastore.create_user_dir(
+ username, user_storage_path, serializer.validated_data['name'])
+
+ # TODO return representation of created item
+ listing = datastore.list_user_dir(
+ request.user.username, user_storage_path)
+ serializer = self.serializer_class(
+ listing, many=True, context={'request': request})
+ return Response(serializer.data)
+
+
class WorkspacePreferencesView(APIView):
serializer_class = serializers.WorkspacePreferencesSerializer
diff --git a/django_airavata/apps/workspace/migrations/0002_delete_user_files.py b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
new file mode 100644
index 0000000..9ed44e7
--- /dev/null
+++ b/django_airavata/apps/workspace/migrations/0002_delete_user_files.py
@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.20 on 2019-05-10 16:35
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_airavata_workspace', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name='User_Files',
+ ),
+ ]
diff --git a/django_airavata/apps/workspace/models.py b/django_airavata/apps/workspace/models.py
index a17940b..35e0d64 100644
--- a/django_airavata/apps/workspace/models.py
+++ b/django_airavata/apps/workspace/models.py
@@ -1,10 +1,2 @@
# Create your models here.
-
-from django.db import models
-
-
-class User_Files(models.Model):
- file_name = models.CharField(max_length=50)
- file_dpu = models.CharField(max_length=500)
- # username = models.CharField(primary_key=True,max_length=20) not required