You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ma...@apache.org on 2021/04/14 22:03:26 UTC
[airavata-django-portal-sdk] branch mft-integration updated:
AIRAVATA-3420 Implement browsing methods of MFTUserStorageProvider
This is an automated email from the ASF dual-hosted git repository.
machristie pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata-django-portal-sdk.git
The following commit(s) were added to refs/heads/mft-integration by this push:
new 51cfd7e AIRAVATA-3420 Implement browsing methods of MFTUserStorageProvider
51cfd7e is described below
commit 51cfd7e9b6a81a354c4bea63bcf0f3591bd3e3aa
Author: Marcus Christie <ma...@apache.org>
AuthorDate: Wed Apr 14 18:03:12 2021 -0400
AIRAVATA-3420 Implement browsing methods of MFTUserStorageProvider
---
airavata_django_portal_sdk/user_storage/api.py | 27 +-
.../user_storage/backends/CredCommon_pb2.py | 306 ++++++++++++++++++
.../user_storage/backends/CredCommon_pb2_grpc.py | 4 +
.../{ => user_storage/backends}/MFTApi_pb2.py | 189 +++++-------
.../{ => user_storage/backends}/MFTApi_pb2_grpc.py | 0
.../user_storage/backends/__init__.py | 3 +-
.../user_storage/backends/base.py | 4 +
.../backends/django_filesystem_provider.py | 11 +-
.../user_storage/backends/mft_provider.py | 341 ++++++++++++---------
9 files changed, 608 insertions(+), 277 deletions(-)
diff --git a/airavata_django_portal_sdk/user_storage/api.py b/airavata_django_portal_sdk/user_storage/api.py
index 96cd0da..cfde16a 100644
--- a/airavata_django_portal_sdk/user_storage/api.py
+++ b/airavata_django_portal_sdk/user_storage/api.py
@@ -51,7 +51,8 @@ def get_user_storage_provider(request, owner_username=None, storage_resource_id=
options = dict(directory=settings.GATEWAY_DATA_STORE_DIR)
logger.warning("Please add the USER_STORAGES setting. Using legacy GATEWAY_DATA_STORE_RESOURCE_ID and GATEWAY_DATA_STORE_DIR settings.")
else:
- for conf in settings.USER_STORAGES:
+ for key in settings.USER_STORAGES:
+ conf = settings.USER_STORAGES[key]
if conf['STORAGE_RESOURCE_ID'] == storage_resource_id:
module_class_name = conf['BACKEND']
options = conf.get('OPTIONS', {})
@@ -241,7 +242,7 @@ def dir_exists(request, path, storage_resource_id=None):
return resp.json()['isDir']
else:
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- return backend.exists(path)
+ return backend.is_dir(path)
def user_file_exists(request, path, storage_resource_id=None):
@@ -256,7 +257,7 @@ def user_file_exists(request, path, storage_resource_id=None):
resp.raise_for_status()
return resp.json()['files'][0]['dataProductURI']
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- if backend.exists(path) and backend.is_file(path):
+ if backend.is_file(path):
_, files = backend.get_metadata(path)
full_path = files[0]['resource_path']
data_product_uri = _get_data_product_uri(request, full_path, backend.resource_id)
@@ -343,7 +344,7 @@ def get_file_metadata(request, path, storage_resource_id=None):
return file
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
- if backend.exists(path) and backend.is_file(path):
+ if backend.is_file(path):
_, files = backend.get_metadata(path)
file = files[0]
data_product_uri = _get_data_product_uri(request, file['resource_path'],
@@ -418,6 +419,9 @@ def listdir(request, path, storage_resource_id=None):
backend = get_user_storage_provider(request, storage_resource_id=storage_resource_id)
directories, files = backend.get_metadata(path)
+ # Mark the TMP_INPUT_FILE_UPLOAD_DIR directory as hidden in the UI
+ for directory in directories:
+ directory['hidden'] = directory['path'] == TMP_INPUT_FILE_UPLOAD_DIR
# for each file, lookup or register a data product and enrich the file
# metadata with data-product-uri and mime-type
for file in files:
@@ -642,13 +646,14 @@ def _determine_content_type(full_path, content_type=None):
# Try to guess the content-type from file extension
guessed_type, encoding = mimetypes.guess_type(full_path)
result = guessed_type
- if result is None or result == "application/octet-stream":
- # Check if file is Unicode text by trying to read some of it
- try:
- open(full_path, "r").read(1024)
- result = "text/plain"
- except UnicodeDecodeError:
- logger.debug(f"Failed to read as Unicode text: {full_path}")
+ # TODO: implement change to use UserStorageProvider.open to read from file
+ # if result is None or result == "application/octet-stream":
+ # # Check if file is Unicode text by trying to read some of it
+ # try:
+ # open(full_path, "r").read(1024)
+ # result = "text/plain"
+ # except UnicodeDecodeError:
+ # logger.debug(f"Failed to read as Unicode text: {full_path}")
return result
diff --git a/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py
new file mode 100644
index 0000000..3c6a6c3
--- /dev/null
+++ b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2.py
@@ -0,0 +1,306 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: CredCommon.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor.FileDescriptor(
+ name='CredCommon.proto',
+ package='org.apache.airavata.mft.common',
+ syntax='proto3',
+ serialized_options=b'P\001',
+ create_key=_descriptor._internal_create_key,
+ serialized_pb=b'\n\x10\x43redCommon.proto\x12\x1eorg.apache.airavata.mft.common\"\x1e\n\rUserTokenAuth\x12\r\n\x05token\x18\x01 \x01(\t\"@\n\tAgentAuth\x12\r\n\x05token\x18\x01 \x01(\t\x12\x0f\n\x07\x61gentId\x18\x02 \x01(\t\x12\x13\n\x0b\x61gentSecret\x18\x03 \x01(\t\"\xcb\x01\n\x0c\x44\x65legateAuth\x12\x0e\n\x06userId\x18\x01 \x01(\t\x12\x10\n\x08\x63lientId\x18\x02 \x01(\t\x12\x14\n\x0c\x63lientSecret\x18\x03 \x01(\t\x12P\n\nproperties\x18\x04 \x03(\x0b\x32<.org.apache.airavata.mft [...]
+)
+
+
+
+
+_USERTOKENAUTH = _descriptor.Descriptor(
+ name='UserTokenAuth',
+ full_name='org.apache.airavata.mft.common.UserTokenAuth',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='token', full_name='org.apache.airavata.mft.common.UserTokenAuth.token', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=52,
+ serialized_end=82,
+)
+
+
+_AGENTAUTH = _descriptor.Descriptor(
+ name='AgentAuth',
+ full_name='org.apache.airavata.mft.common.AgentAuth',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='token', full_name='org.apache.airavata.mft.common.AgentAuth.token', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='agentId', full_name='org.apache.airavata.mft.common.AgentAuth.agentId', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='agentSecret', full_name='org.apache.airavata.mft.common.AgentAuth.agentSecret', index=2,
+ number=3, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=84,
+ serialized_end=148,
+)
+
+
+_DELEGATEAUTH_PROPERTIESENTRY = _descriptor.Descriptor(
+ name='PropertiesEntry',
+ full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='key', full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry.key', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='value', full_name='org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry.value', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=b'8\001',
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=305,
+ serialized_end=354,
+)
+
+_DELEGATEAUTH = _descriptor.Descriptor(
+ name='DelegateAuth',
+ full_name='org.apache.airavata.mft.common.DelegateAuth',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='userId', full_name='org.apache.airavata.mft.common.DelegateAuth.userId', index=0,
+ number=1, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='clientId', full_name='org.apache.airavata.mft.common.DelegateAuth.clientId', index=1,
+ number=2, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='clientSecret', full_name='org.apache.airavata.mft.common.DelegateAuth.clientSecret', index=2,
+ number=3, type=9, cpp_type=9, label=1,
+ has_default_value=False, default_value=b"".decode('utf-8'),
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='properties', full_name='org.apache.airavata.mft.common.DelegateAuth.properties', index=3,
+ number=4, type=11, cpp_type=10, label=3,
+ has_default_value=False, default_value=[],
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ ],
+ extensions=[
+ ],
+ nested_types=[_DELEGATEAUTH_PROPERTIESENTRY, ],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ ],
+ serialized_start=151,
+ serialized_end=354,
+)
+
+
+_AUTHTOKEN = _descriptor.Descriptor(
+ name='AuthToken',
+ full_name='org.apache.airavata.mft.common.AuthToken',
+ filename=None,
+ file=DESCRIPTOR,
+ containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[
+ _descriptor.FieldDescriptor(
+ name='userTokenAuth', full_name='org.apache.airavata.mft.common.AuthToken.userTokenAuth', index=0,
+ number=1, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='agentAuth', full_name='org.apache.airavata.mft.common.AuthToken.agentAuth', index=1,
+ number=2, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ _descriptor.FieldDescriptor(
+ name='delegateAuth', full_name='org.apache.airavata.mft.common.AuthToken.delegateAuth', index=2,
+ number=3, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
+ message_type=None, enum_type=None, containing_type=None,
+ is_extension=False, extension_scope=None,
+ serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
+ ],
+ extensions=[
+ ],
+ nested_types=[],
+ enum_types=[
+ ],
+ serialized_options=None,
+ is_extendable=False,
+ syntax='proto3',
+ extension_ranges=[],
+ oneofs=[
+ _descriptor.OneofDescriptor(
+ name='authMechanism', full_name='org.apache.airavata.mft.common.AuthToken.authMechanism',
+ index=0, containing_type=None,
+ create_key=_descriptor._internal_create_key,
+ fields=[]),
+ ],
+ serialized_start=357,
+ serialized_end=591,
+)
+
+_DELEGATEAUTH_PROPERTIESENTRY.containing_type = _DELEGATEAUTH
+_DELEGATEAUTH.fields_by_name['properties'].message_type = _DELEGATEAUTH_PROPERTIESENTRY
+_AUTHTOKEN.fields_by_name['userTokenAuth'].message_type = _USERTOKENAUTH
+_AUTHTOKEN.fields_by_name['agentAuth'].message_type = _AGENTAUTH
+_AUTHTOKEN.fields_by_name['delegateAuth'].message_type = _DELEGATEAUTH
+_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append(
+ _AUTHTOKEN.fields_by_name['userTokenAuth'])
+_AUTHTOKEN.fields_by_name['userTokenAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism']
+_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append(
+ _AUTHTOKEN.fields_by_name['agentAuth'])
+_AUTHTOKEN.fields_by_name['agentAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism']
+_AUTHTOKEN.oneofs_by_name['authMechanism'].fields.append(
+ _AUTHTOKEN.fields_by_name['delegateAuth'])
+_AUTHTOKEN.fields_by_name['delegateAuth'].containing_oneof = _AUTHTOKEN.oneofs_by_name['authMechanism']
+DESCRIPTOR.message_types_by_name['UserTokenAuth'] = _USERTOKENAUTH
+DESCRIPTOR.message_types_by_name['AgentAuth'] = _AGENTAUTH
+DESCRIPTOR.message_types_by_name['DelegateAuth'] = _DELEGATEAUTH
+DESCRIPTOR.message_types_by_name['AuthToken'] = _AUTHTOKEN
+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
+
+UserTokenAuth = _reflection.GeneratedProtocolMessageType('UserTokenAuth', (_message.Message,), {
+ 'DESCRIPTOR' : _USERTOKENAUTH,
+ '__module__' : 'CredCommon_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.UserTokenAuth)
+ })
+_sym_db.RegisterMessage(UserTokenAuth)
+
+AgentAuth = _reflection.GeneratedProtocolMessageType('AgentAuth', (_message.Message,), {
+ 'DESCRIPTOR' : _AGENTAUTH,
+ '__module__' : 'CredCommon_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.AgentAuth)
+ })
+_sym_db.RegisterMessage(AgentAuth)
+
+DelegateAuth = _reflection.GeneratedProtocolMessageType('DelegateAuth', (_message.Message,), {
+
+ 'PropertiesEntry' : _reflection.GeneratedProtocolMessageType('PropertiesEntry', (_message.Message,), {
+ 'DESCRIPTOR' : _DELEGATEAUTH_PROPERTIESENTRY,
+ '__module__' : 'CredCommon_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.DelegateAuth.PropertiesEntry)
+ })
+ ,
+ 'DESCRIPTOR' : _DELEGATEAUTH,
+ '__module__' : 'CredCommon_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.DelegateAuth)
+ })
+_sym_db.RegisterMessage(DelegateAuth)
+_sym_db.RegisterMessage(DelegateAuth.PropertiesEntry)
+
+AuthToken = _reflection.GeneratedProtocolMessageType('AuthToken', (_message.Message,), {
+ 'DESCRIPTOR' : _AUTHTOKEN,
+ '__module__' : 'CredCommon_pb2'
+ # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.common.AuthToken)
+ })
+_sym_db.RegisterMessage(AuthToken)
+
+
+DESCRIPTOR._options = None
+_DELEGATEAUTH_PROPERTIESENTRY._options = None
+# @@protoc_insertion_point(module_scope)
diff --git a/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py
new file mode 100644
index 0000000..2daafff
--- /dev/null
+++ b/airavata_django_portal_sdk/user_storage/backends/CredCommon_pb2_grpc.py
@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
diff --git a/airavata_django_portal_sdk/MFTApi_pb2.py b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py
similarity index 79%
rename from airavata_django_portal_sdk/MFTApi_pb2.py
rename to airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py
index 1e89796..2301467 100644
--- a/airavata_django_portal_sdk/MFTApi_pb2.py
+++ b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2.py
@@ -13,6 +13,7 @@ _sym_db = _symbol_database.Default()
from google.api import annotations_pb2 as google_dot_api_dot_annotations__pb2
from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2
+from . import CredCommon_pb2 as CredCommon__pb2
DESCRIPTOR = _descriptor.FileDescriptor(
@@ -21,9 +22,9 @@ DESCRIPTOR = _descriptor.FileDescriptor(
syntax='proto3',
serialized_options=b'P\001',
create_key=_descriptor._internal_create_key,
- serialized_pb=b'\n\x0cMFTApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\"\x9f\x04\n\x12TransferApiRequest\x12\x17\n\x0fsourceStorageId\x18\x01 \x01(\t\x12\x12\n\nsourcePath\x18\x02 \x01(\t\x12\x12\n\nsourceType\x18\x03 \x01(\t\x12\x13\n\x0bsourceToken\x18\x04 \x01(\t\x12\x1d\n\x15sourceResourceBackend\x18\x05 \x01(\t\x12\x1f\n\x17sourceCredentialBackend\x18\x06 \x01(\t\x12\x1c\n\x14\x64\x65stinationStorageId\x18\ [...]
+ serialized_pb=b'\n\x0cMFTApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x1cgoogle/api/annotations.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x10\x43redCommon.proto\"\xa3\x03\n\x12TransferApiRequest\x12\x18\n\x10sourceResourceId\x18\x01 \x01(\t\x12\x12\n\nsourceType\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x65stinationResourceId\x18\x07 \x01(\t\x12\x17\n\x0f\x64\x65stinationType\x18\t \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\n \x01(\t\x1 [...]
,
- dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,])
+ dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,CredCommon__pb2.DESCRIPTOR,])
@@ -62,8 +63,8 @@ _TRANSFERAPIREQUEST_TARGETAGENTSENTRY = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=605,
- serialized_end=656,
+ serialized_start=499,
+ serialized_end=550,
)
_TRANSFERAPIREQUEST = _descriptor.Descriptor(
@@ -75,107 +76,65 @@ _TRANSFERAPIREQUEST = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
- name='sourceStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceStorageId', index=0,
+ name='sourceResourceId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceResourceId', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='sourcePath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourcePath', index=1,
+ name='sourceType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceType', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='sourceType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceType', index=2,
+ name='sourceToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceToken', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='sourceToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceToken', index=3,
- number=4, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='sourceResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceResourceBackend', index=4,
- number=5, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='sourceCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.sourceCredentialBackend', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='destinationStorageId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationStorageId', index=6,
+ name='destinationResourceId', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationResourceId', index=3,
number=7, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='destinationPath', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationPath', index=7,
- number=8, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='destinationType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationType', index=8,
+ name='destinationType', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationType', index=4,
number=9, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='destinationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationToken', index=9,
+ name='destinationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destinationToken', index=5,
number=10, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='destResourceBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destResourceBackend', index=10,
- number=11, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='destCredentialBackend', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.destCredentialBackend', index=11,
- number=12, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='affinityTransfer', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.affinityTransfer', index=12,
+ name='affinityTransfer', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.affinityTransfer', index=6,
number=13, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='targetAgents', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.targetAgents', index=13,
+ name='targetAgents', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.targetAgents', index=7,
number=14, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.mftAuthorizationToken', index=14,
- number=15, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferApiRequest.mftAuthorizationToken', index=8,
+ number=15, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -191,8 +150,8 @@ _TRANSFERAPIREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=113,
- serialized_end=656,
+ serialized_start=131,
+ serialized_end=550,
)
@@ -223,8 +182,8 @@ _TRANSFERAPIRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=658,
- serialized_end=699,
+ serialized_start=552,
+ serialized_end=593,
)
@@ -237,44 +196,37 @@ _HTTPUPLOADAPIREQUEST = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
- name='destinationStoreId', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationStoreId', index=0,
+ name='destinationResourceId', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationResourceId', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='destinationPath', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationPath', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='destinationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationToken', index=2,
+ name='destinationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationToken', index=1,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='destinationType', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationType', index=3,
+ name='destinationType', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.destinationType', index=2,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.targetAgent', index=4,
+ name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.targetAgent', index=3,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.mftAuthorizationToken', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpUploadApiRequest.mftAuthorizationToken', index=4,
+ number=6, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -290,8 +242,8 @@ _HTTPUPLOADAPIREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=702,
- serialized_end=880,
+ serialized_start=596,
+ serialized_end=795,
)
@@ -329,8 +281,8 @@ _HTTPUPLOADAPIRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=882,
- serialized_end=939,
+ serialized_start=797,
+ serialized_end=854,
)
@@ -343,44 +295,37 @@ _HTTPDOWNLOADAPIREQUEST = _descriptor.Descriptor(
create_key=_descriptor._internal_create_key,
fields=[
_descriptor.FieldDescriptor(
- name='sourceStoreId', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceStoreId', index=0,
+ name='sourceResourceId', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceResourceId', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='sourcePath', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourcePath', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
- message_type=None, enum_type=None, containing_type=None,
- is_extension=False, extension_scope=None,
- serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
- _descriptor.FieldDescriptor(
- name='sourceToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceToken', index=2,
+ name='sourceToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceToken', index=1,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='sourceType', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceType', index=3,
+ name='sourceType', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.sourceType', index=2,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.targetAgent', index=4,
+ name='targetAgent', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.targetAgent', index=3,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
- name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.mftAuthorizationToken', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.HttpDownloadApiRequest.mftAuthorizationToken', index=4,
+ number=6, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -396,8 +341,8 @@ _HTTPDOWNLOADAPIREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=942,
- serialized_end=1102,
+ serialized_start=857,
+ serialized_end=1043,
)
@@ -435,8 +380,8 @@ _HTTPDOWNLOADAPIRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1104,
- serialized_end=1163,
+ serialized_start=1045,
+ serialized_end=1104,
)
@@ -457,8 +402,8 @@ _TRANSFERSTATEAPIREQUEST = _descriptor.Descriptor(
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.TransferStateApiRequest.mftAuthorizationToken', index=1,
- number=2, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ number=2, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -474,8 +419,8 @@ _TRANSFERSTATEAPIREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1165,
- serialized_end=1241,
+ serialized_start=1106,
+ serialized_end=1225,
)
@@ -527,8 +472,8 @@ _TRANSFERSTATEAPIRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1243,
- serialized_end=1349,
+ serialized_start=1227,
+ serialized_end=1333,
)
@@ -577,8 +522,8 @@ _RESOURCEAVAILABILITYREQUEST = _descriptor.Descriptor(
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.ResourceAvailabilityRequest.mftAuthorizationToken', index=5,
- number=6, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ number=6, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -594,8 +539,8 @@ _RESOURCEAVAILABILITYREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1352,
- serialized_end=1537,
+ serialized_start=1336,
+ serialized_end=1564,
)
@@ -626,8 +571,8 @@ _RESOURCEAVAILABILITYRESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1539,
- serialized_end=1588,
+ serialized_start=1566,
+ serialized_end=1615,
)
@@ -707,8 +652,8 @@ _FILEMETADATARESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1591,
- serialized_end=1790,
+ serialized_start=1618,
+ serialized_end=1817,
)
@@ -795,8 +740,8 @@ _DIRECTORYMETADATARESPONSE = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=1793,
- serialized_end=2143,
+ serialized_start=1820,
+ serialized_end=2170,
)
@@ -859,8 +804,8 @@ _FETCHRESOURCEMETADATAREQUEST = _descriptor.Descriptor(
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
_descriptor.FieldDescriptor(
name='mftAuthorizationToken', full_name='org.apache.airavata.mft.api.service.FetchResourceMetadataRequest.mftAuthorizationToken', index=7,
- number=9, type=9, cpp_type=9, label=1,
- has_default_value=False, default_value=b"".decode('utf-8'),
+ number=9, type=11, cpp_type=10, label=1,
+ has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key),
@@ -876,14 +821,20 @@ _FETCHRESOURCEMETADATAREQUEST = _descriptor.Descriptor(
extension_ranges=[],
oneofs=[
],
- serialized_start=2146,
- serialized_end=2374,
+ serialized_start=2173,
+ serialized_end=2444,
)
_TRANSFERAPIREQUEST_TARGETAGENTSENTRY.containing_type = _TRANSFERAPIREQUEST
_TRANSFERAPIREQUEST.fields_by_name['targetAgents'].message_type = _TRANSFERAPIREQUEST_TARGETAGENTSENTRY
+_TRANSFERAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
+_HTTPUPLOADAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
+_HTTPDOWNLOADAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
+_TRANSFERSTATEAPIREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
+_RESOURCEAVAILABILITYREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
_DIRECTORYMETADATARESPONSE.fields_by_name['directories'].message_type = _DIRECTORYMETADATARESPONSE
_DIRECTORYMETADATARESPONSE.fields_by_name['files'].message_type = _FILEMETADATARESPONSE
+_FETCHRESOURCEMETADATAREQUEST.fields_by_name['mftAuthorizationToken'].message_type = CredCommon__pb2._AUTHTOKEN
DESCRIPTOR.message_types_by_name['TransferApiRequest'] = _TRANSFERAPIREQUEST
DESCRIPTOR.message_types_by_name['TransferApiResponse'] = _TRANSFERAPIRESPONSE
DESCRIPTOR.message_types_by_name['HttpUploadApiRequest'] = _HTTPUPLOADAPIREQUEST
@@ -1009,8 +960,8 @@ _MFTAPISERVICE = _descriptor.ServiceDescriptor(
index=0,
serialized_options=None,
create_key=_descriptor._internal_create_key,
- serialized_start=2377,
- serialized_end=3838,
+ serialized_start=2447,
+ serialized_end=3908,
methods=[
_descriptor.MethodDescriptor(
name='submitTransfer',
diff --git a/airavata_django_portal_sdk/MFTApi_pb2_grpc.py b/airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2_grpc.py
similarity index 100%
rename from airavata_django_portal_sdk/MFTApi_pb2_grpc.py
rename to airavata_django_portal_sdk/user_storage/backends/MFTApi_pb2_grpc.py
diff --git a/airavata_django_portal_sdk/user_storage/backends/__init__.py b/airavata_django_portal_sdk/user_storage/backends/__init__.py
index eefe980..48846df 100644
--- a/airavata_django_portal_sdk/user_storage/backends/__init__.py
+++ b/airavata_django_portal_sdk/user_storage/backends/__init__.py
@@ -1,3 +1,4 @@
from .django_filesystem_provider import DjangoFileSystemProvider
+from .mft_provider import MFTUserStorageProvider
-__all__ = ['DjangoFileSystemProvider']
+__all__ = ['DjangoFileSystemProvider', 'MFTUserStorageProvider']
diff --git a/airavata_django_portal_sdk/user_storage/backends/base.py b/airavata_django_portal_sdk/user_storage/backends/base.py
index 409a794..9936765 100644
--- a/airavata_django_portal_sdk/user_storage/backends/base.py
+++ b/airavata_django_portal_sdk/user_storage/backends/base.py
@@ -62,3 +62,7 @@ class UserStorageProvider:
@property
def username(self):
return self.authz_token.claimsMap['userName']
+
+ @property
+ def access_token(self):
+ return self.authz_token.accessToken
diff --git a/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py b/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py
index 78812bd..97e2f9e 100644
--- a/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py
+++ b/airavata_django_portal_sdk/user_storage/backends/django_filesystem_provider.py
@@ -6,12 +6,9 @@ from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation
from django.core.files.storage import FileSystemStorage
from .base import UserStorageProvider
-from django.core.files import File
logger = logging.getLogger(__name__)
-TMP_INPUT_FILE_UPLOAD_DIR = "tmp"
-
class DjangoFileSystemProvider(UserStorageProvider):
def __init__(self, authz_token, resource_id, context=None, directory=None, storage_resource_id=None, **kwargs):
@@ -58,9 +55,9 @@ class DjangoFileSystemProvider(UserStorageProvider):
{
"name": d,
"path": datastore.rel_path(dpath),
+ "resource_path": datastore.rel_path(dpath),
"created_time": created_time,
"size": size,
- "hidden": dpath == TMP_INPUT_FILE_UPLOAD_DIR,
}
)
files_data = []
@@ -77,10 +74,9 @@ class DjangoFileSystemProvider(UserStorageProvider):
{
"name": f,
"path": datastore.rel_path(full_path),
- "resource_path": full_path,
+ "resource_path": datastore.rel_path(full_path),
"created_time": created_time,
"size": size,
- "hidden": False,
}
)
return directories_data, files_data
@@ -93,10 +89,9 @@ class DjangoFileSystemProvider(UserStorageProvider):
{
"name": os.path.basename(resource_path),
"path": datastore.rel_path(full_path),
- "resource_path": full_path,
+ "resource_path": datastore.rel_path(full_path),
"created_time": created_time,
"size": size,
- "hidden": False,
}
]
else:
diff --git a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
index 6678572..488e0b0 100644
--- a/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
+++ b/airavata_django_portal_sdk/user_storage/backends/mft_provider.py
@@ -1,152 +1,217 @@
+import logging
+import os
+from datetime import datetime
+
+import grpc
+
+from . import CredCommon_pb2, MFTApi_pb2, MFTApi_pb2_grpc
from .base import UserStorageProvider
+logger = logging.getLogger(__name__)
+
class MFTUserStorageProvider(UserStorageProvider):
+ def __init__(self, authz_token, resource_id, context=None, resource_token=None, mft_api_endpoint=None, mft_api_secure=False, resource_per_gateway=False, **kwargs):
+ super().__init__(authz_token, resource_id, context=context, **kwargs)
+ self.resource_token = resource_token
+ self.mft_api_endpoint = mft_api_endpoint
+ self.mft_api_secure = mft_api_secure
+ self.resource_per_gateway = resource_per_gateway
+
def exists(self, resource_path):
- return super().exists(resource_path)
-# with grpc.insecure_channel('localhost:7004') as channel:
-# # remove trailing slash and figure out parent path
-# # FIXME remove the hard coded /tmp path
-# parent_path, child_path = os.path.split(f"/tmp/{path}".rstrip("/"))
-# logger.debug(f"parent_path={parent_path}, child_path={child_path}")
-# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
-# # Get metadata for parent directory and see if child_path exists
-# request = MFTApi_pb2.FetchResourceMetadataRequest(
-# resourceId="remote-ssh-dir-resource",
-# resourceType="SCP",
-# resourceToken="local-ssh-cred",
-# resourceBackend="FILE",
-# resourceCredentialBackend="FILE",
-# targetAgentId="agent0",
-# childPath=parent_path,
-# mftAuthorizationToken="user token")
-# response = stub.getDirectoryResourceMetadata(request)
-# # if not child_path, then return True since the response was
-# # successful and we just need to confirm the existence of the root dir
-# if child_path == '':
-# return True
-# return child_path in map(lambda f: f.friendlyName, response.directories)
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ # TODO: is this still needed?
+ # parent_path, child_path = os.path.split(f"/tmp/{resource_path}".rstrip("/"))
+ # get metadata for the parent path
+ if child_path is not None:
+ parent_path, child_path = os.path.split(child_path)
+ else:
+ parent_path = None
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ # Get metadata for parent directory and see if child_path exists
+ request = MFTApi_pb2.FetchResourceMetadataRequest(
+ # resourceId="remote-ssh-dir-resource",
+ resourceId=self.resource_id,
+ resourceType="SCP",
+ # resourceToken="local-ssh-cred",
+ resourceToken=self.resource_token,
+ resourceBackend="FILE",
+ resourceCredentialBackend="FILE",
+ targetAgentId="agent0",
+ childPath=parent_path,
+ mftAuthorizationToken=self.auth_token,
+ )
+ try:
+ response = stub.getDirectoryResourceMetadata(request)
+ except Exception:
+ # Could not find the parent path, so apparently doesn't exist
+ logger.warning(f"Could not get metadata for {parent_path} on {self.resource_id}")
+ return False
+ # if not child_path, then return True since the response was
+ # successful and we just need to confirm the existence of the root dir
+ if child_path is None:
+ return True
+ return child_path in map(lambda f: f.friendlyName, list(response.directories) + list(response.files))
def get_metadata(self, resource_path):
- return super().get_metadata(resource_path)
-# def listdir(self, request, path):
-# # TODO setup resourceId, etc from __init__ arguments
-# channel = grpc.insecure_channel('localhost:7004')
-# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
-# request = MFTApi_pb2.FetchResourceMetadataRequest(
-# resourceId="remote-ssh-dir-resource",
-# resourceType="SCP",
-# resourceToken="local-ssh-cred",
-# resourceBackend="FILE",
-# resourceCredentialBackend="FILE",
-# targetAgentId="agent0",
-# childPath=f"/tmp/{path}",
-# mftAuthorizationToken="user token")
-# response = stub.getDirectoryResourceMetadata(request)
-# directories_data = []
-# for d in response.directories:
-
-# dpath = os.path.join(path, d.friendlyName)
-# created_time = datetime.fromtimestamp(d.createdTime)
-# # TODO MFT API doesn't report size
-# size = 0
-# directories_data.append(
-# {
-# "name": d.friendlyName,
-# "path": dpath,
-# "created_time": created_time,
-# "size": size,
-# # TODO how to handle hidden directories or directories for
-# # staging input file uploads
-# "hidden": False
-# }
-# )
-# files_data = []
-# for f in response.files:
-# user_rel_path = os.path.join(path, f.friendlyName)
-# # TODO do we need to check for broken symlinks?
-# created_time = datetime.fromtimestamp(f.createdTime)
-# # TODO get the size as well
-# size = 0
-# # full_path = datastore.path(request.user.username, user_rel_path)
-# # TODO how do we register these as data products, do we need to?
-# # data_product_uri = _get_data_product_uri(request, full_path)
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ request = MFTApi_pb2.FetchResourceMetadataRequest(
+ # resourceId="remote-ssh-dir-resource",
+ resourceId=self.resource_id,
+ resourceType="SCP",
+ # resourceToken="local-ssh-cred",
+ resourceToken=self.resource_token,
+ resourceBackend="FILE",
+ resourceCredentialBackend="FILE",
+ targetAgentId="agent0",
+ childPath=child_path,
+ mftAuthorizationToken=self.auth_token)
+ try:
+ logger.debug(f"getDirectoryResourceMetadata({request})")
+ response = stub.getDirectoryResourceMetadata(request)
+ logger.debug(f"getDirectoryResourceMetadata response={response}")
+ directories = response.directories
+ files = response.files
+ except Exception:
+ # if getting metadata for directory fails, try as file
+ # FIXME is there a better way to determine if directory or file?
+ logger.debug(f"getFileResourceMetadata({request})")
+ response = stub.getFileResourceMetadata(request)
+ logger.debug(f"getFileResourceMetadata response={response}")
+ directories = []
+ files = [response]
+ directories_data = []
+ for d in directories:
-# # data_product = request.airavata_client.getDataProduct(
-# # request.authz_token, data_product_uri)
-# # mime_type = None
-# # if 'mime-type' in data_product.productMetadata:
-# # mime_type = data_product.productMetadata['mime-type']
-# files_data.append(
-# {
-# "name": f.friendlyName,
-# "path": user_rel_path,
-# "data-product-uri": None,
-# "created_time": created_time,
-# "mime_type": None,
-# "size": size,
-# "hidden": False,
-# }
-# )
-# return directories_data, files_data
+ dpath = os.path.join(resource_path, d.friendlyName)
+ created_time = datetime.fromtimestamp(d.createdTime)
+ # TODO MFT API doesn't report size
+ size = 0
+ directories_data.append(
+ {
+ "name": d.friendlyName,
+ # path is the relative path, or at least, relative to given resource_path
+ "path": dpath,
+ # resource_path is the id or full path to the resource
+ "resource_path": d.resourcePath,
+ "created_time": created_time,
+ "size": size,
+ # TODO how to handle hidden directories or directories for
+ # staging input file uploads
+ "hidden": False
+ }
+ )
+ files_data = []
+ for f in files:
+ user_rel_path = os.path.join(resource_path, f.friendlyName)
+ # TODO do we need to check for broken symlinks?
+ created_time = datetime.fromtimestamp(f.createdTime)
+ size = f.resourceSize
+ # full_path = datastore.path(request.user.username, user_rel_path)
+ # TODO how do we register these as data products, do we need to?
+ # data_product_uri = _get_data_product_uri(request, full_path)
-# def get_file(self, request, path):
-# # FIXME remove hard coded /tmp path
-# path = f"/tmp/{path}".rstrip("/")
-# file_metadata = self._get_file(path)
-# if file_metadata is not None:
-# user_rel_path = os.path.join(path, file_metadata.friendlyName)
-# created_time = datetime.fromtimestamp(file_metadata.createdTime)
-# # TODO get the size as well
-# size = 0
+ # data_product = request.airavata_client.getDataProduct(
+ # request.authz_token, data_product_uri)
+ # mime_type = None
+ # if 'mime-type' in data_product.productMetadata:
+ # mime_type = data_product.productMetadata['mime-type']
+ files_data.append(
+ {
+ "name": f.friendlyName,
+ "path": user_rel_path,
+ "resource_path": f.resourcePath,
+ "created_time": created_time,
+ "size": size,
+ "hidden": False,
+ }
+ )
+ return directories_data, files_data
-# return {
-# "name": file_metadata.friendlyName,
-# "path": user_rel_path,
-# "data-product-uri": None,
-# "created_time": created_time,
-# "mime_type": None,
-# "size": size,
-# "hidden": False,
-# }
-# else:
-# raise ObjectDoesNotExist("User storage file path does not exist")
+ def is_file(self, resource_path):
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ # Get metadata for parent directory and see if child_path exists
+ request = MFTApi_pb2.FetchResourceMetadataRequest(
+ # resourceId="remote-ssh-dir-resource",
+ resourceId=self.resource_id,
+ resourceType="SCP",
+ # resourceToken="local-ssh-cred",
+ resourceToken=self.resource_token,
+ resourceBackend="FILE",
+ resourceCredentialBackend="FILE",
+ targetAgentId="agent0",
+ childPath=child_path,
+ mftAuthorizationToken=self.auth_token,
+ )
+ try:
+ stub.getFileResourceMetadata(request)
+ return True
+ except Exception:
+ # assume that is doesn't exist, or isn't a file
+ logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}")
+ return False
-# def _get_file(self, path):
-# with grpc.insecure_channel('localhost:7004') as channel:
-# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
-# # Get metadata for parent directory and see if child_path exists
-# request = MFTApi_pb2.FetchResourceMetadataRequest(
-# resourceId="remote-ssh-dir-resource",
-# resourceType="SCP",
-# resourceToken="local-ssh-cred",
-# resourceBackend="FILE",
-# resourceCredentialBackend="FILE",
-# targetAgentId="agent0",
-# childPath=path,
-# mftAuthorizationToken="user token")
-# try:
-# # TODO is there a better way to check if file exists than catching exception?
-# return stub.getFileResourceMetadata(request)
-# except Exception:
-# logger.exception(f"_get_file({path})")
-# return None
+ def is_dir(self, resource_path):
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ child_path = self._get_child_path(resource_path)
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ # Get metadata for parent directory and see if child_path exists
+ request = MFTApi_pb2.FetchResourceMetadataRequest(
+ # resourceId="remote-ssh-dir-resource",
+ resourceId=self.resource_id,
+ resourceType="SCP",
+ # resourceToken="local-ssh-cred",
+ resourceToken=self.resource_token,
+ resourceBackend="FILE",
+ resourceCredentialBackend="FILE",
+ targetAgentId="agent0",
+ childPath=child_path,
+ mftAuthorizationToken=self.auth_token,
+ )
+ try:
+ stub.getDirectoryResourceMetadata(request)
+ return True
+ except Exception:
+ # assume that it doesn't exist or isn't a file
+ logger.warning(f"Could not get metadata for {child_path} on {self.resource_id}")
+ return False
-# def _get_download_url(self, path):
+ def _get_child_path(self, resource_path):
+ """Convert possibly relative child path into absolute path."""
+ if not resource_path.startswith("/"):
+ # resource_path is relative, need to construct an absolute path
+ if self.resource_per_gateway:
+ resource_path = os.path.join(self.username, resource_path).rstrip("/")
+ # If there is no child path, just return none
+ if resource_path == '':
+ return None
+ logger.debug(f"figuring out resourcePath of {self.resource_id} ...")
+ with grpc.insecure_channel(self.mft_api_endpoint) as channel:
+ stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
+ request = MFTApi_pb2.FetchResourceMetadataRequest(
+ # resourceId="remote-ssh-dir-resource",
+ resourceId=self.resource_id,
+ resourceType="SCP",
+ # resourceToken="local-ssh-cred",
+ resourceToken=self.resource_token,
+ resourceBackend="FILE",
+ resourceCredentialBackend="FILE",
+ targetAgentId="agent0",
+ mftAuthorizationToken=self.auth_token)
+ response = stub.getDirectoryResourceMetadata(request)
+ logger.debug(f"metadata of {self.resource_id} is {response}")
+ return os.path.join(response.resourcePath, resource_path)
+ else:
+ # resource_path appears to be absolute path
+ return resource_path
-# with grpc.insecure_channel('localhost:7004') as channel:
-# stub = MFTApi_pb2_grpc.MFTApiServiceStub(channel)
-# download_request = MFTApi_pb2.HttpDownloadApiRequest(sourceStoreId="remote-ssh-storage",
-# sourcePath="/tmp/a.txt",
-# sourceToken="local-ssh-cred",
-# sourceType="SCP",
-# targetAgent="agent0",
-# mftAuthorizationToken="")
-# try:
-# # TODO is there a better way to check if file exists than catching exception?
-# # response stub.submitHttpDownload(request)
-# pass
-# except Exception:
-# logger.exception(f"_get_file({path})")
-# return None
+ @property
+ def auth_token(self):
+ """Instance of CredCommon.AuthToken wrapping user's access token."""
+ return CredCommon_pb2.AuthToken(userTokenAuth=CredCommon_pb2.UserTokenAuth(token=self.access_token))