You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2022/12/21 06:38:53 UTC
[airavata-mft] branch master updated: Python CLI with S3 endpoint configuration support
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new d5cfc1d Python CLI with S3 endpoint configuration support
d5cfc1d is described below
commit d5cfc1d63198db7907c4f95a5cdd40f6ad51edcf
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Dec 21 01:38:45 2022 -0500
Python CLI with S3 endpoint configuration support
---
python-cli/README.md | 28 +++++-
python-cli/mft_cli/README.md | 1 +
.../mft_cli/mft_cli}/__init__.py | 0
python-cli/mft_cli/mft_cli/main.py | 23 +----
python-cli/mft_cli/mft_cli/storage/__init__.py | 18 ++++
python-cli/mft_cli/mft_cli/storage/s3.py | 103 +++++++++++++++++++++
python-cli/mft_cli/pyproject.toml | 19 ++++
python-sdk/src/airavata_mft_sdk/mft_client.py | 7 +-
8 files changed, 173 insertions(+), 26 deletions(-)
diff --git a/python-cli/README.md b/python-cli/README.md
index 0d8ae5c..200641d 100644
--- a/python-cli/README.md
+++ b/python-cli/README.md
@@ -1,6 +1,28 @@
-python3.10 -m venv venv
+### MFT-Cli Build Instructions
+
+Install Poetry
+```
+python3.10 -m venv venv ## Use python 3.10.6 or higher
source venv/bin/activate
pip install poetry
-cd mft-cli
+pip install pick
+```
+
+Load Poetry shell
+```
+cd mft_cli
poetry shell
-poetry install
\ No newline at end of file
+```
+
+Install dependencies
+```
+pip install grpcio==1.46.3
+pip install grpcio-tools==1.46.3
+pip install airavata_mft_sdk==0.0.1-alpha15
+```
+
+Build the binary
+```
+poetry install
+mft-cli
+```
\ No newline at end of file
diff --git a/python-cli/mft_cli/README.md b/python-cli/mft_cli/README.md
new file mode 100644
index 0000000..bb6056f
--- /dev/null
+++ b/python-cli/mft_cli/README.md
@@ -0,0 +1 @@
+# Airavata MFT Command Line Client
\ No newline at end of file
diff --git a/python-sdk/src/airavata_mft_sdk/resource/__init__.py b/python-cli/mft_cli/mft_cli/__init__.py
similarity index 100%
rename from python-sdk/src/airavata_mft_sdk/resource/__init__.py
rename to python-cli/mft_cli/mft_cli/__init__.py
diff --git a/python-cli/mft_cli/mft_cli/main.py b/python-cli/mft_cli/mft_cli/main.py
index 3c8e134..9dc6534 100644
--- a/python-cli/mft_cli/mft_cli/main.py
+++ b/python-cli/mft_cli/mft_cli/main.py
@@ -1,24 +1,11 @@
import typer
+import mft_cli.storage
app = typer.Typer()
-@app.callback()
-def callback():
- """
- Awesome Portal Gun
- """
+storage_app = typer.Typer()
+app.add_typer(mft_cli.storage.app, name="storage")
-@app.command()
-def shoot():
- """
- Shoot the portal gun
- """
- typer.echo("Shooting portal gun")
-
-@app.command()
-def load():
- """
- Load the portal gun
- """
- typer.echo("Loading portal gun")
\ No newline at end of file
+if __name__ == "__main__":
+ app()
\ No newline at end of file
diff --git a/python-cli/mft_cli/mft_cli/storage/__init__.py b/python-cli/mft_cli/mft_cli/storage/__init__.py
new file mode 100644
index 0000000..70067e3
--- /dev/null
+++ b/python-cli/mft_cli/mft_cli/storage/__init__.py
@@ -0,0 +1,18 @@
+import typer
+from pick import pick
+import mft_cli.storage.s3 as s3
+
+app = typer.Typer()
+
+@app.command("add")
+def add_storage():
+ title = "Select storage type: "
+ options = ["S3", "Google Cloud Storage (GCS)", "Azure Storage", "Openstack SWIFT", "SCP", "FTP", "Box", "DropBox", "OData", "Agent" ]
+ option, index = pick(options, title, indicator="=>")
+ if option == "S3":
+ s3.handle_add_storage()
+
+
+@app.command("list")
+def list_storage():
+ print("Listing storages")
diff --git a/python-cli/mft_cli/mft_cli/storage/s3.py b/python-cli/mft_cli/mft_cli/storage/s3.py
new file mode 100644
index 0000000..cfe1b1b
--- /dev/null
+++ b/python-cli/mft_cli/mft_cli/storage/s3.py
@@ -0,0 +1,103 @@
+from rich import print
+from pick import pick
+import typer
+from airavata_mft_sdk import mft_client
+from airavata_mft_sdk.s3 import S3Credential_pb2
+from airavata_mft_sdk.s3 import S3Storage_pb2
+from airavata_mft_sdk import MFTTransferApi_pb2
+from airavata_mft_sdk import MFTAgentStubs_pb2
+from airavata_mft_sdk.common import StorageCommon_pb2
+import configparser
+import os
+
+def handle_add_storage():
+
+ session_token = ""
+ aws_regions = ["us-east-2", "us-east-1", "us-west-1", "us-west-2", "af-south-1", "ap-east-1", "ap-east-1",
+ "ap-southeast-3", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1",
+ "ap-southeast-2", "ap-northeast-1", "ca-central-1", "cn-north-1", "cn-northwest-1", "eu-central-1",
+ "eu-west-1", "eu-west-2", "eu-south-1", "eu-west-3", "eu-north-1", "eu-south-2", "eu-central-2",
+ "sa-east-1", "me-south-1", "me-central-1", "us-gov-east-1", "us-gov-west-1"]
+
+ options = ["Through AWS Cli config file", "Enter manually" ]
+ option, index = pick(options, "How do you want to load credentials", indicator="=>")
+
+ if index == 1: # Manual configuration
+ client_id = typer.prompt("Access Key ID")
+ client_secret = typer.prompt("Secret Access Key")
+ has_session_token = typer.confirm("Do you have a session token?", False)
+ if has_session_token:
+ session_token = typer.prompt("Session Token")
+
+ is_aws = typer.confirm("Is this an AWS S3 bucket?", True)
+
+ if is_aws:
+ region, index = pick(aws_regions, "Select the AWS Region", indicator="=>")
+ endpoint = "https://s3." + region + ".amazonaws.com"
+
+ else: # If endpoint is a S3 compatible endpoint
+ endpoint = typer.prompt("What is the S3 endpoint URL?")
+ region = typer.prompt("What is the region of the bucket?")
+
+ else: # Loading credentials from the aws cli config file
+ config = configparser.RawConfigParser()
+ path = os.path.join(os.path.expanduser('~'), '.aws/credentials')
+ config.read(path)
+ cred_sections = config.sections()
+
+ if len(cred_sections) > 1:
+ section, index = pick(cred_sections, "Which section do you need to use?", indicator="=>")
+ elif len(cred_sections) == 1:
+ section = cred_sections[0]
+ else:
+ print("No credential found in ~/.aws/credentials file")
+ exit()
+
+ client_id = config.get(section, 'aws_access_key_id')
+ client_secret = config.get(section, 'aws_secret_access_key')
+ if config.has_option(section, 'aws_session_token'):
+ session_token = config.get(section, 'aws_session_token')
+
+ region, index = pick(aws_regions, "Select the AWS Region", indicator="=>")
+ endpoint = "https://s3." + region + ".amazonaws.com"
+
+ client = mft_client.MFTClient()
+
+ s3_secret = S3Credential_pb2.S3Secret(accessKey=client_id, secretKey=client_secret, sessionToken = session_token)
+ secret_wrapper = MFTAgentStubs_pb2.SecretWrapper(s3=s3_secret)
+
+ s3_storage = S3Storage_pb2.S3Storage(endpoint=endpoint, region=region)
+ storage_wrapper = MFTAgentStubs_pb2.StorageWrapper(s3=s3_storage)
+
+ direct_req = MFTAgentStubs_pb2.GetResourceMetadataRequest(resourcePath="", secret=secret_wrapper, storage=storage_wrapper)
+ resource_medata_req = MFTTransferApi_pb2.FetchResourceMetadataRequest(directRequest = direct_req)
+ metadata_resp = client.transfer_api.resourceMetadata(resource_medata_req)
+
+ bucket_options = ["Manually Enter"]
+
+ bucket_list = metadata_resp.directory.directories
+ if len(bucket_list) > 0:
+ for b in bucket_list:
+ bucket_options.append(b.friendlyName)
+
+ title = "Select the Bucket: "
+ selected_bucket, index = pick(bucket_options, title, indicator="=>")
+ if index == 0:
+ selected_bucket = typer.prompt("Enter bucket name ")
+ storage_name = typer.prompt("Name of the storage ", selected_bucket)
+
+ s3_storage.bucketName = selected_bucket
+ s3_storage.name = storage_name
+
+ created_storage = client.s3_storage_api.createS3Storage(s3_storage)
+
+ secret_create_req= S3Credential_pb2.S3SecretCreateRequest(accessKey=client_id, secretKey=client_secret)
+ created_secret = client.s3_secret_api.createS3Secret(secret_create_req)
+
+ secret_for_storage_req = StorageCommon_pb2.SecretForStorage(storageId = created_storage.storageId,
+ secretId = created_secret.secretId,
+ storageType = StorageCommon_pb2.StorageType.S3)
+
+ client.common_api.registerSecretForStorage(secret_for_storage_req)
+
+ print("Successfully added the S3 Bucket...")
diff --git a/python-cli/mft_cli/pyproject.toml b/python-cli/mft_cli/pyproject.toml
new file mode 100644
index 0000000..efc0fad
--- /dev/null
+++ b/python-cli/mft_cli/pyproject.toml
@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "mft-cli"
+version = "0.1.0"
+description = "Command Line Client for Airavata MFT data transfer framework"
+authors = ["Dimuthu Wannipurage <di...@gmail.com>"]
+readme = "README.md"
+
+[tool.poetry.scripts]
+mft-cli = "mft_cli.main:app"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+typer = {extras = ["all"], version = "^0.7.0"}
+pick = {version= "2.2.0"}
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/python-sdk/src/airavata_mft_sdk/mft_client.py b/python-sdk/src/airavata_mft_sdk/mft_client.py
index 56f362c..498da53 100644
--- a/python-sdk/src/airavata_mft_sdk/mft_client.py
+++ b/python-sdk/src/airavata_mft_sdk/mft_client.py
@@ -1,6 +1,5 @@
import grpc
import airavata_mft_sdk.MFTTransferApi_pb2_grpc as transfer_grpc
-from airavata_mft_sdk.resource import ResourceService_pb2_grpc
from airavata_mft_sdk.azure import AzureStorageService_pb2_grpc
from airavata_mft_sdk.box import BoxStorageService_pb2_grpc
from airavata_mft_sdk.dropbox import DropboxStorageService_pb2_grpc
@@ -9,7 +8,7 @@ from airavata_mft_sdk.gcs import GCSStorageService_pb2_grpc
from airavata_mft_sdk.local import LocalStorageService_pb2_grpc
from airavata_mft_sdk.s3 import S3StorageService_pb2_grpc
from airavata_mft_sdk.scp import SCPStorageService_pb2_grpc
-from airavata_mft_sdk.resourcesecretmap import StorageSecretMap_pb2_grpc
+from airavata_mft_sdk.common import StorageCommon_pb2_grpc
from airavata_mft_sdk.azure import AzureSecretService_pb2_grpc
@@ -40,7 +39,6 @@ class MFTClient:
if (not resource_service_secured):
self.resource_channel = grpc.insecure_channel('{}:{}'.format(resource_service_host, resource_service_port))
# TODO implement secure channel
- self.resource_api = ResourceService_pb2_grpc.GenericResourceServiceStub(self.resource_channel)
self.azure_storage_api = AzureStorageService_pb2_grpc.AzureStorageServiceStub(self.resource_channel)
self.box_storage_api = BoxStorageService_pb2_grpc.BoxStorageServiceStub(self.resource_channel)
self.dropbox_storage_api = DropboxStorageService_pb2_grpc.DropboxStorageServiceStub(self.resource_channel)
@@ -49,8 +47,7 @@ class MFTClient:
self.local_storage_api = LocalStorageService_pb2_grpc.LocalStorageServiceStub(self.resource_channel)
self.s3_storage_api = S3StorageService_pb2_grpc.S3StorageServiceStub(self.resource_channel)
self.scp_storage_api = SCPStorageService_pb2_grpc.SCPStorageServiceStub(self.resource_channel)
- self.storage_secret_map_api = StorageSecretMap_pb2_grpc.StorageSecretServiceStub(self.resource_channel)
-
+ self.common_api = StorageCommon_pb2_grpc.StorageCommonServiceStub(self.resource_channel)
if (not secret_service_secured):
self.secret_channel = grpc.insecure_channel('{}:{}'.format(secret_service_host, secret_service_port))