You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/09/02 02:25:59 UTC
[airavata-mft] 01/01: Enabling path style Access to custom S3 endpoint
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch ISSUE-108-Path-style
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 23ecf9dd2cc4e15f25df45c90285014bedbf07af
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Sat Sep 2 07:55:51 2023 +0530
Enabling path style Access to custom S3 endpoint
---
python-cli/mft_cli/airavata_mft_cli/storage/s3.py | 5 ++-
python-cli/mft_cli/pyproject.toml | 4 +--
python-sdk/setup.cfg | 2 +-
.../src/airavata_mft_sdk/s3/S3Storage_pb2.py | 38 +++++++++++-----------
.../server/backend/sql/entity/S3StorageEntity.java | 11 +++++++
.../stub/src/main/proto/s3/S3Storage.proto | 3 ++
.../apache/airavata/mft/transport/s3/S3Util.java | 12 +++++--
7 files changed, 49 insertions(+), 26 deletions(-)
diff --git a/python-cli/mft_cli/airavata_mft_cli/storage/s3.py b/python-cli/mft_cli/airavata_mft_cli/storage/s3.py
index 770d2fd..c6651b8 100644
--- a/python-cli/mft_cli/airavata_mft_cli/storage/s3.py
+++ b/python-cli/mft_cli/airavata_mft_cli/storage/s3.py
@@ -43,6 +43,8 @@ def handle_add_storage():
options = ["Through AWS Cli config file", "Enter manually" ]
option, index = pick(options, "How do you want to load credentials", indicator="=>")
+ enable_path_style_access = False
+
if index == 1: # Manual configuration
client_id = typer.prompt("Access Key ID")
client_secret = typer.prompt("Secret Access Key")
@@ -59,6 +61,7 @@ def handle_add_storage():
else: # If endpoint is a S3 compatible endpoint
endpoint = typer.prompt("What is the S3 endpoint URL?")
region = typer.prompt("What is the region of the bucket?")
+ enable_path_style_access = typer.confirm("Enable Path Style Access?", False)
else: # Loading credentials from the aws cli config file
config = configparser.RawConfigParser()
@@ -93,7 +96,7 @@ def handle_add_storage():
s3_secret = S3Credential_pb2.S3Secret(accessKey=client_id, secretKey=client_secret, sessionToken = session_token)
secret_wrapper = MFTAgentStubs_pb2.SecretWrapper(s3=s3_secret)
- s3_storage = S3Storage_pb2.S3Storage(endpoint=endpoint, region=region)
+ s3_storage = S3Storage_pb2.S3Storage(endpoint=endpoint, region=region, enablePathStyleAccess=enable_path_style_access)
storage_wrapper = MFTAgentStubs_pb2.StorageWrapper(s3=s3_storage)
direct_req = MFTAgentStubs_pb2.GetResourceMetadataRequest(resourcePath="", secret=secret_wrapper, storage=storage_wrapper)
diff --git a/python-cli/mft_cli/pyproject.toml b/python-cli/mft_cli/pyproject.toml
index 9ec3419..924f904 100644
--- a/python-cli/mft_cli/pyproject.toml
+++ b/python-cli/mft_cli/pyproject.toml
@@ -18,7 +18,7 @@
[tool.poetry]
name = "airavata-mft-cli"
-version = "0.1.17"
+version = "0.1.18"
description = "Command Line Client for Apache Airavata MFT data transfer software"
authors = [
"Dimuthu Wannipurage <dw...@iu.edu>",
@@ -40,7 +40,7 @@ typer = {extras = ["all"], version = "^0.7.0"}
pick = {version= "2.2.0"}
grpcio= [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}]
grpcio-tools = [{version="1.46.3", markers = "platform_machine != 'arm64'"},{version="1.47.0rc1", markers = "platform_machine == 'arm64'"}]
-airavata-mft-sdk = "0.0.1a33"
+airavata-mft-sdk = "0.0.1a34"
pandas = "^2.0.3"
[build-system]
diff --git a/python-sdk/setup.cfg b/python-sdk/setup.cfg
index 9d72928..2f1caa5 100644
--- a/python-sdk/setup.cfg
+++ b/python-sdk/setup.cfg
@@ -16,7 +16,7 @@
# under the License.
[metadata]
name = airavata_mft_sdk
-version = 0.0.1-alpha33
+version = 0.0.1-alpha34
author = Airavata MFT Developers
author_email = dev@airavata.apache.org
description = Python SDK for Apache Airavata Managed File Transfers (MFT)
diff --git a/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py b/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py
index f510529..3d3fcef 100644
--- a/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py
+++ b/python-sdk/src/airavata_mft_sdk/s3/S3Storage_pb2.py
@@ -14,7 +14,7 @@ _sym_db = _symbol_database.Default()
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12s3/S3Storage.proto\x12\x31org.apache.airavata.mft.resource.stubs.s3.storage\"r\n\tS3Storage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x12\n\nbucketName\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0e\n\x06useTLS\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\"5\n\x14S3StorageListRequest\x12\x0e\n\x06offset\x18\x01 \x01(\x05\x12\r\n\x05limit\x18\x02 \x01(\x05\"g\n\x15 [...]
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12s3/S3Storage.proto\x12\x31org.apache.airavata.mft.resource.stubs.s3.storage\"\x91\x01\n\tS3Storage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x12\n\nbucketName\x18\x02 \x01(\t\x12\x0e\n\x06region\x18\x03 \x01(\t\x12\x10\n\x08\x65ndpoint\x18\x04 \x01(\t\x12\x0e\n\x06useTLS\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x1d\n\x15\x65nablePathStyleAccess\x18\x07 \x01(\x08\"5\n\x14S3StorageListRequest\x12\x0e\n\x06offse [...]
@@ -94,22 +94,22 @@ if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'P\001'
- _S3STORAGE._serialized_start=73
- _S3STORAGE._serialized_end=187
- _S3STORAGELISTREQUEST._serialized_start=189
- _S3STORAGELISTREQUEST._serialized_end=242
- _S3STORAGELISTRESPONSE._serialized_start=244
- _S3STORAGELISTRESPONSE._serialized_end=347
- _S3STORAGEGETREQUEST._serialized_start=349
- _S3STORAGEGETREQUEST._serialized_end=389
- _S3STORAGECREATEREQUEST._serialized_start=391
- _S3STORAGECREATEREQUEST._serialized_end=518
- _S3STORAGEUPDATEREQUEST._serialized_start=520
- _S3STORAGEUPDATEREQUEST._serialized_end=647
- _S3STORAGEUPDATERESPONSE._serialized_start=649
- _S3STORAGEUPDATERESPONSE._serialized_end=693
- _S3STORAGEDELETEREQUEST._serialized_start=695
- _S3STORAGEDELETEREQUEST._serialized_end=738
- _S3STORAGEDELETERESPONSE._serialized_start=740
- _S3STORAGEDELETERESPONSE._serialized_end=781
+ _S3STORAGE._serialized_start=74
+ _S3STORAGE._serialized_end=219
+ _S3STORAGELISTREQUEST._serialized_start=221
+ _S3STORAGELISTREQUEST._serialized_end=274
+ _S3STORAGELISTRESPONSE._serialized_start=276
+ _S3STORAGELISTRESPONSE._serialized_end=379
+ _S3STORAGEGETREQUEST._serialized_start=381
+ _S3STORAGEGETREQUEST._serialized_end=421
+ _S3STORAGECREATEREQUEST._serialized_start=424
+ _S3STORAGECREATEREQUEST._serialized_end=582
+ _S3STORAGEUPDATEREQUEST._serialized_start=585
+ _S3STORAGEUPDATEREQUEST._serialized_end=743
+ _S3STORAGEUPDATERESPONSE._serialized_start=745
+ _S3STORAGEUPDATERESPONSE._serialized_end=789
+ _S3STORAGEDELETEREQUEST._serialized_start=791
+ _S3STORAGEDELETEREQUEST._serialized_end=834
+ _S3STORAGEDELETERESPONSE._serialized_start=836
+ _S3STORAGEDELETERESPONSE._serialized_end=877
# @@protoc_insertion_point(module_scope)
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java
index 402ff06..0698aea 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/S3StorageEntity.java
@@ -48,6 +48,9 @@ public class S3StorageEntity {
@Column(name = "USE_TLS")
private boolean useTLS;
+ @Column(name = "PATH_STYLE_ACCESS")
+ private boolean enablePathStyleAccess = false;
+
public String getStorageId() {
return storageId;
}
@@ -95,4 +98,12 @@ public class S3StorageEntity {
public void setName(String name) {
this.name = name;
}
+
+ public boolean isEnablePathStyleAccess() {
+ return enablePathStyleAccess;
+ }
+
+ public void setEnablePathStyleAccess(boolean enablePathStyleAccess) {
+ this.enablePathStyleAccess = enablePathStyleAccess;
+ }
}
diff --git a/services/resource-service/stub/src/main/proto/s3/S3Storage.proto b/services/resource-service/stub/src/main/proto/s3/S3Storage.proto
index 76396ac..8a026d3 100644
--- a/services/resource-service/stub/src/main/proto/s3/S3Storage.proto
+++ b/services/resource-service/stub/src/main/proto/s3/S3Storage.proto
@@ -27,6 +27,7 @@ message S3Storage {
string endpoint = 4;
bool useTLS = 5;
string name = 6;
+ bool enablePathStyleAccess = 7;
}
message S3StorageListRequest {
@@ -49,6 +50,7 @@ message S3StorageCreateRequest {
string endpoint = 4;
bool useTLS = 5;
string name = 6;
+ bool enablePathStyleAccess = 7;
}
message S3StorageUpdateRequest {
@@ -58,6 +60,7 @@ message S3StorageUpdateRequest {
string endpoint = 4;
bool useTLS = 5;
string name = 6;
+ bool enablePathStyleAccess = 7;
}
message S3StorageUpdateResponse {
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java
index 9495ec8..8f026d7 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3Util.java
@@ -97,13 +97,19 @@ public class S3Util {
ClientConfiguration clientConfiguration = new ClientConfiguration();
clientConfiguration.setUseTcpKeepAlive(true);
- AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
+ AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder.standard()
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
s3Storage.getEndpoint(), s3Storage.getRegion()))
.withCredentials(new AWSStaticCredentialsProvider(awsCreds))
.withClientConfiguration(clientConfiguration)
- .disableChunkedEncoding()
- .build();
+ .enablePathStyleAccess()
+ .disableChunkedEncoding();
+
+ if (s3Storage.getEnablePathStyleAccess()) {
+ amazonS3ClientBuilder = amazonS3ClientBuilder.enablePathStyleAccess();
+ }
+
+ AmazonS3 s3Client = amazonS3ClientBuilder.build();
s3ClientCache.get().put(secretKey, s3Client);
return s3Client;