You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/01/09 17:57:11 UTC
[airavata-mft] 02/02: Azure metadata, streaming transport and cli support
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 35c87225f13d30bce7e8f0edbf8b47a3d5ddacb0
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Mon Jan 9 12:56:59 2023 -0500
Azure metadata, streaming transport and cli support
---
.gitignore | 6 +-
agent/service/pom.xml | 26 +++-
.../airavata/mft/core/ConnectorResolver.java | 6 +
python-cli/mft_cli/mft_cli/main.py | 2 +-
python-cli/mft_cli/mft_cli/storage/__init__.py | 3 +
python-cli/mft_cli/mft_cli/storage/azure.py | 59 +++++++++
.../server/backend/sql/SQLResourceBackend.java | 28 ++++-
.../backend/sql/entity/AzureStorageEntity.java | 68 +++++++++++
.../sql/repository/AzureStorageRepository.java | 31 +++++
.../server/backend/sql/SQLSecretBackend.java | 27 ++---
.../backend/sql/entity/AzureSecretEntity.java | 55 +++++++++
.../sql/repository/AzureSecretRepository.java | 30 +++++
transport/azure-transport/pom.xml | 4 +-
.../azure/AzureIncomingStreamingConnector.java | 78 ++++++++++++
.../transport/azure/AzureMetadataCollector.java | 31 ++++-
.../azure/AzureOutgoingStreamingConnector.java | 78 ++++++++++++
.../mft/transport/azure/AzureReceiver.java | 134 ---------------------
.../airavata/mft/transport/azure/AzureSender.java | 102 ----------------
.../airavata/mft/transport/box/BoxReceiver.java | 2 -
.../airavata/mft/transport/box/BoxSender.java | 2 -
.../mft/transport/dropbox/DropboxReceiver.java | 2 -
.../mft/transport/dropbox/DropboxSender.java | 2 -
.../airavata/mft/transport/ftp/FTPReceiver.java | 2 -
.../airavata/mft/transport/ftp/FTPSender.java | 2 -
transport/pom.xml | 14 ---
.../mft/transport/s3/S3OutgoingConnector.java | 4 -
.../transport/swift/SwiftIncomingConnector.java | 4 -
.../transport/swift/SwiftOutgoingConnector.java | 4 -
28 files changed, 503 insertions(+), 303 deletions(-)
diff --git a/.gitignore b/.gitignore
index 861afbf..d43a85d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1208,4 +1208,8 @@ airavata-mft/
# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
-python-sdk/env
\ No newline at end of file
+python-sdk/env
+python-sdk/samples/env/*
+python-sdk/venv/*
+python-cli/venv/*
+python-cli/mft_cli/poetry.lock
\ No newline at end of file
diff --git a/agent/service/pom.xml b/agent/service/pom.xml
index 94660d9..0d1a5cb 100644
--- a/agent/service/pom.xml
+++ b/agent/service/pom.xml
@@ -37,6 +37,17 @@
<groupId>org.apache.airavata</groupId>
<artifactId>mft-agent-stub</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.54.Final</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
@@ -58,6 +69,17 @@
<groupId>org.apache.airavata</groupId>
<artifactId>mft-s3-transport</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ <version>4.1.82.Final</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
@@ -141,7 +163,7 @@
</dependency>
</dependencies>
- <build>
+ <!--build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -166,7 +188,7 @@
</executions>
</plugin>
</plugins>
- </build>
+ </build-->
<properties>
<maven.compiler.source>11</maven.compiler.source>
diff --git a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
index ef5792c..ce5abf3 100644
--- a/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
+++ b/core/src/main/java/org/apache/airavata/mft/core/ConnectorResolver.java
@@ -39,6 +39,9 @@ public final class ConnectorResolver {
case "ODATA":
className = "org.apache.airavata.mft.transport.odata.ODataIncomingConnector";
break;
+ case "AZURE":
+ className = "org.apache.airavata.mft.transport.azure.AzureIncomingStreamingConnector";
+ break;
}
if (className != null) {
@@ -59,6 +62,9 @@ public final class ConnectorResolver {
case "S3":
className = "org.apache.airavata.mft.transport.s3.S3OutgoingStreamingConnector";
break;
+ case "AZURE":
+ className = "org.apache.airavata.mft.transport.azure.AzureOutgoingStreamingConnector";
+ break;
}
diff --git a/python-cli/mft_cli/mft_cli/main.py b/python-cli/mft_cli/mft_cli/main.py
index 1accfd6..c89065f 100644
--- a/python-cli/mft_cli/mft_cli/main.py
+++ b/python-cli/mft_cli/mft_cli/main.py
@@ -102,7 +102,7 @@ def copy(source, destination):
sourceSecretId = source_secret_id,
destinationStorageId = dest_storage_id,
destinationSecretId = dest_secret_id,
- optimizeTransferPath = True)
+ optimizeTransferPath = False)
if (source_metadata.WhichOneof('metadata') == 'directory') :
if (destination[-1] != "/"):
diff --git a/python-cli/mft_cli/mft_cli/storage/__init__.py b/python-cli/mft_cli/mft_cli/storage/__init__.py
index 51fbdac..08799bc 100644
--- a/python-cli/mft_cli/mft_cli/storage/__init__.py
+++ b/python-cli/mft_cli/mft_cli/storage/__init__.py
@@ -1,6 +1,7 @@
import typer
from pick import pick
import mft_cli.storage.s3 as s3
+import mft_cli.storage.azure as azure
from airavata_mft_sdk import mft_client
from airavata_mft_sdk.common import StorageCommon_pb2
from rich.console import Console
@@ -15,6 +16,8 @@ def add_storage():
option, index = pick(options, title, indicator="=>")
if option == "S3":
s3.handle_add_storage()
+ elif option == "Azure Storage":
+ azure.handle_add_storage()
@app.command("list")
diff --git a/python-cli/mft_cli/mft_cli/storage/azure.py b/python-cli/mft_cli/mft_cli/storage/azure.py
new file mode 100644
index 0000000..c83b0bb
--- /dev/null
+++ b/python-cli/mft_cli/mft_cli/storage/azure.py
@@ -0,0 +1,59 @@
+from rich import print
+from pick import pick
+import typer
+from airavata_mft_sdk import mft_client
+from airavata_mft_sdk.azure import AzureCredential_pb2
+from airavata_mft_sdk.azure import AzureStorage_pb2
+from airavata_mft_sdk import MFTTransferApi_pb2
+from airavata_mft_sdk import MFTAgentStubs_pb2
+from airavata_mft_sdk.common import StorageCommon_pb2
+
+def handle_add_storage():
+
+ options = ["Through Azure Cli config file", "Enter manually" ]
+ option, index = pick(options, "How do you want to load credentials", indicator="=>")
+
+ if index == 1: # Manual configuration
+ connection_string = typer.prompt("Connection String")
+
+ client = mft_client.MFTClient()
+
+ azure_secret = AzureCredential_pb2.AzureSecret(connectionString = connection_string)
+ secret_wrapper = MFTAgentStubs_pb2.SecretWrapper(azure=azure_secret)
+
+ azure_storage = AzureStorage_pb2.AzureStorage()
+ storage_wrapper = MFTAgentStubs_pb2.StorageWrapper(azure=azure_storage)
+
+ direct_req = MFTAgentStubs_pb2.GetResourceMetadataRequest(resourcePath="", secret=secret_wrapper, storage=storage_wrapper)
+ resource_medata_req = MFTTransferApi_pb2.FetchResourceMetadataRequest(directRequest = direct_req)
+ metadata_resp = client.transfer_api.resourceMetadata(resource_medata_req)
+
+ container_options = ["Manually Enter"]
+
+ container_list = metadata_resp.directory.directories
+ if len(container_list) > 0:
+ for c in container_list:
+ container_options.append(c.friendlyName)
+
+ title = "Select the Container: "
+ selected_container, index = pick(container_options, title, indicator="=>")
+
+ if index == 0:
+ selected_container = typer.prompt("Enter container name ")
+ storage_name = typer.prompt("Name of the storage ", selected_container)
+
+ azure_storage_create_req = AzureStorage_pb2.AzureStorageCreateRequest(container= selected_container, name =storage_name)
+
+ created_storage = client.azure_storage_api.createAzureStorage(azure_storage_create_req)
+
+ secret_create_req= AzureCredential_pb2.AzureSecretCreateRequest(connectionString = connection_string)
+ created_secret = client.azure_secret_api.createAzureSecret(secret_create_req)
+
+ secret_for_storage_req = StorageCommon_pb2.SecretForStorage(storageId = created_storage.storageId,
+ secretId = created_secret.secretId,
+ storageType = StorageCommon_pb2.StorageType.AZURE)
+
+ client.common_api.registerSecretForStorage(secret_for_storage_req)
+
+ print("Successfully added the Azure Bucket...")
+
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
index ec9067c..df5d620 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
@@ -54,6 +54,9 @@ public class SQLResourceBackend implements ResourceBackend {
@Autowired
private S3StorageRepository s3StorageRepository;
+ @Autowired
+ private AzureStorageRepository azureStorageRepository;
+
@Autowired
private GCSStorageRepository gcsStorageRepository;
@@ -318,27 +321,42 @@ public class SQLResourceBackend implements ResourceBackend {
@Override
public AzureStorageListResponse listAzureStorage(AzureStorageListRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ AzureStorageListResponse.Builder respBuilder = AzureStorageListResponse.newBuilder();
+ List<AzureStorageEntity> all = azureStorageRepository.findAll(PageRequest.of(request.getOffset(), request.getLimit()));
+ all.forEach(ety -> respBuilder.addStorages(mapper.map(ety, AzureStorage.newBuilder().getClass())));
+ return respBuilder.build();
}
@Override
public Optional<AzureStorage> getAzureStorage(AzureStorageGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ Optional<AzureStorageEntity> entity = azureStorageRepository.findById(request.getStorageId());
+ return entity.map(e -> mapper.map(e, AzureStorage.newBuilder().getClass()).build());
}
@Override
public AzureStorage createAzureStorage(AzureStorageCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ AzureStorageEntity savedEntity = azureStorageRepository.save(mapper.map(request, AzureStorageEntity.class));
+
+ ResolveStorageEntity storageTypeEty = new ResolveStorageEntity();
+ storageTypeEty.setStorageId(savedEntity.getStorageId());
+ storageTypeEty.setStorageType(ResolveStorageEntity.StorageType.AZURE);
+ storageTypeEty.setStorageName(savedEntity.getName());
+ resolveStorageRepository.save(storageTypeEty);
+
+ return mapper.map(savedEntity, AzureStorage.newBuilder().getClass()).build();
}
@Override
public boolean updateAzureStorage(AzureStorageUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ azureStorageRepository.save(mapper.map(request, AzureStorageEntity.class));
+ return true;
}
@Override
public boolean deleteAzureStorage(AzureStorageDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ azureStorageRepository.deleteById(request.getStorageId());
+ resourceRepository.deleteByStorageIdAndStorageType(request.getStorageId(), GenericResourceEntity.StorageType.AZURE);
+ return true;
}
@Override
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java
new file mode 100644
index 0000000..8fadd5e
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/AzureStorageEntity.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.resource.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class AzureStorageEntity {
+
+ @Id
+ @Column(name = "AZURE_STORAGE_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
+ private String storageId;
+
+ @Column(name = "STORAGE_NAME")
+ private String name;
+
+ @Column(name = "CONTAINER")
+ private String container;
+
+ public String getStorageId() {
+ return storageId;
+ }
+
+ public AzureStorageEntity setStorageId(String storageId) {
+ this.storageId = storageId;
+ return this;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public AzureStorageEntity setName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public String getContainer() {
+ return container;
+ }
+
+ public AzureStorageEntity setContainer(String container) {
+ this.container = container;
+ return this;
+ }
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java
new file mode 100644
index 0000000..0a671b1
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/AzureStorageRepository.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.resource.server.backend.sql.repository;
+
+import org.apache.airavata.mft.resource.server.backend.sql.entity.AzureStorageEntity;
+import org.apache.airavata.mft.resource.server.backend.sql.entity.S3StorageEntity;
+import org.springframework.data.domain.Pageable;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.List;
+
+public interface AzureStorageRepository extends CrudRepository<AzureStorageEntity, String> {
+
+ List<AzureStorageEntity> findAll(Pageable pageable);
+
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
index 2a6cd22..8c6c356 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
@@ -27,19 +27,11 @@ import org.apache.airavata.mft.credential.stubs.s3.*;
import org.apache.airavata.mft.credential.stubs.scp.*;
import org.apache.airavata.mft.credential.stubs.swift.*;
import org.apache.airavata.mft.secret.server.backend.SecretBackend;
-import org.apache.airavata.mft.secret.server.backend.sql.entity.FTPSecretEntity;
-import org.apache.airavata.mft.secret.server.backend.sql.entity.GCSSecretEntity;
-import org.apache.airavata.mft.secret.server.backend.sql.entity.ODataSecretEntity;
-import org.apache.airavata.mft.secret.server.backend.sql.entity.S3SecretEntity;
-import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.*;
import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftAuthCredentialSecretEntity;
import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftPasswordSecretEntity;
import org.apache.airavata.mft.secret.server.backend.sql.entity.swift.SwiftSecretEntity;
-import org.apache.airavata.mft.secret.server.backend.sql.repository.FTPSecretRepository;
-import org.apache.airavata.mft.secret.server.backend.sql.repository.GCSSecretRepository;
-import org.apache.airavata.mft.secret.server.backend.sql.repository.ODataSecretRepository;
-import org.apache.airavata.mft.secret.server.backend.sql.repository.S3SecretRepository;
-import org.apache.airavata.mft.secret.server.backend.sql.repository.SCPSecretRepository;
+import org.apache.airavata.mft.secret.server.backend.sql.repository.*;
import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftAuthCredentialSecretRepository;
import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftPasswordSecretRepository;
import org.apache.airavata.mft.secret.server.backend.sql.repository.swift.SwiftSecretRepository;
@@ -63,6 +55,9 @@ public class SQLSecretBackend implements SecretBackend {
@Autowired
private S3SecretRepository s3SecretRepository;
+ @Autowired
+ private AzureSecretRepository azureSecretRepository;
+
@Autowired
private SwiftSecretRepository swiftSecretRepository;
@@ -160,22 +155,26 @@ public class SQLSecretBackend implements SecretBackend {
@Override
public Optional<AzureSecret> getAzureSecret(AzureSecretGetRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ Optional<AzureSecretEntity> secretEty = azureSecretRepository.findBySecretId(request.getSecretId());
+ return secretEty.map(azSecretEntity -> mapper.map(azSecretEntity, AzureSecret.newBuilder().getClass()).build());
}
@Override
public AzureSecret createAzureSecret(AzureSecretCreateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ AzureSecretEntity savedEntity = azureSecretRepository.save(mapper.map(request, AzureSecretEntity.class));
+ return mapper.map(savedEntity, AzureSecret.newBuilder().getClass()).build();
}
@Override
public boolean updateAzureSecret(AzureSecretUpdateRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ azureSecretRepository.save(mapper.map(request, AzureSecretEntity.class));
+ return true;
}
@Override
public boolean deleteAzureSecret(AzureSecretDeleteRequest request) throws Exception {
- throw new UnsupportedOperationException("Operation is not supported in backend");
+ azureSecretRepository.deleteById(request.getSecretId());
+ return true;
}
@Override
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java
new file mode 100644
index 0000000..c39ddd4
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/AzureSecretEntity.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class AzureSecretEntity {
+ @Id
+ @Column(name = "SECRET_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator( name = "uuid", strategy = "uuid2")
+ private String secretId;
+
+ @Column(name = "CONNECTION_STRING")
+ private String connectionString;
+
+ public String getSecretId() {
+ return secretId;
+ }
+
+ public AzureSecretEntity setSecretId(String secretId) {
+ this.secretId = secretId;
+ return this;
+ }
+
+ public String getConnectionString() {
+ return connectionString;
+ }
+
+ public AzureSecretEntity setConnectionString(String connectionString) {
+ this.connectionString = connectionString;
+ return this;
+ }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java
new file mode 100644
index 0000000..3c240a8
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/AzureSecretRepository.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.repository;
+
+import org.apache.airavata.mft.secret.server.backend.sql.entity.AzureSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.S3SecretEntity;
+import org.springframework.data.repository.CrudRepository;
+
+import java.util.Optional;
+
+public interface AzureSecretRepository extends CrudRepository<AzureSecretEntity, String> {
+
+ Optional<AzureSecretEntity> findBySecretId(String secretId);
+
+}
diff --git a/transport/azure-transport/pom.xml b/transport/azure-transport/pom.xml
index 6d05805..b6aff6e 100644
--- a/transport/azure-transport/pom.xml
+++ b/transport/azure-transport/pom.xml
@@ -36,12 +36,12 @@
<dependency>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
- <version>12.13.0</version>
+ <version>12.20.1</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-core</artifactId>
- <version>0.01-SNAPSHOT</version>
+ <version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java
new file mode 100644
index 0000000..971f708
--- /dev/null
+++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureIncomingStreamingConnector.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.transport.azure;
+
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.IncomingStreamingConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.InputStream;
+
+public class AzureIncomingStreamingConnector implements IncomingStreamingConnector {
+
+ private static final Logger logger = LoggerFactory.getLogger(AzureIncomingStreamingConnector.class);
+
+ private ConnectorConfig connectorConfig;
+ private InputStream is;
+
+ @Override
+ public void init(ConnectorConfig connectorConfig) throws Exception {
+ this.connectorConfig = connectorConfig;
+ }
+
+ @Override
+ public void complete() throws Exception {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close the input stream", e);
+ }
+ }
+ }
+
+ @Override
+ public void failed() throws Exception {
+ if (is != null) {
+ try {
+ is.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close the input stream", e);
+ }
+ }
+ }
+
+ @Override
+ public InputStream fetchInputStream() throws Exception {
+
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+ .connectionString(connectorConfig.getSecret().getAzure().getConnectionString()).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient
+ .getBlobContainerClient(connectorConfig.getStorage().getAzure().getContainer());
+
+ BlobClient blobClient = blobContainerClient.getBlobClient(connectorConfig.getResourcePath());
+
+ is = blobClient.openInputStream();
+ return is;
+ }
+}
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
index 5b08c80..0fb3f02 100644
--- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
+++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java
@@ -22,6 +22,7 @@ import com.azure.storage.blob.BlobClient;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobContainerItem;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.BlobItemProperties;
import com.azure.storage.blob.models.BlobProperties;
@@ -55,17 +56,34 @@ public class AzureMetadataCollector implements MetadataCollector {
// Azure does not have a concept called hierarchical containers. So we assume that there are no containers inside
// the given container
+
ResourceMetadata.Builder metadataBuilder = ResourceMetadata.newBuilder();
- if (!isAvailable(resourcePath)) {
- metadataBuilder.setError(MetadataFetchError.NOT_FOUND);
+
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
+
+ if (resourcePath.isEmpty() && azureStorage.getContainer().isEmpty()) { // List containers
+ PagedIterable<BlobContainerItem> blobContainerItems = blobServiceClient.listBlobContainers();
+ DirectoryMetadata.Builder parentDir = DirectoryMetadata.newBuilder();
+ parentDir.setResourcePath("");
+ parentDir.setFriendlyName("");
+
+ blobContainerItems.forEach(containerItem -> {
+ DirectoryMetadata.Builder containerDir = DirectoryMetadata.newBuilder();
+ containerDir.setFriendlyName(containerItem.getName());
+ containerDir.setResourcePath(containerItem.getName());
+ containerDir.setCreatedTime(containerItem.getProperties().getLastModified().toEpochSecond());
+ containerDir.setUpdateTime(containerItem.getProperties().getLastModified().toEpochSecond());
+ parentDir.addDirectories(containerDir);
+ });
+ metadataBuilder.setDirectory(parentDir);
+
return metadataBuilder.build();
}
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
BlobContainerClient blobContainerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer());
- if (resourcePath.isEmpty()) { // List the container
+ if (resourcePath.isEmpty()) { // List inside the container
PagedIterable<BlobItem> blobItems = blobContainerClient.listBlobs();
DirectoryMetadata.Builder directoryBuilder = DirectoryMetadata.newBuilder();
blobItems.forEach(blobItem -> {
@@ -89,6 +107,11 @@ public class AzureMetadataCollector implements MetadataCollector {
} else { // If resource is a file
+ if (!isAvailable(resourcePath)) {
+ metadataBuilder.setError(MetadataFetchError.NOT_FOUND);
+ return metadataBuilder.build();
+ }
+
BlobClient blobClient = blobContainerClient.getBlobClient(resourcePath);
FileMetadata.Builder fileBuilder = FileMetadata.newBuilder();
BlobProperties properties = blobClient.getProperties();
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java
new file mode 100644
index 0000000..45ca17d
--- /dev/null
+++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureOutgoingStreamingConnector.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.transport.azure;
+
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.OutgoingStreamingConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.OutputStream;
+
+public class AzureOutgoingStreamingConnector implements OutgoingStreamingConnector {
+
+ private static final Logger logger = LoggerFactory.getLogger(AzureOutgoingStreamingConnector.class);
+
+ private ConnectorConfig connectorConfig;
+ private OutputStream os;
+
+ @Override
+ public void init(ConnectorConfig connectorConfig) throws Exception {
+ this.connectorConfig = connectorConfig;
+ }
+
+ @Override
+ public void complete() throws Exception {
+ if (os != null) {
+ try {
+ os.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close the output stream", e);
+ }
+ }
+ }
+
+ @Override
+ public void failed() throws Exception {
+ if (os != null) {
+ try {
+ os.close();
+ } catch (Exception e) {
+ logger.warn("Failed to close the output stream", e);
+ }
+ }
+ }
+
+ @Override
+ public OutputStream fetchOutputStream() throws Exception {
+
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+ .connectionString(connectorConfig.getSecret().getAzure().getConnectionString()).buildClient();
+ BlobContainerClient blobContainerClient = blobServiceClient
+ .getBlobContainerClient(connectorConfig.getStorage().getAzure().getContainer());
+
+ BlobClient blobClient = blobContainerClient.getBlobClient(connectorConfig.getResourcePath());
+
+ os = blobClient.getBlockBlobClient().getBlobOutputStream(true);
+ return os;
+ }
+}
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java
deleted file mode 100644
index 6d752ff..0000000
--- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureReceiver.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.airavata.mft.transport.azure;
-
-import com.azure.storage.blob.BlobClient;
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.specialized.BlobInputStream;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.api.Connector;
-import org.apache.airavata.mft.credential.stubs.azure.AzureSecret;
-import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest;
-import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.OutputStream;
-
-public class AzureReceiver implements Connector {
-
- private static final Logger logger = LoggerFactory.getLogger(AzureReceiver.class);
-
- private boolean initialized = false;
- private BlobContainerClient containerClient;
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
-
- @Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
- this.initialized = true;
-
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
- }
-
- @Override
- public void destroy() {
-
- }
-
- private void checkInitialized() {
- if (!initialized) {
- throw new IllegalStateException("Azure Receiver is not initialized");
- }
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception {
- logger.info("Starting azure receive for remote server for transfer {}", context.getTransferId());
- /*
- checkInitialized();
-
- ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
- GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder()
- .setResourceId(resourceId).build());
-
- if (resource.getStorageCase() != GenericResource.StorageCase.AZURESTORAGE) {
- logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId);
- throw new Exception("Invalid storage type specified for resource " + resourceId);
- }
- AzureStorage azureStorage = resource.getAzureStorage();
-
- SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
- AzureSecret azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
- this.containerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer());
-
- BlobClient blobClient = containerClient.getBlobClient(resource.getFile().getResourcePath());
- BlobInputStream blobInputStream = blobClient.openInputStream();
-
- OutputStream streamOs = context.getStreamBuffer().getOutputStream();
-
- long fileSize = context.getMetadata().getResourceSize();
-
- byte[] buf = new byte[1024];
- while (true) {
- int bufSize = 0;
-
- if (buf.length < fileSize) {
- bufSize = buf.length;
- } else {
- bufSize = (int) fileSize;
- }
- bufSize = blobInputStream.read(buf, 0, bufSize);
-
- if (bufSize < 0) {
- break;
- }
-
- streamOs.write(buf, 0, bufSize);
- streamOs.flush();
-
- fileSize -= bufSize;
- if (fileSize == 0L)
- break;
- }
-
- streamOs.close();
- logger.info("Completed azure receive for remote server for transfer {}", context.getTransferId());
-
- */
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken,
- ConnectorContext context) throws Exception {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java
deleted file mode 100644
index 7b578b0..0000000
--- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureSender.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.airavata.mft.transport.azure;
-
-import com.azure.storage.blob.BlobContainerClient;
-import com.azure.storage.blob.BlobServiceClient;
-import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.specialized.BlockBlobClient;
-import org.apache.airavata.mft.common.AuthToken;
-import org.apache.airavata.mft.core.ConnectorContext;
-import org.apache.airavata.mft.core.api.Connector;
-import org.apache.airavata.mft.credential.stubs.azure.AzureSecret;
-import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest;
-import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class AzureSender implements Connector {
-
- private static final Logger logger = LoggerFactory.getLogger(AzureSender.class);
-
- private boolean initialized = false;
- BlobContainerClient containerClient;
-
- private String resourceServiceHost;
- private int resourceServicePort;
- private String secretServiceHost;
- private int secretServicePort;
-
- @Override
- public void init(String resourceServiceHost, int resourceServicePort, String secretServiceHost, int secretServicePort) throws Exception {
- this.initialized = true;
-
- this.resourceServiceHost = resourceServiceHost;
- this.resourceServicePort = resourceServicePort;
- this.secretServiceHost = secretServiceHost;
- this.secretServicePort = secretServicePort;
- }
-
- @Override
- public void destroy() {
-
- }
-
- private void checkInitialized() {
- if (!initialized) {
- throw new IllegalStateException("Azure Sender is not initialized");
- }
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String credentialToken, ConnectorContext context) throws Exception {
- logger.info("Starting Azure send for remote server for transfer {}", context.getTransferId());
- /*
- checkInitialized();
-
- ResourceServiceClient resourceClient = ResourceServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
- GenericResource resource = resourceClient.get().getGenericResource(GenericResourceGetRequest.newBuilder()
- .setResourceId(resourceId).build());
-
- if (resource.getStorageCase() != GenericResource.StorageCase.AZURESTORAGE) {
- logger.error("Invalid storage type {} specified for resource {}", resource.getStorageCase(), resourceId);
- throw new Exception("Invalid storage type specified for resource " + resourceId);
- }
- AzureStorage azureStorage = resource.getAzureStorage();
-
- SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
- AzureSecret azureSecret = secretClient.azure().getAzureSecret(AzureSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
-
- BlobServiceClient blobServiceClient = new BlobServiceClientBuilder().connectionString(azureSecret.getConnectionString()).buildClient();
- this.containerClient = blobServiceClient.getBlobContainerClient(azureStorage.getContainer());
-
- BlockBlobClient blockBlobClient = containerClient.getBlobClient(resource.getFile().getResourcePath()).getBlockBlobClient();
- blockBlobClient.upload(context.getStreamBuffer().getInputStream(), context.getMetadata().getResourceSize(), true);
- logger.info("Completed Azure send for remote server for transfer {}", context.getTransferId());
-
- */
- }
-
- @Override
- public void startStream(AuthToken authToken, String resourceId, String childResourcePath, String credentialToken,
- ConnectorContext context) throws Exception {
- throw new UnsupportedOperationException();
- }
-}
diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java
index 5103126..9dbd094 100644
--- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java
+++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxReceiver.java
@@ -25,8 +25,6 @@ import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.box.BoxSecret;
import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java
index 10b80cb..70724e8 100644
--- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java
+++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxSender.java
@@ -25,8 +25,6 @@ import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.box.BoxSecret;
import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java
index 9d96972..fab26fe 100644
--- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java
+++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxReceiver.java
@@ -24,8 +24,6 @@ import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java
index 0000522..92cf62a 100644
--- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java
+++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxSender.java
@@ -26,8 +26,6 @@ import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
index 7aee312..68619ed 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPReceiver.java
@@ -23,8 +23,6 @@ import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest;
import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.commons.net.ftp.FTPClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
index 4960243..3b24c7a 100644
--- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
+++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPSender.java
@@ -23,8 +23,6 @@ import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest;
import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.commons.net.ftp.FTPClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/pom.xml b/transport/pom.xml
index 2cc9729..4df6b25 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -44,18 +44,4 @@
<module>swift-transport</module>
<module>odata-transport</module>
</modules>
- <dependencies>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-resource-service-client</artifactId>
- <version>0.01-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.airavata</groupId>
- <artifactId>mft-secret-service-client</artifactId>
- <version>0.01-SNAPSHOT</version>
- </dependency>
- </dependencies>
-
-
</project>
\ No newline at end of file
diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
index f6eebaf..860a82e 100644
--- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
+++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3OutgoingConnector.java
@@ -12,12 +12,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.OutgoingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.s3.S3Secret;
import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
index e4555c5..e578b5f 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftIncomingConnector.java
@@ -21,12 +21,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.IncomingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.jclouds.ContextBuilder;
import org.jclouds.http.options.GetOptions;
import org.jclouds.openstack.keystone.auth.config.CredentialTypes;
diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
index 52f7a50..310143b 100644
--- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
+++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftOutgoingConnector.java
@@ -21,12 +21,8 @@ import org.apache.airavata.mft.core.api.ConnectorConfig;
import org.apache.airavata.mft.core.api.OutgoingChunkedConnector;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
-import org.apache.airavata.mft.resource.client.StorageServiceClient;
-import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
-import org.apache.airavata.mft.secret.client.SecretServiceClient;
-import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.jclouds.ContextBuilder;
import org.jclouds.io.payloads.InputStreamPayload;
import org.jclouds.openstack.keystone.auth.config.CredentialTypes;