You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/31 08:59:11 UTC

[airavata-mft] branch master updated (1c8c495 -> aff6236)

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git.


    from 1c8c495  String id auto generation for entities
     new c4417d1  Upgrading spring version
     new aff6236  Implementing a SQL backend for the Secret service and end to end workflow for scp transfers

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/airavata/mft/agent/ConsulTester.java    | 10 ++--
 .../org/apache/airavata/mft/agent/MFTAgent.java    |  5 +-
 pom.xml                                            |  4 +-
 services/pom.xml                                   |  6 +++
 .../server/backend/sql/SQLResourceBackend.java     | 13 +++--
 .../backend/sql/entity/LocalResourceEntity.java    |  1 +
 .../backend/sql/entity/SCPResourceEntity.java      |  8 +--
 services/secret-service/server/pom.xml             |  6 +++
 .../airavata/mft/secret}/server/AppConfig.java     | 11 ++--
 .../secret/server/SecretServiceApplication.java    |  2 +
 .../mft/secret/server/backend/SecretBackend.java}  | 12 +++--
 .../server/backend/sql/SQLSecretBackend.java       | 59 ++++++++++++++++++++
 .../backend/sql/entity/SCPSecretEntity.java}       | 63 +++++++++++++++-------
 .../backend/sql/repository/SecretRepository.java}  |  8 +--
 .../server/handler/SecretServiceHandler.java       | 46 ++++++++++++----
 .../stub/src/main/proto/SecretService.proto        | 54 +++++++++++++++++--
 transport/pom.xml                                  | 12 +++++
 .../mft/transport/scp/SCPMetadataCollector.java    | 23 ++++++--
 .../airavata/mft/transport/scp/SCPReceiver.java    | 27 +++++++---
 .../airavata/mft/transport/scp/SCPSender.java      | 28 +++++++---
 .../mft/transport/scp/SCPTransportUtil.java        | 30 +++++++++++
 21 files changed, 344 insertions(+), 84 deletions(-)
 copy services/{resource-service/server/src/main/java/org/apache/airavata/mft/resource => secret-service/server/src/main/java/org/apache/airavata/mft/secret}/server/AppConfig.java (76%)
 copy services/{resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/LocalResourceRepository.java => secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java} (66%)
 create mode 100644 services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
 copy services/{resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPStorageEntity.java => secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java} (50%)
 copy services/{resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/SCPStorageRepository.java => secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java} (74%)


[airavata-mft] 01/02: Upgrading spring version

Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git

commit c4417d18ef56af9b6f4825ab2bbf8fd75969ad26
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Dec 31 03:57:46 2019 -0500

    Upgrading spring version
---
 pom.xml          | 4 ++--
 services/pom.xml | 6 ++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/pom.xml b/pom.xml
index ad5b7fe..3ade4e2 100755
--- a/pom.xml
+++ b/pom.xml
@@ -110,8 +110,8 @@
         <consul.client>1.3.8</consul.client>
         <h2>1.4.191</h2>
         <protobuf.java>3.9.1</protobuf.java>
-        <grpc.spring.boot>3.5.0</grpc.spring.boot>
-        <spring.boot.data.jpa>1.5.13.RELEASE</spring.boot.data.jpa>
+        <grpc.spring.boot>3.5.1</grpc.spring.boot>
+        <spring.boot.data.jpa>2.2.1.RELEASE</spring.boot.data.jpa>
         <dozer>5.5.1</dozer>
         <jsch>0.1.55</jsch>
         <sshj>0.23.0</sshj>
diff --git a/services/pom.xml b/services/pom.xml
index e9bb24b..813cd3b 100644
--- a/services/pom.xml
+++ b/services/pom.xml
@@ -47,6 +47,12 @@
             <groupId>io.github.lognet</groupId>
             <artifactId>grpc-spring-boot-starter</artifactId>
             <version>${grpc.spring.boot}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.springframework.boot</groupId>
+                    <artifactId>spring-boot-starter</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>


[airavata-mft] 02/02: Implementing a SQL backend for the Secret service and end to end workflow for scp transfers

Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git

commit aff62367ee1439f7e2ee54540d4b060d2dc057b1
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Dec 31 03:58:59 2019 -0500

    Implementing a SQL backend for the Secret service and end to end workflow for scp transfers
---
 .../apache/airavata/mft/agent/ConsulTester.java    | 10 ++-
 .../org/apache/airavata/mft/agent/MFTAgent.java    |  5 +-
 .../server/backend/sql/SQLResourceBackend.java     | 13 ++--
 .../backend/sql/entity/LocalResourceEntity.java    |  1 +
 .../backend/sql/entity/SCPResourceEntity.java      |  8 +-
 services/secret-service/server/pom.xml             |  6 ++
 ...ecretServiceApplication.java => AppConfig.java} | 16 ++--
 .../secret/server/SecretServiceApplication.java    |  2 +
 .../SecretBackend.java}                            | 17 +++--
 .../server/backend/sql/SQLSecretBackend.java       | 59 +++++++++++++++
 .../server/backend/sql/entity/SCPSecretEntity.java | 87 ++++++++++++++++++++++
 .../sql/repository/SecretRepository.java}          | 15 ++--
 .../server/handler/SecretServiceHandler.java       | 46 +++++++++---
 .../stub/src/main/proto/SecretService.proto        | 54 +++++++++++++-
 transport/pom.xml                                  | 12 +++
 .../mft/transport/scp/SCPMetadataCollector.java    | 23 ++++--
 .../airavata/mft/transport/scp/SCPReceiver.java    | 27 +++++--
 .../airavata/mft/transport/scp/SCPSender.java      | 28 +++++--
 .../mft/transport/scp/SCPTransportUtil.java        | 30 ++++++++
 19 files changed, 389 insertions(+), 70 deletions(-)

diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
index 39240de..0a0f1c8 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
@@ -29,12 +29,16 @@ public class ConsulTester {
         ObjectMapper mapper = new ObjectMapper();
 
         TransferRequest request = new TransferRequest();
-        request.setSourceId("1");
+        request.setSourceId("40107348-c457-4b0d-b206-b923caa2ab8d");
         request.setSourceType("SCP");
-        request.setDestinationId("2");
+        request.setSourceToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
+        request.setDestinationId("24cf4870-ee50-4076-b97c-aeb481fec324");
         request.setDestinationType("SCP");
+        request.setDestinationToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
         request.setAgentList(Collections.singletonList("agent0"));
-        request.setTransferId("transfer005");
+        request.setTransferId("transfer010");
 
         String asString = mapper.writeValueAsString(request);
 
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index 4424e96..92e851b 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -71,10 +71,11 @@ public class MFTAgent {
                         String transferId = mediator.transfer(inConnector, outConnector, metadata);
                         System.out.println("Submitted transfer " + transferId);
 
-                        System.out.println("Deleting key " + value.getKey());
-                        kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
                     } catch (Exception e) {
                         e.printStackTrace();
+                    } finally {
+                        System.out.println("Deleting key " + value.getKey());
+                        kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
                     }
                 });
 
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
index 0be7b6d..1ffabe3 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
@@ -63,20 +63,23 @@ public class SQLResourceBackend implements ResourceBackend {
 
     @Override
     public boolean deleteSCPStorage(SCPStorageDeleteRequest request) {
-        scpStorageRepository.delete(request.getStorageId());
+        //scpStorageRepository.delete(request.getStorageId());
         return true;
     }
 
     @Override
     public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request) {
         Optional<SCPResourceEntity> resourceEntity = scpResourceRepository.findByResourceId(request.getResourceId());
-        return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass()).build());
+
+        return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass())
+                .setScpStorage(mapper.map(scpResourceEntity.getScpStorage(), SCPStorage.newBuilder().getClass())).build());
+        // Here we have to do nested mapping as the dozer -> protobuf conversion is not happening for inner objects
     }
 
     @Override
     public SCPResource createSCPResource(SCPResourceCreateRequest request) {
         SCPResourceEntity savedEntity = scpResourceRepository.save(mapper.map(request, SCPResourceEntity.class));
-        return mapper.map(savedEntity, SCPResource.newBuilder().getClass()).build();
+        return getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(savedEntity.getResourceId()).build()).get();
     }
 
     @Override
@@ -87,7 +90,7 @@ public class SQLResourceBackend implements ResourceBackend {
 
     @Override
     public boolean deleteSCPResource(SCPResourceDeleteRequest request) {
-        scpResourceRepository.delete(request.getResourceId());
+        scpResourceRepository.deleteById(request.getResourceId());
         return true;
     }
 
@@ -111,7 +114,7 @@ public class SQLResourceBackend implements ResourceBackend {
 
     @Override
     public boolean deleteLocalResource(LocalResourceDeleteRequest request) {
-        localResourceRepository.delete(request.getResourceId());
+        localResourceRepository.deleteById(request.getResourceId());
         return true;
     }
 }
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
index 2d48b5f..2a8ed81 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
@@ -33,6 +33,7 @@ public class LocalResourceEntity {
     @GeneratedValue(generator = "uuid")
     @GenericGenerator(name = "uuid", strategy = "uuid2")
     private String resourceId;
+
     @Column(name = "RESOURCE_PATH")
     private String resourcePath;
 
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
index cab753b..63a6b32 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
@@ -29,13 +29,13 @@ public class SCPResourceEntity {
     @GenericGenerator(name = "uuid", strategy = "uuid2")
     private String resourceId;
 
+    @Column(name = "SCP_STORAGE_ID")
+    private String scpStorageId;
+
     @ManyToOne(fetch = FetchType.EAGER)
-    @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, updatable = false)
+    @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, insertable = false, updatable = false)
     private SCPStorageEntity scpStorage;
 
-    @Column(name = "SCP_STORAGE_ID", insertable = false, updatable = false)
-    private String scpStorageId;
-
     @Column(name = "RESOURCE_PATH")
     private String resourcePath;
 
diff --git a/services/secret-service/server/pom.xml b/services/secret-service/server/pom.xml
index bfa1c52..0f0a042 100644
--- a/services/secret-service/server/pom.xml
+++ b/services/secret-service/server/pom.xml
@@ -38,5 +38,11 @@
             <artifactId>mft-secret-service-stub</artifactId>
             <version>0.01-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>${h2}</version>
+            <scope>runtime</scope>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
similarity index 69%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
index d7830ac..fb2d2d1 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
@@ -17,12 +17,14 @@
 
 package org.apache.airavata.mft.secret.server;
 
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
 
-@SpringBootApplication
-public class SecretServiceApplication {
-    public static void main(String args[]) {
-        SpringApplication.run(SecretServiceApplication.class, args);
-    }
+@Configuration
+public class AppConfig {
+
+    @Bean
+    SecretBackend secretBackend() {return new SQLSecretBackend();}
 }
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
index d7830ac..c9e930d 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
@@ -19,7 +19,9 @@ package org.apache.airavata.mft.secret.server;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
 
+@ComponentScan(basePackages = {"org.apache.airavata"})
 @SpringBootApplication
 public class SecretServiceApplication {
     public static void main(String args[]) {
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
similarity index 64%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
index d7830ac..9648747 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
@@ -15,14 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.airavata.mft.secret.server;
+package org.apache.airavata.mft.secret.server.backend;
 
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.service.*;
 
-@SpringBootApplication
-public class SecretServiceApplication {
-    public static void main(String args[]) {
-        SpringApplication.run(SecretServiceApplication.class, args);
-    }
+import java.util.Optional;
+
+public interface SecretBackend {
+    public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request);
+    public SCPSecret createSCPSecret(SCPSecretCreateRequest request);
+    public boolean updateSCPSecret(SCPSecretUpdateRequest request);
+    public boolean deleteSCPSecret(SCPSecretDeleteRequest request);
 }
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
new file mode 100644
index 0000000..513bbba
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql;
+
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.repository.SecretRepository;
+import org.apache.airavata.mft.secret.service.*;
+import org.dozer.DozerBeanMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Optional;
+
+public class SQLSecretBackend implements SecretBackend {
+
+    @Autowired
+    private SecretRepository secretRepository;
+
+    private DozerBeanMapper mapper = new DozerBeanMapper();
+
+    @Override
+    public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) {
+        Optional<SCPSecretEntity> secretEty = secretRepository.findBySecretId(request.getSecretId());
+        return secretEty.map(scpSecretEntity -> mapper.map(scpSecretEntity, SCPSecret.newBuilder().getClass()).build());
+    }
+
+    @Override
+    public SCPSecret createSCPSecret(SCPSecretCreateRequest request) {
+        SCPSecretEntity savedEntity = secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+        return mapper.map(savedEntity, SCPSecret.newBuilder().getClass()).build();
+    }
+
+    @Override
+    public boolean updateSCPSecret(SCPSecretUpdateRequest request) {
+        secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+        return true;
+    }
+
+    @Override
+    public boolean deleteSCPSecret(SCPSecretDeleteRequest request) {
+        secretRepository.deleteById(request.getSecretId());
+        return true;
+    }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java
new file mode 100644
index 0000000..3fd75d9
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class SCPSecretEntity {
+
+    @Id
+    @Column(name = "SECRET_ID")
+    @GeneratedValue(generator = "uuid")
+    @GenericGenerator(name = "uuid", strategy = "uuid2")
+    private String secretId;
+
+    @Column(name = "PRIVATE_KEY")
+    private String privateKey;
+
+    @Column(name = "PUBLIC_KEY")
+    private String publicKey;
+
+    @Column(name = "PASSPHRASE")
+    private String passphrase;
+
+    @Column(name = "USER_NAME")
+    private String user;
+
+    public String getSecretId() {
+        return secretId;
+    }
+
+    public void setSecretId(String secretId) {
+        this.secretId = secretId;
+    }
+
+    public String getPrivateKey() {
+        return privateKey;
+    }
+
+    public void setPrivateKey(String privateKey) {
+        this.privateKey = privateKey;
+    }
+
+    public String getPublicKey() {
+        return publicKey;
+    }
+
+    public void setPublicKey(String publicKey) {
+        this.publicKey = publicKey;
+    }
+
+    public String getPassphrase() {
+        return passphrase;
+    }
+
+    public void setPassphrase(String passphrase) {
+        this.passphrase = passphrase;
+    }
+
+    public String getUser() {
+        return user;
+    }
+
+    public void setUser(String user) {
+        this.user = user;
+    }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
similarity index 67%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
index d7830ac..70bf6c9 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
@@ -15,14 +15,13 @@
  * limitations under the License.
  */
 
-package org.apache.airavata.mft.secret.server;
+package org.apache.airavata.mft.secret.server.backend.sql.repository;
 
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.springframework.data.repository.CrudRepository;
 
-@SpringBootApplication
-public class SecretServiceApplication {
-    public static void main(String args[]) {
-        SpringApplication.run(SecretServiceApplication.class, args);
-    }
+import java.util.Optional;
+
+public interface SecretRepository extends CrudRepository<SCPSecretEntity, String> {
+    Optional<SCPSecretEntity> findBySecretId(String resourceId);
 }
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
index 26c741a..26f20fb 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
@@ -17,22 +17,48 @@
 
 package org.apache.airavata.mft.secret.server.handler;
 
+import com.google.protobuf.Empty;
 import io.grpc.stub.StreamObserver;
-import org.apache.airavata.mft.secret.service.SCPSecret;
-import org.apache.airavata.mft.secret.service.SCPSecretRequest;
-import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.service.*;
 import org.lognet.springboot.grpc.GRpcService;
+import org.springframework.beans.factory.annotation.Autowired;
 
 @GRpcService
 public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBase {
+
+    @Autowired
+    private SecretBackend backend;
+
+    @Override
+    public void getSCPSecret(SCPSecretGetRequest request, StreamObserver<SCPSecret> responseObserver) {
+        this.backend.getSCPSecret(request).ifPresentOrElse(secret -> {
+            responseObserver.onNext(secret);
+            responseObserver.onCompleted();
+        }, () -> {
+            responseObserver.onError(new Exception("No SCP Secret with id " + request.getSecretId()));
+        });
+    }
+
+    @Override
+    public void createSCPSecret(SCPSecretCreateRequest request, StreamObserver<SCPSecret> responseObserver) {
+        responseObserver.onNext(this.backend.createSCPSecret(request));
+        responseObserver.onCompleted();
+    }
+
     @Override
-    public void getSCPSecret(SCPSecretRequest request, StreamObserver<SCPSecret> responseObserver) {
-        SCPSecret.Builder builder = SCPSecret.newBuilder()
-                .setPrivateKey("private")
-                .setPrivateKey("pubKey")
-                .setPassphrase("pass")
-                .setSecretId("sec1");
-        responseObserver.onNext(builder.build());
+    public void updateSCPSecret(SCPSecretUpdateRequest request, StreamObserver<Empty> responseObserver) {
+        this.backend.updateSCPSecret(request);
         responseObserver.onCompleted();
     }
+
+    @Override
+    public void deleteSCPSecret(SCPSecretDeleteRequest request, StreamObserver<Empty> responseObserver) {
+        boolean res = this.backend.deleteSCPSecret(request);
+        if (res) {
+            responseObserver.onCompleted();
+        } else {
+            responseObserver.onError(new Exception("Failed to delete SCP Secret with id " + request.getSecretId()));
+        }
+    }
 }
diff --git a/services/secret-service/stub/src/main/proto/SecretService.proto b/services/secret-service/stub/src/main/proto/SecretService.proto
index f2d0cc7..9fa99b2 100644
--- a/services/secret-service/stub/src/main/proto/SecretService.proto
+++ b/services/secret-service/stub/src/main/proto/SecretService.proto
@@ -4,23 +4,69 @@ option java_multiple_files = true;
 package org.apache.airavata.mft.secret.service;
 
 import "google/api/annotations.proto";
+import "google/protobuf/empty.proto";
+
+message AuthToken {
+    string token = 1;
+}
 
 message SCPSecret {
     string secretId = 1;
     string privateKey = 2;
     string publicKey = 3;
     string passphrase = 4;
+    string user = 5;
+}
+
+message SCPSecretGetRequest {
+    string secretId = 1;
+    AuthToken authzToken = 2;
+}
+
+message SCPSecretCreateRequest {
+    string privateKey = 1;
+    string publicKey = 2;
+    string passphrase = 3;
+    string user = 4;
+    AuthToken authzToken = 5;
+}
+
+message SCPSecretUpdateRequest {
+    string secretId = 1;
+    string privateKey = 2;
+    string publicKey = 3;
+    string passphrase = 4;
+    string user = 5;
+    AuthToken authzToken = 6;
 }
 
-message SCPSecretRequest {
-    string resourceId = 1;
-    string authzToken = 2;
+message SCPSecretDeleteRequest {
+    string secretId = 1;
+    AuthToken authzToken = 2;
 }
 
 service  SecretService {
-    rpc getSCPSecret (SCPSecretRequest) returns (SCPSecret) {
+    rpc getSCPSecret (SCPSecretGetRequest) returns (SCPSecret) {
         option (google.api.http) = {
            get: "/v1.0/secret/scp"
         };
     }
+
+    rpc createSCPSecret (SCPSecretCreateRequest) returns (SCPSecret) {
+        option (google.api.http) = {
+           post: "/v1.0/secret/scp"
+        };
+    }
+
+    rpc updateSCPSecret (SCPSecretUpdateRequest) returns (google.protobuf.Empty) {
+        option (google.api.http) = {
+           put: "/v1.0/secret/scp"
+        };
+    }
+
+    rpc deleteSCPSecret (SCPSecretDeleteRequest) returns (google.protobuf.Empty) {
+        option (google.api.http) = {
+           delete: "/v1.0/secret/scp"
+        };
+    }
 }
\ No newline at end of file
diff --git a/transport/pom.xml b/transport/pom.xml
index 64266c5..27e3cba 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -36,6 +36,18 @@
         <module>scp-transport</module>
         <module>local-transport</module>
     </modules>
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>mft-resource-service-client</artifactId>
+            <version>0.01-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.airavata</groupId>
+            <artifactId>mft-secret-service-client</artifactId>
+            <version>0.01-SNAPSHOT</version>
+        </dependency>
+    </dependencies>
 
 
 </project>
\ No newline at end of file
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
index abe5305..6f95895 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
@@ -28,6 +28,14 @@ import net.schmizz.sshj.userauth.method.ChallengeResponseProvider;
 import net.schmizz.sshj.userauth.password.Resource;
 import org.apache.airavata.mft.core.ResourceMetadata;
 import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.resource.service.SCPResource;
+import org.apache.airavata.mft.resource.service.SCPResourceGetRequest;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -38,11 +46,16 @@ public class SCPMetadataCollector implements MetadataCollector {
 
     public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws IOException {
 
-        SSHResourceIdentifier sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
+        ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+        SCPResource scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+        SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+        SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
 
         try (SSHClient sshClient = new SSHClient()) {
             sshClient.addHostKeyVerifier((h, p, key) -> true);
-            KeyProvider keyProvider = sshClient.loadKeys(sshResourceIdentifier.getKeyFile(), sshResourceIdentifier.getKeyPassphrase());
+            KeyProvider keyProvider = sshClient.loadKeys(scpSecret.getPrivateKey(), scpSecret.getPassphrase());
             final List<AuthMethod> am = new LinkedList<>();
             am.add(new AuthPublickey(keyProvider));
             am.add(new AuthKeyboardInteractive(new ChallengeResponseProvider() {
@@ -65,11 +78,11 @@ public class SCPMetadataCollector implements MetadataCollector {
                 }
             }));
 
-            sshClient.connect(sshResourceIdentifier.getHost(), sshResourceIdentifier.getPort());
-            sshClient.auth(sshResourceIdentifier.getUser(), am);
+            sshClient.connect(scpResource.getScpStorage().getHost(), scpResource.getScpStorage().getPort());
+            sshClient.auth(scpSecret.getUser(), am);
 
             try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
-                FileAttributes lstat = sftpClient.lstat(sshResourceIdentifier.getRemotePath());
+                FileAttributes lstat = sftpClient.lstat(scpResource.getResourcePath());
                 sftpClient.close();
 
                 ResourceMetadata metadata = new ResourceMetadata();
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index e56e9e0..141395f 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -22,6 +22,12 @@ import com.jcraft.jsch.Session;
 import org.apache.airavata.mft.core.CircularStreamingBuffer;
 import org.apache.airavata.mft.core.ConnectorContext;
 import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -29,14 +35,21 @@ import java.io.OutputStream;
 
 public class SCPReceiver implements Connector {
     private Session session;
-    private SSHResourceIdentifier sshResourceIdentifier;
+    private SCPResource scpResource;
 
     public void init(String resourceId, String credentialToken) throws Exception {
-        this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
-        this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(),
-                sshResourceIdentifier.getPort(),
-                sshResourceIdentifier.getKeyFile(),
-                sshResourceIdentifier.getKeyPassphrase());
+        ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+        this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+        SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+        SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+        this.session = SCPTransportUtil.createSession(
+                scpSecret.getUser(),
+                scpResource.getScpStorage().getHost(),
+                scpResource.getScpStorage().getPort(),
+                scpSecret.getPrivateKey(),
+                scpSecret.getPassphrase());
     }
 
     public void destroy() {
@@ -53,7 +66,7 @@ public class SCPReceiver implements Connector {
             throw new Exception("Session can not be null. Make sure that SCP Receiver is properly initialized");
         }
 
-        transferRemoteToStream(session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer());
+        transferRemoteToStream(session, this.scpResource.getResourcePath(), context.getStreamBuffer());
     }
 
     private void transferRemoteToStream(Session session, String from, CircularStreamingBuffer streamBuffer) throws Exception {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index 31bb138..7f6b217 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -23,20 +23,34 @@ import com.jcraft.jsch.Session;
 import org.apache.airavata.mft.core.CircularStreamingBuffer;
 import org.apache.airavata.mft.core.ConnectorContext;
 import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.resource.service.SCPResource;
+import org.apache.airavata.mft.resource.service.SCPResourceGetRequest;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
 public class SCPSender implements Connector {
-    private SSHResourceIdentifier sshResourceIdentifier;
     private Session session;
+    private SCPResource scpResource;
     public void init(String resourceId, String credentialToken) {
-        this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
-        this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(),
-                sshResourceIdentifier.getPort(),
-                sshResourceIdentifier.getKeyFile(),
-                sshResourceIdentifier.getKeyPassphrase());
+
+        ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+        this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+        SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+        SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+        this.session = SCPTransportUtil.createSession(scpSecret.getUser(), scpResource.getScpStorage().getHost(),
+                scpResource.getScpStorage().getPort(),
+                scpSecret.getPrivateKey(),
+                scpSecret.getPassphrase());
     }
 
     public void destroy() {
@@ -53,7 +67,7 @@ public class SCPSender implements Connector {
             System.out.println("Session can not be null. Make sure that SCP Sender is properly initialized");
             throw new Exception("Session can not be null. Make sure that SCP Sender is properly initialized");
         }
-        copyLocalToRemote(this.session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
+        copyLocalToRemote(this.session, this.scpResource.getResourcePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
     }
 
     private void copyLocalToRemote(Session session, String to, CircularStreamingBuffer streamBuffer, long fileSize) throws JSchException, IOException {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
index d81e3dc..bcb8342 100755
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
@@ -20,11 +20,41 @@ package org.apache.airavata.mft.transport.scp;
 import com.jcraft.jsch.JSch;
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretCreateRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
 
 import java.util.Properties;
 
 public class SCPTransportUtil {
     // TODO replace with an API call to the registry
+
+
+    public static void main(String args[]) {
+        ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+        SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+
+        //resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId("82848d70-ef71-47d2-935a-4fd3e0184a42").build());
+        SCPStorage scpStorage1 = resourceClient.createSCPStorage(SCPStorageCreateRequest.newBuilder().setPort(22).setHost("pgadev.scigap.org").build());
+        SCPResource scpResource1 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId())
+                .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/2mb.txt").build());
+        SCPResource scpResource2 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId())
+                .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/new-file.txt").build());
+
+        System.out.println("Resource 1 id " + scpResource1.getResourceId());
+        System.out.println("Resource 2 id " + scpResource2.getResourceId());
+
+        SCPSecret scpSecret1 = secretClient.createSCPSecret(SCPSecretCreateRequest.newBuilder()
+                .setUser("pga")
+                .setPassphrase("shithappens@12")
+                .setPrivateKey("/Users/dimuthu/.ssh/id_rsa").build());
+
+        System.out.println("Secret id " + scpSecret1.getSecretId());
+    }
+
     public static SSHResourceIdentifier getSSHResourceIdentifier(String resourceId) {
 
         SSHResourceIdentifier identifier = new SSHResourceIdentifier();