You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/31 08:59:13 UTC
[airavata-mft] 02/02: Implementing a SQL backend for the Secret
service and end to end workflow for scp transfers
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit aff62367ee1439f7e2ee54540d4b060d2dc057b1
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Dec 31 03:58:59 2019 -0500
Implementing a SQL backend for the Secret service and end to end workflow for scp transfers
---
.../apache/airavata/mft/agent/ConsulTester.java | 10 ++-
.../org/apache/airavata/mft/agent/MFTAgent.java | 5 +-
.../server/backend/sql/SQLResourceBackend.java | 13 ++--
.../backend/sql/entity/LocalResourceEntity.java | 1 +
.../backend/sql/entity/SCPResourceEntity.java | 8 +-
services/secret-service/server/pom.xml | 6 ++
...ecretServiceApplication.java => AppConfig.java} | 16 ++--
.../secret/server/SecretServiceApplication.java | 2 +
.../SecretBackend.java} | 17 +++--
.../server/backend/sql/SQLSecretBackend.java | 59 +++++++++++++++
.../server/backend/sql/entity/SCPSecretEntity.java | 87 ++++++++++++++++++++++
.../sql/repository/SecretRepository.java} | 15 ++--
.../server/handler/SecretServiceHandler.java | 46 +++++++++---
.../stub/src/main/proto/SecretService.proto | 54 +++++++++++++-
transport/pom.xml | 12 +++
.../mft/transport/scp/SCPMetadataCollector.java | 23 ++++--
.../airavata/mft/transport/scp/SCPReceiver.java | 27 +++++--
.../airavata/mft/transport/scp/SCPSender.java | 28 +++++--
.../mft/transport/scp/SCPTransportUtil.java | 30 ++++++++
19 files changed, 389 insertions(+), 70 deletions(-)
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
index 39240de..0a0f1c8 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
@@ -29,12 +29,16 @@ public class ConsulTester {
ObjectMapper mapper = new ObjectMapper();
TransferRequest request = new TransferRequest();
- request.setSourceId("1");
+ request.setSourceId("40107348-c457-4b0d-b206-b923caa2ab8d");
request.setSourceType("SCP");
- request.setDestinationId("2");
+ request.setSourceToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
+ request.setDestinationId("24cf4870-ee50-4076-b97c-aeb481fec324");
request.setDestinationType("SCP");
+ request.setDestinationToken("866d421e-3624-434f-ae71-04a90d39e70c");
+
request.setAgentList(Collections.singletonList("agent0"));
- request.setTransferId("transfer005");
+ request.setTransferId("transfer010");
String asString = mapper.writeValueAsString(request);
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index 4424e96..92e851b 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -71,10 +71,11 @@ public class MFTAgent {
String transferId = mediator.transfer(inConnector, outConnector, metadata);
System.out.println("Submitted transfer " + transferId);
- System.out.println("Deleting key " + value.getKey());
- kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
} catch (Exception e) {
e.printStackTrace();
+ } finally {
+ System.out.println("Deleting key " + value.getKey());
+ kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
}
});
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
index 0be7b6d..1ffabe3 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java
@@ -63,20 +63,23 @@ public class SQLResourceBackend implements ResourceBackend {
@Override
public boolean deleteSCPStorage(SCPStorageDeleteRequest request) {
- scpStorageRepository.delete(request.getStorageId());
+ //scpStorageRepository.delete(request.getStorageId());
return true;
}
@Override
public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request) {
Optional<SCPResourceEntity> resourceEntity = scpResourceRepository.findByResourceId(request.getResourceId());
- return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass()).build());
+
+ return resourceEntity.map(scpResourceEntity -> mapper.map(scpResourceEntity, SCPResource.newBuilder().getClass())
+ .setScpStorage(mapper.map(scpResourceEntity.getScpStorage(), SCPStorage.newBuilder().getClass())).build());
+ // Here we have to do nested mapping as the dozer -> protobuf conversion is not happening for inner objects
}
@Override
public SCPResource createSCPResource(SCPResourceCreateRequest request) {
SCPResourceEntity savedEntity = scpResourceRepository.save(mapper.map(request, SCPResourceEntity.class));
- return mapper.map(savedEntity, SCPResource.newBuilder().getClass()).build();
+ return getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(savedEntity.getResourceId()).build()).get();
}
@Override
@@ -87,7 +90,7 @@ public class SQLResourceBackend implements ResourceBackend {
@Override
public boolean deleteSCPResource(SCPResourceDeleteRequest request) {
- scpResourceRepository.delete(request.getResourceId());
+ scpResourceRepository.deleteById(request.getResourceId());
return true;
}
@@ -111,7 +114,7 @@ public class SQLResourceBackend implements ResourceBackend {
@Override
public boolean deleteLocalResource(LocalResourceDeleteRequest request) {
- localResourceRepository.delete(request.getResourceId());
+ localResourceRepository.deleteById(request.getResourceId());
return true;
}
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
index 2d48b5f..2a8ed81 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/LocalResourceEntity.java
@@ -33,6 +33,7 @@ public class LocalResourceEntity {
@GeneratedValue(generator = "uuid")
@GenericGenerator(name = "uuid", strategy = "uuid2")
private String resourceId;
+
@Column(name = "RESOURCE_PATH")
private String resourcePath;
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
index cab753b..63a6b32 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/entity/SCPResourceEntity.java
@@ -29,13 +29,13 @@ public class SCPResourceEntity {
@GenericGenerator(name = "uuid", strategy = "uuid2")
private String resourceId;
+ @Column(name = "SCP_STORAGE_ID")
+ private String scpStorageId;
+
@ManyToOne(fetch = FetchType.EAGER)
- @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, updatable = false)
+ @JoinColumn(name = "SCP_STORAGE_ID", referencedColumnName = "SCP_STORAGE_ID", nullable = false, insertable = false, updatable = false)
private SCPStorageEntity scpStorage;
- @Column(name = "SCP_STORAGE_ID", insertable = false, updatable = false)
- private String scpStorageId;
-
@Column(name = "RESOURCE_PATH")
private String resourcePath;
diff --git a/services/secret-service/server/pom.xml b/services/secret-service/server/pom.xml
index bfa1c52..0f0a042 100644
--- a/services/secret-service/server/pom.xml
+++ b/services/secret-service/server/pom.xml
@@ -38,5 +38,11 @@
<artifactId>mft-secret-service-stub</artifactId>
<version>0.01-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>com.h2database</groupId>
+ <artifactId>h2</artifactId>
+ <version>${h2}</version>
+ <scope>runtime</scope>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
similarity index 69%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
index d7830ac..fb2d2d1 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
@@ -17,12 +17,14 @@
package org.apache.airavata.mft.secret.server;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
-@SpringBootApplication
-public class SecretServiceApplication {
- public static void main(String args[]) {
- SpringApplication.run(SecretServiceApplication.class, args);
- }
+@Configuration
+public class AppConfig {
+
+ @Bean
+ SecretBackend secretBackend() {return new SQLSecretBackend();}
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
index d7830ac..c9e930d 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
@@ -19,7 +19,9 @@ package org.apache.airavata.mft.secret.server;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
+@ComponentScan(basePackages = {"org.apache.airavata"})
@SpringBootApplication
public class SecretServiceApplication {
public static void main(String args[]) {
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
similarity index 64%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
index d7830ac..9648747 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
@@ -15,14 +15,15 @@
* limitations under the License.
*/
-package org.apache.airavata.mft.secret.server;
+package org.apache.airavata.mft.secret.server.backend;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.service.*;
-@SpringBootApplication
-public class SecretServiceApplication {
- public static void main(String args[]) {
- SpringApplication.run(SecretServiceApplication.class, args);
- }
+import java.util.Optional;
+
+public interface SecretBackend {
+ public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request);
+ public SCPSecret createSCPSecret(SCPSecretCreateRequest request);
+ public boolean updateSCPSecret(SCPSecretUpdateRequest request);
+ public boolean deleteSCPSecret(SCPSecretDeleteRequest request);
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
new file mode 100644
index 0000000..513bbba
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/SQLSecretBackend.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql;
+
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.apache.airavata.mft.secret.server.backend.sql.repository.SecretRepository;
+import org.apache.airavata.mft.secret.service.*;
+import org.dozer.DozerBeanMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.util.Optional;
+
+public class SQLSecretBackend implements SecretBackend {
+
+ @Autowired
+ private SecretRepository secretRepository;
+
+ private DozerBeanMapper mapper = new DozerBeanMapper();
+
+ @Override
+ public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) {
+ Optional<SCPSecretEntity> secretEty = secretRepository.findBySecretId(request.getSecretId());
+ return secretEty.map(scpSecretEntity -> mapper.map(scpSecretEntity, SCPSecret.newBuilder().getClass()).build());
+ }
+
+ @Override
+ public SCPSecret createSCPSecret(SCPSecretCreateRequest request) {
+ SCPSecretEntity savedEntity = secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+ return mapper.map(savedEntity, SCPSecret.newBuilder().getClass()).build();
+ }
+
+ @Override
+ public boolean updateSCPSecret(SCPSecretUpdateRequest request) {
+ secretRepository.save(mapper.map(request, SCPSecretEntity.class));
+ return true;
+ }
+
+ @Override
+ public boolean deleteSCPSecret(SCPSecretDeleteRequest request) {
+ secretRepository.deleteById(request.getSecretId());
+ return true;
+ }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java
new file mode 100644
index 0000000..3fd75d9
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/entity/SCPSecretEntity.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.sql.entity;
+
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+@Entity
+public class SCPSecretEntity {
+
+ @Id
+ @Column(name = "SECRET_ID")
+ @GeneratedValue(generator = "uuid")
+ @GenericGenerator(name = "uuid", strategy = "uuid2")
+ private String secretId;
+
+ @Column(name = "PRIVATE_KEY")
+ private String privateKey;
+
+ @Column(name = "PUBLIC_KEY")
+ private String publicKey;
+
+ @Column(name = "PASSPHRASE")
+ private String passphrase;
+
+ @Column(name = "USER_NAME")
+ private String user;
+
+ public String getSecretId() {
+ return secretId;
+ }
+
+ public void setSecretId(String secretId) {
+ this.secretId = secretId;
+ }
+
+ public String getPrivateKey() {
+ return privateKey;
+ }
+
+ public void setPrivateKey(String privateKey) {
+ this.privateKey = privateKey;
+ }
+
+ public String getPublicKey() {
+ return publicKey;
+ }
+
+ public void setPublicKey(String publicKey) {
+ this.publicKey = publicKey;
+ }
+
+ public String getPassphrase() {
+ return passphrase;
+ }
+
+ public void setPassphrase(String passphrase) {
+ this.passphrase = passphrase;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
similarity index 67%
copy from services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
copy to services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
index d7830ac..70bf6c9 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/SecretServiceApplication.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/sql/repository/SecretRepository.java
@@ -15,14 +15,13 @@
* limitations under the License.
*/
-package org.apache.airavata.mft.secret.server;
+package org.apache.airavata.mft.secret.server.backend.sql.repository;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.apache.airavata.mft.secret.server.backend.sql.entity.SCPSecretEntity;
+import org.springframework.data.repository.CrudRepository;
-@SpringBootApplication
-public class SecretServiceApplication {
- public static void main(String args[]) {
- SpringApplication.run(SecretServiceApplication.class, args);
- }
+import java.util.Optional;
+
+public interface SecretRepository extends CrudRepository<SCPSecretEntity, String> {
+ Optional<SCPSecretEntity> findBySecretId(String resourceId);
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
index 26c741a..26f20fb 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
@@ -17,22 +17,48 @@
package org.apache.airavata.mft.secret.server.handler;
+import com.google.protobuf.Empty;
import io.grpc.stub.StreamObserver;
-import org.apache.airavata.mft.secret.service.SCPSecret;
-import org.apache.airavata.mft.secret.service.SCPSecretRequest;
-import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.service.*;
import org.lognet.springboot.grpc.GRpcService;
+import org.springframework.beans.factory.annotation.Autowired;
@GRpcService
public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBase {
+
+ @Autowired
+ private SecretBackend backend;
+
+ @Override
+ public void getSCPSecret(SCPSecretGetRequest request, StreamObserver<SCPSecret> responseObserver) {
+ this.backend.getSCPSecret(request).ifPresentOrElse(secret -> {
+ responseObserver.onNext(secret);
+ responseObserver.onCompleted();
+ }, () -> {
+ responseObserver.onError(new Exception("No SCP Secret with id " + request.getSecretId()));
+ });
+ }
+
+ @Override
+ public void createSCPSecret(SCPSecretCreateRequest request, StreamObserver<SCPSecret> responseObserver) {
+ responseObserver.onNext(this.backend.createSCPSecret(request));
+ responseObserver.onCompleted();
+ }
+
@Override
- public void getSCPSecret(SCPSecretRequest request, StreamObserver<SCPSecret> responseObserver) {
- SCPSecret.Builder builder = SCPSecret.newBuilder()
- .setPrivateKey("private")
- .setPrivateKey("pubKey")
- .setPassphrase("pass")
- .setSecretId("sec1");
- responseObserver.onNext(builder.build());
+ public void updateSCPSecret(SCPSecretUpdateRequest request, StreamObserver<Empty> responseObserver) {
+ this.backend.updateSCPSecret(request);
responseObserver.onCompleted();
}
+
+ @Override
+ public void deleteSCPSecret(SCPSecretDeleteRequest request, StreamObserver<Empty> responseObserver) {
+ boolean res = this.backend.deleteSCPSecret(request);
+ if (res) {
+ responseObserver.onCompleted();
+ } else {
+ responseObserver.onError(new Exception("Failed to delete SCP Secret with id " + request.getSecretId()));
+ }
+ }
}
diff --git a/services/secret-service/stub/src/main/proto/SecretService.proto b/services/secret-service/stub/src/main/proto/SecretService.proto
index f2d0cc7..9fa99b2 100644
--- a/services/secret-service/stub/src/main/proto/SecretService.proto
+++ b/services/secret-service/stub/src/main/proto/SecretService.proto
@@ -4,23 +4,69 @@ option java_multiple_files = true;
package org.apache.airavata.mft.secret.service;
import "google/api/annotations.proto";
+import "google/protobuf/empty.proto";
+
+message AuthToken {
+ string token = 1;
+}
message SCPSecret {
string secretId = 1;
string privateKey = 2;
string publicKey = 3;
string passphrase = 4;
+ string user = 5;
+}
+
+message SCPSecretGetRequest {
+ string secretId = 1;
+ AuthToken authzToken = 2;
+}
+
+message SCPSecretCreateRequest {
+ string privateKey = 1;
+ string publicKey = 2;
+ string passphrase = 3;
+ string user = 4;
+ AuthToken authzToken = 5;
+}
+
+message SCPSecretUpdateRequest {
+ string secretId = 1;
+ string privateKey = 2;
+ string publicKey = 3;
+ string passphrase = 4;
+ string user = 5;
+ AuthToken authzToken = 6;
}
-message SCPSecretRequest {
- string resourceId = 1;
- string authzToken = 2;
+message SCPSecretDeleteRequest {
+ string secretId = 1;
+ AuthToken authzToken = 2;
}
service SecretService {
- rpc getSCPSecret (SCPSecretRequest) returns (SCPSecret) {
+ rpc getSCPSecret (SCPSecretGetRequest) returns (SCPSecret) {
option (google.api.http) = {
get: "/v1.0/secret/scp"
};
}
+
+ rpc createSCPSecret (SCPSecretCreateRequest) returns (SCPSecret) {
+ option (google.api.http) = {
+ post: "/v1.0/secret/scp"
+ };
+ }
+
+ rpc updateSCPSecret (SCPSecretUpdateRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ put: "/v1.0/secret/scp"
+ };
+ }
+
+ rpc deleteSCPSecret (SCPSecretDeleteRequest) returns (google.protobuf.Empty) {
+ option (google.api.http) = {
+ delete: "/v1.0/secret/scp"
+ };
+ }
}
\ No newline at end of file
diff --git a/transport/pom.xml b/transport/pom.xml
index 64266c5..27e3cba 100755
--- a/transport/pom.xml
+++ b/transport/pom.xml
@@ -36,6 +36,18 @@
<module>scp-transport</module>
<module>local-transport</module>
</modules>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-resource-service-client</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>mft-secret-service-client</artifactId>
+ <version>0.01-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
</project>
\ No newline at end of file
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
index abe5305..6f95895 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
@@ -28,6 +28,14 @@ import net.schmizz.sshj.userauth.method.ChallengeResponseProvider;
import net.schmizz.sshj.userauth.password.Resource;
import org.apache.airavata.mft.core.ResourceMetadata;
import org.apache.airavata.mft.core.api.MetadataCollector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.resource.service.SCPResource;
+import org.apache.airavata.mft.resource.service.SCPResourceGetRequest;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
import java.io.IOException;
import java.util.ArrayList;
@@ -38,11 +46,16 @@ public class SCPMetadataCollector implements MetadataCollector {
public ResourceMetadata getGetResourceMetadata(String resourceId, String credentialToken) throws IOException {
- SSHResourceIdentifier sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+ SCPResource scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+ SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
try (SSHClient sshClient = new SSHClient()) {
sshClient.addHostKeyVerifier((h, p, key) -> true);
- KeyProvider keyProvider = sshClient.loadKeys(sshResourceIdentifier.getKeyFile(), sshResourceIdentifier.getKeyPassphrase());
+ KeyProvider keyProvider = sshClient.loadKeys(scpSecret.getPrivateKey(), scpSecret.getPassphrase());
final List<AuthMethod> am = new LinkedList<>();
am.add(new AuthPublickey(keyProvider));
am.add(new AuthKeyboardInteractive(new ChallengeResponseProvider() {
@@ -65,11 +78,11 @@ public class SCPMetadataCollector implements MetadataCollector {
}
}));
- sshClient.connect(sshResourceIdentifier.getHost(), sshResourceIdentifier.getPort());
- sshClient.auth(sshResourceIdentifier.getUser(), am);
+ sshClient.connect(scpResource.getScpStorage().getHost(), scpResource.getScpStorage().getPort());
+ sshClient.auth(scpSecret.getUser(), am);
try (SFTPClient sftpClient = sshClient.newSFTPClient()) {
- FileAttributes lstat = sftpClient.lstat(sshResourceIdentifier.getRemotePath());
+ FileAttributes lstat = sftpClient.lstat(scpResource.getResourcePath());
sftpClient.close();
ResourceMetadata metadata = new ResourceMetadata();
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index e56e9e0..141395f 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -22,6 +22,12 @@ import com.jcraft.jsch.Session;
import org.apache.airavata.mft.core.CircularStreamingBuffer;
import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
import java.io.IOException;
import java.io.InputStream;
@@ -29,14 +35,21 @@ import java.io.OutputStream;
public class SCPReceiver implements Connector {
private Session session;
- private SSHResourceIdentifier sshResourceIdentifier;
+ private SCPResource scpResource;
public void init(String resourceId, String credentialToken) throws Exception {
- this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
- this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(),
- sshResourceIdentifier.getPort(),
- sshResourceIdentifier.getKeyFile(),
- sshResourceIdentifier.getKeyPassphrase());
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+ this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+ SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ this.session = SCPTransportUtil.createSession(
+ scpSecret.getUser(),
+ scpResource.getScpStorage().getHost(),
+ scpResource.getScpStorage().getPort(),
+ scpSecret.getPrivateKey(),
+ scpSecret.getPassphrase());
}
public void destroy() {
@@ -53,7 +66,7 @@ public class SCPReceiver implements Connector {
throw new Exception("Session can not be null. Make sure that SCP Receiver is properly initialized");
}
- transferRemoteToStream(session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer());
+ transferRemoteToStream(session, this.scpResource.getResourcePath(), context.getStreamBuffer());
}
private void transferRemoteToStream(Session session, String from, CircularStreamingBuffer streamBuffer) throws Exception {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index 31bb138..7f6b217 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -23,20 +23,34 @@ import com.jcraft.jsch.Session;
import org.apache.airavata.mft.core.CircularStreamingBuffer;
import org.apache.airavata.mft.core.ConnectorContext;
import org.apache.airavata.mft.core.api.Connector;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import org.apache.airavata.mft.resource.service.SCPResource;
+import org.apache.airavata.mft.resource.service.SCPResourceGetRequest;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
public class SCPSender implements Connector {
- private SSHResourceIdentifier sshResourceIdentifier;
private Session session;
+ private SCPResource scpResource;
public void init(String resourceId, String credentialToken) {
- this.sshResourceIdentifier = SCPTransportUtil.getSSHResourceIdentifier(resourceId);
- this.session = SCPTransportUtil.createSession(sshResourceIdentifier.getUser(), sshResourceIdentifier.getHost(),
- sshResourceIdentifier.getPort(),
- sshResourceIdentifier.getKeyFile(),
- sshResourceIdentifier.getKeyPassphrase());
+
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+ this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
+
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+ SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+
+ this.session = SCPTransportUtil.createSession(scpSecret.getUser(), scpResource.getScpStorage().getHost(),
+ scpResource.getScpStorage().getPort(),
+ scpSecret.getPrivateKey(),
+ scpSecret.getPassphrase());
}
public void destroy() {
@@ -53,7 +67,7 @@ public class SCPSender implements Connector {
System.out.println("Session can not be null. Make sure that SCP Sender is properly initialized");
throw new Exception("Session can not be null. Make sure that SCP Sender is properly initialized");
}
- copyLocalToRemote(this.session, sshResourceIdentifier.getRemotePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
+ copyLocalToRemote(this.session, this.scpResource.getResourcePath(), context.getStreamBuffer(), context.getMetadata().getResourceSize());
}
private void copyLocalToRemote(Session session, String to, CircularStreamingBuffer streamBuffer, long fileSize) throws JSchException, IOException {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
index d81e3dc..bcb8342 100755
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
@@ -20,11 +20,41 @@ package org.apache.airavata.mft.transport.scp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
+import org.apache.airavata.mft.resource.client.ResourceServiceClient;
+import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.mft.secret.client.SecretServiceClient;
+import org.apache.airavata.mft.secret.service.SCPSecret;
+import org.apache.airavata.mft.secret.service.SCPSecretCreateRequest;
+import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
import java.util.Properties;
public class SCPTransportUtil {
// TODO replace with an API call to the registry
+
+
+ public static void main(String args[]) {
+ ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
+ SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
+
+ //resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId("82848d70-ef71-47d2-935a-4fd3e0184a42").build());
+ SCPStorage scpStorage1 = resourceClient.createSCPStorage(SCPStorageCreateRequest.newBuilder().setPort(22).setHost("pgadev.scigap.org").build());
+ SCPResource scpResource1 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId())
+ .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/2mb.txt").build());
+ SCPResource scpResource2 = resourceClient.createSCPResource(SCPResourceCreateRequest.newBuilder().setScpStorageId(scpStorage1.getStorageId())
+ .setResourcePath("/var/www/portals/gateway-user-data/dev-seagrid/eromads6/DefaultProject/Gaussian_C11470169729/new-file.txt").build());
+
+ System.out.println("Resource 1 id " + scpResource1.getResourceId());
+ System.out.println("Resource 2 id " + scpResource2.getResourceId());
+
+ SCPSecret scpSecret1 = secretClient.createSCPSecret(SCPSecretCreateRequest.newBuilder()
+ .setUser("pga")
+ .setPassphrase("shithappens@12")
+ .setPrivateKey("/Users/dimuthu/.ssh/id_rsa").build());
+
+ System.out.println("Secret id " + scpSecret1.getSecretId());
+ }
+
public static SSHResourceIdentifier getSSHResourceIdentifier(String resourceId) {
SSHResourceIdentifier identifier = new SSHResourceIdentifier();