You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/01/02 10:42:12 UTC
[airavata-mft] branch master updated: Integrating Resource and
Secret backends with Airavata
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new f4c8814 Integrating Resource and Secret backends with Airavata
f4c8814 is described below
commit f4c8814f5e76e7fd0fe50bc8be14e010879d38af
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Jan 2 05:41:55 2020 -0500
Integrating Resource and Secret backends with Airavata
---
.../org/apache/airavata/mft/admin/MFTAdmin.java | 14 +-
.../airavata/mft/admin/models/TransferRequest.java | 9 --
agent/pom.xml | 5 +
.../apache/airavata/mft/agent/ConsulTester.java | 1 -
.../org/apache/airavata/mft/agent/MFTAgent.java | 14 +-
agent/src/main/resources/logback.xml | 54 ++++++++
.../mft/resource/client/ResourceServiceClient.java | 18 ++-
services/resource-service/server/pom.xml | 5 +
.../airavata/mft/resource/server/AppConfig.java | 3 +-
.../resource/server/backend/ResourceBackend.java | 5 +-
.../backend/airavata/AiravataResourceBackend.java | 141 +++++++++++++++++++++
.../server/handler/ResourceServiceHandler.java | 34 +++--
.../mft/secret/client/SecretServiceClient.java | 18 ++-
services/secret-service/server/pom.xml | 24 ++++
.../airavata/mft/secret/server/AppConfig.java | 3 +-
.../mft/secret/server/backend/SecretBackend.java | 2 +-
.../backend/airavata/AiravataSecretBackend.java | 62 +++++++++
.../server/handler/SecretServiceHandler.java | 17 ++-
.../mft/transport/scp/SCPMetadataCollector.java | 11 +-
.../airavata/mft/transport/scp/SCPReceiver.java | 13 +-
.../airavata/mft/transport/scp/SCPSender.java | 13 +-
.../mft/transport/scp/SCPTransportUtil.java | 2 +-
22 files changed, 413 insertions(+), 55 deletions(-)
diff --git a/admin/src/main/java/org/apache/airavata/mft/admin/MFTAdmin.java b/admin/src/main/java/org/apache/airavata/mft/admin/MFTAdmin.java
index 27098a1..2eb9d66 100644
--- a/admin/src/main/java/org/apache/airavata/mft/admin/MFTAdmin.java
+++ b/admin/src/main/java/org/apache/airavata/mft/admin/MFTAdmin.java
@@ -27,10 +27,7 @@ import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferRequest;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
+import java.util.*;
import java.util.stream.Collectors;
/*
@@ -45,10 +42,13 @@ public class MFTAdmin {
private KeyValueClient kvClient = client.keyValueClient();
private ObjectMapper mapper = new ObjectMapper();
- public void submitTransfer(String agentId, TransferRequest transferRequest) throws MFTAdminException {
+ public String submitTransfer(String agentId, TransferRequest transferRequest) throws MFTAdminException {
try {
+ String transferId = UUID.randomUUID().toString();
+ transferRequest.setTransferId(transferId);
String asString = mapper.writeValueAsString(transferRequest);
- kvClient.putValue("mft/agents/messages/" + agentId + "/" + transferRequest.getTransferId(), asString);
+ kvClient.putValue("mft/agents/messages/" + agentId + "/" + transferId, asString);
+ return transferId;
} catch (JsonProcessingException e) {
throw new MFTAdminException("Error in serializing transfer request", e);
}
@@ -69,7 +69,7 @@ public class MFTAdmin {
if (value.isPresent()) {
Value absVal = value.get();
if (absVal.getValue().isPresent()) {
- String asStr = absVal.getValue().get();
+ String asStr = absVal.getValueAsString().get();
try {
return Optional.of(mapper.readValue(asStr, AgentInfo.class));
} catch (IOException e) {
diff --git a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java
index 5183a1c..13a549f 100644
--- a/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java
+++ b/admin/src/main/java/org/apache/airavata/mft/admin/models/TransferRequest.java
@@ -28,7 +28,6 @@ public class TransferRequest {
private String destinationId;
private String destinationType;
private String destinationToken;
- private List<String> agentList;
public String getSourceId() {
return sourceId;
@@ -78,14 +77,6 @@ public class TransferRequest {
this.destinationToken = destinationToken;
}
- public List<String> getAgentList() {
- return agentList;
- }
-
- public void setAgentList(List<String> agentList) {
- this.agentList = agentList;
- }
-
public String getTransferId() {
return transferId;
}
diff --git a/agent/pom.xml b/agent/pom.xml
index 77b2679..b317647 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -53,6 +53,11 @@
<artifactId>consul-client</artifactId>
<version>${consul.client}</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>log4j-over-slf4j</artifactId>
+ <version>1.7.26</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
index 499e603..f11dfbf 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
@@ -39,7 +39,6 @@ public class ConsulTester {
request.setDestinationType("SCP");
request.setDestinationToken("866d421e-3624-434f-ae71-04a90d39e70c");
- request.setAgentList(Collections.singletonList("agent0"));
request.setTransferId("transfer010");
String asString = mapper.writeValueAsString(request);
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index fba770f..c19906f 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -26,6 +26,9 @@ import com.orbitz.consul.cache.KVCache;
import com.orbitz.consul.model.kv.Value;
import com.orbitz.consul.model.session.ImmutableSession;
import com.orbitz.consul.model.session.SessionCreatedResponse;
+import org.apache.airavata.mft.admin.MFTAdmin;
+import org.apache.airavata.mft.admin.MFTAdminException;
+import org.apache.airavata.mft.admin.models.AgentInfo;
import org.apache.airavata.mft.admin.models.TransferRequest;
import org.apache.airavata.mft.core.ResourceMetadata;
import org.apache.airavata.mft.core.TransportMediator;
@@ -38,6 +41,7 @@ import org.apache.airavata.mft.transport.scp.SCPMetadataCollector;
import org.apache.airavata.mft.transport.scp.SCPReceiver;
import org.apache.airavata.mft.transport.scp.SCPSender;
+import java.util.Collections;
import java.util.Optional;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -106,7 +110,7 @@ public class MFTAgent {
messageCache.start();
}
- private boolean connectAgent() {
+ private boolean connectAgent() throws MFTAdminException {
ImmutableSession session = ImmutableSession.builder().name(agentId).behavior("delete").ttl(sessionTTLSeconds + "s").build();
SessionCreatedResponse sessResp = client.sessionClient().createSession(session);
String lockPath = "mft/agent/live/" + agentId;
@@ -131,7 +135,13 @@ public class MFTAgent {
stop();
}
}
- }, sessionRenewSeconds, sessionRenewSeconds, TimeUnit.SECONDS);
+ }, sessionRenewSeconds, sessionRenewSeconds, TimeUnit.SECONDS);
+ MFTAdmin admin = new MFTAdmin();
+ admin.registerAgent(new AgentInfo()
+ .setId(agentId)
+ .setHost("localhost")
+ .setUser("dimuthu")
+ .setSupportedProtocols(Collections.singletonList("SCP")));
}
System.out.println("Lock status " + acquired);
diff --git a/agent/src/main/resources/logback.xml b/agent/src/main/resources/logback.xml
new file mode 100644
index 0000000..a1e67fe
--- /dev/null
+++ b/agent/src/main/resources/logback.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>../logs/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %m [%X]%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>../logs/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="net.schmizz.sshj" level="WARN"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>
diff --git a/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/ResourceServiceClient.java b/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/ResourceServiceClient.java
index e83f668..4618a47 100644
--- a/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/ResourceServiceClient.java
+++ b/services/resource-service/client/src/main/java/org/apache/airavata/mft/resource/client/ResourceServiceClient.java
@@ -21,9 +21,25 @@ import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import org.apache.airavata.mft.resource.service.ResourceServiceGrpc;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
public class ResourceServiceClient {
+
+ private static Map<String, Map<Integer, ResourceServiceGrpc.ResourceServiceBlockingStub>> stubCache = new ConcurrentHashMap<>();
+
public static ResourceServiceGrpc.ResourceServiceBlockingStub buildClient(String hostName, int port) {
+
+ if (stubCache.containsKey(hostName)) {
+ if (stubCache.get(hostName).containsKey(port)) {
+ return stubCache.get(hostName).get(port);
+ }
+ }
+
ManagedChannel channel = ManagedChannelBuilder.forAddress(hostName, port).usePlaintext().build();
- return ResourceServiceGrpc.newBlockingStub(channel);
+ ResourceServiceGrpc.ResourceServiceBlockingStub stub = ResourceServiceGrpc.newBlockingStub(channel);
+ stubCache.put(hostName, Collections.singletonMap(port, stub));
+ return stub;
}
}
diff --git a/services/resource-service/server/pom.xml b/services/resource-service/server/pom.xml
index 9fe67fc..ed6c8f6 100644
--- a/services/resource-service/server/pom.xml
+++ b/services/resource-service/server/pom.xml
@@ -44,5 +44,10 @@
<version>${h2}</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>registry-api-stubs</artifactId>
+ <version>0.19-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/AppConfig.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/AppConfig.java
index a21a9d3..dce3d7a 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/AppConfig.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/AppConfig.java
@@ -18,6 +18,7 @@
package org.apache.airavata.mft.resource.server;
import org.apache.airavata.mft.resource.server.backend.ResourceBackend;
+import org.apache.airavata.mft.resource.server.backend.airavata.AiravataResourceBackend;
import org.apache.airavata.mft.resource.server.backend.sql.SQLResourceBackend;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -26,6 +27,6 @@ import org.springframework.context.annotation.Configuration;
public class AppConfig {
@Bean
public ResourceBackend resourceBackend() {
- return new SQLResourceBackend();
+ return new AiravataResourceBackend();
}
}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
index 02c5d28..8b5beb0 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java
@@ -18,16 +18,17 @@
package org.apache.airavata.mft.resource.server.backend;
import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.registry.api.exception.RegistryServiceException;
import java.util.Optional;
public interface ResourceBackend {
- public Optional<SCPStorage> getSCPStorage(SCPStorageGetRequest request);
+ public Optional<SCPStorage> getSCPStorage(SCPStorageGetRequest request) throws Exception;
public SCPStorage createSCPStorage(SCPStorageCreateRequest request);
public boolean updateSCPStorage(SCPStorageUpdateRequest request);
public boolean deleteSCPStorage(SCPStorageDeleteRequest request);
- public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request);
+ public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request) throws Exception;
public SCPResource createSCPResource(SCPResourceCreateRequest request);
public boolean updateSCPResource(SCPResourceUpdateRequest request);
public boolean deleteSCPResource(SCPResourceDeleteRequest request);
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java
new file mode 100644
index 0000000..74c4005
--- /dev/null
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/airavata/AiravataResourceBackend.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.resource.server.backend.airavata;
+
+import org.apache.airavata.mft.resource.server.backend.ResourceBackend;
+import org.apache.airavata.mft.resource.service.*;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
+import org.apache.airavata.model.data.movement.DataMovementInterface;
+import org.apache.airavata.model.data.movement.DataMovementProtocol;
+import org.apache.airavata.model.data.movement.SCPDataMovement;
+import org.apache.airavata.registry.api.RegistryService;
+import org.apache.airavata.registry.api.client.RegistryServiceClientFactory;
+import org.apache.airavata.registry.api.exception.RegistryServiceException;
+
+import java.util.Optional;
+
+public class AiravataResourceBackend implements ResourceBackend {
+ @Override
+ public Optional<SCPStorage> getSCPStorage(SCPStorageGetRequest request) throws Exception {
+
+ String resourceId = request.getStorageId();
+ String[] parts = resourceId.split(":");
+ String type = parts[0];
+ String path = parts[1];
+ String gateway = parts[2];
+ String storageOrComputeId = parts[3];
+
+ RegistryService.Client registryClient = RegistryServiceClientFactory.createRegistryClient("localhost", 8970);
+ SCPStorage.Builder builder = SCPStorage.newBuilder().setStorageId(resourceId);
+ if ("STORAGE".equals(type)) {
+
+ StorageResourceDescription storageResource = registryClient.getStorageResource(storageOrComputeId);
+
+ Optional<DataMovementInterface> dmInterfaceOp = storageResource.getDataMovementInterfaces()
+ .stream().filter(iface -> iface.getDataMovementProtocol() == DataMovementProtocol.SCP).findFirst();
+
+ DataMovementInterface scpInterface = dmInterfaceOp
+ .orElseThrow(() -> new Exception("Could not find a SCP interface for storage resource " + storageOrComputeId));
+
+ SCPDataMovement scpDataMovement = registryClient.getSCPDataMovement(scpInterface.getDataMovementInterfaceId());
+
+ String alternateHostName = scpDataMovement.getAlternativeSCPHostName();
+ String selectedHostName = (alternateHostName == null || "".equals(alternateHostName))?
+ storageResource.getHostName() : alternateHostName;
+
+ int selectedPort = scpDataMovement.getSshPort() == 0 ? 22 : scpDataMovement.getSshPort();
+
+ builder.setHost(selectedHostName);
+ builder.setPort(selectedPort);
+
+ } else if ("CLUSTER".equals(type)) {
+ ComputeResourceDescription computeResource = registryClient.getComputeResource(storageOrComputeId);
+ builder.setHost(computeResource.getHostName());
+ builder.setPort(22);
+ }
+ return Optional.of(builder.build());
+ }
+
+ @Override
+ public SCPStorage createSCPStorage(SCPStorageCreateRequest request) {
+ return null;
+ }
+
+ @Override
+ public boolean updateSCPStorage(SCPStorageUpdateRequest request) {
+ return false;
+ }
+
+ @Override
+ public boolean deleteSCPStorage(SCPStorageDeleteRequest request) {
+ return false;
+ }
+
+ @Override
+ public Optional<SCPResource> getSCPResource(SCPResourceGetRequest request) throws Exception {
+ String resourceId = request.getResourceId();
+ String[] parts = resourceId.split(":");
+ String type = parts[0];
+ String path = parts[1];
+ String gateway = parts[2];
+ String storageOrComputeId = parts[3];
+
+ SCPResource scpResource = SCPResource.newBuilder()
+ .setResourceId(resourceId)
+ .setResourcePath(path)
+ .setScpStorage(getSCPStorage(SCPStorageGetRequest.newBuilder().setStorageId(resourceId).build()).get())
+ .build();
+ return Optional.of(scpResource);
+ }
+
+ @Override
+ public SCPResource createSCPResource(SCPResourceCreateRequest request) {
+ return null;
+ }
+
+ @Override
+ public boolean updateSCPResource(SCPResourceUpdateRequest request) {
+ return false;
+ }
+
+ @Override
+ public boolean deleteSCPResource(SCPResourceDeleteRequest request) {
+ return false;
+ }
+
+ @Override
+ public Optional<LocalResource> getLocalResource(LocalResourceGetRequest request) {
+ return Optional.empty();
+ }
+
+ @Override
+ public LocalResource createLocalResource(LocalResourceCreateRequest request) {
+ return null;
+ }
+
+ @Override
+ public boolean updateLocalResource(LocalResourceUpdateRequest request) {
+ return false;
+ }
+
+ @Override
+ public boolean deleteLocalResource(LocalResourceDeleteRequest request) {
+ return false;
+ }
+}
diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
index c4baa84..7dd4da4 100644
--- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
+++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/ResourceServiceHandler.java
@@ -48,12 +48,17 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
@Override
public void getSCPStorage(SCPStorageGetRequest request, StreamObserver<SCPStorage> responseObserver) {
- this.backend.getSCPStorage(request).ifPresentOrElse(storage -> {
- responseObserver.onNext(storage);
- responseObserver.onCompleted();
- }, () -> {
- responseObserver.onError(new Exception("No SCP Storage with id " + request.getStorageId()));
- });
+ try {
+ this.backend.getSCPStorage(request).ifPresentOrElse(storage -> {
+ responseObserver.onNext(storage);
+ responseObserver.onCompleted();
+ }, () -> {
+ responseObserver.onError(new Exception("No SCP Storage with id " + request.getStorageId()));
+ });
+ } catch (Exception e) {
+ e.printStackTrace();
+ responseObserver.onError(new Exception("Failed in retrieving storage with id " + request.getStorageId()));
+ }
}
@Override
@@ -80,12 +85,17 @@ public class ResourceServiceHandler extends ResourceServiceGrpc.ResourceServiceI
@Override
public void getSCPResource(SCPResourceGetRequest request, StreamObserver<SCPResource> responseObserver) {
- this.backend.getSCPResource(request).ifPresentOrElse(resource -> {
- responseObserver.onNext(resource);
- responseObserver.onCompleted();
- }, () -> {
- responseObserver.onError(new Exception("No SCP Resource with id " + request.getResourceId()));
- });
+ try {
+ this.backend.getSCPResource(request).ifPresentOrElse(resource -> {
+ responseObserver.onNext(resource);
+ responseObserver.onCompleted();
+ }, () -> {
+ responseObserver.onError(new Exception("No SCP Resource with id " + request.getResourceId()));
+ });
+ } catch (Exception e) {
+ e.printStackTrace();
+ responseObserver.onError(new Exception("Failed in retrieving resource with id " + request.getResourceId()));
+ }
}
@Override
diff --git a/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java b/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java
index bc7bc7e..47050f2 100644
--- a/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java
+++ b/services/secret-service/client/src/main/java/org/apache/airavata/mft/secret/client/SecretServiceClient.java
@@ -21,9 +21,25 @@ import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
public class SecretServiceClient {
+
+ private static Map<String, Map<Integer, SecretServiceGrpc.SecretServiceBlockingStub>> stubCache = new ConcurrentHashMap<>();
+
public static SecretServiceGrpc.SecretServiceBlockingStub buildClient(String hostName, int port) {
+
+ if (stubCache.containsKey(hostName)) {
+ if (stubCache.get(hostName).containsKey(port)) {
+ return stubCache.get(hostName).get(port);
+ }
+ }
+
ManagedChannel channel = ManagedChannelBuilder.forAddress(hostName, port).usePlaintext().build();
- return SecretServiceGrpc.newBlockingStub(channel);
+ SecretServiceGrpc.SecretServiceBlockingStub stub = SecretServiceGrpc.newBlockingStub(channel);
+ stubCache.put(hostName, Collections.singletonMap(port, stub));
+ return stub;
}
}
diff --git a/services/secret-service/server/pom.xml b/services/secret-service/server/pom.xml
index 0f0a042..76026f5 100644
--- a/services/secret-service/server/pom.xml
+++ b/services/secret-service/server/pom.xml
@@ -44,5 +44,29 @@
<version>${h2}</version>
<scope>runtime</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-credential-store-stubs</artifactId>
+ <version>0.19-SNAPSHOT</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.openjpa</groupId>
+ <artifactId>openjpa</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId> org.apache.airavata</groupId>
+ <artifactId>airavata-commons</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-server-configuration</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-data-models</artifactId>
+ <version>0.19-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
index fb2d2d1..4937125 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/AppConfig.java
@@ -18,6 +18,7 @@
package org.apache.airavata.mft.secret.server;
import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.server.backend.airavata.AiravataSecretBackend;
import org.apache.airavata.mft.secret.server.backend.sql.SQLSecretBackend;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -26,5 +27,5 @@ import org.springframework.context.annotation.Configuration;
public class AppConfig {
@Bean
- SecretBackend secretBackend() {return new SQLSecretBackend();}
+ SecretBackend secretBackend() {return new AiravataSecretBackend();}
}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
index 9648747..005bbd4 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/SecretBackend.java
@@ -22,7 +22,7 @@ import org.apache.airavata.mft.secret.service.*;
import java.util.Optional;
public interface SecretBackend {
- public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request);
+ public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) throws Exception;
public SCPSecret createSCPSecret(SCPSecretCreateRequest request);
public boolean updateSCPSecret(SCPSecretUpdateRequest request);
public boolean deleteSCPSecret(SCPSecretDeleteRequest request);
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java
new file mode 100644
index 0000000..c92c197
--- /dev/null
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/backend/airavata/AiravataSecretBackend.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.secret.server.backend.airavata;
+
+import org.apache.airavata.credential.store.client.CredentialStoreClientFactory;
+import org.apache.airavata.credential.store.cpi.CredentialStoreService;
+import org.apache.airavata.mft.secret.server.backend.SecretBackend;
+import org.apache.airavata.mft.secret.service.*;
+import org.apache.airavata.model.credential.store.SSHCredential;
+
+import java.util.Optional;
+
+public class AiravataSecretBackend implements SecretBackend {
+ @Override
+ public Optional<SCPSecret> getSCPSecret(SCPSecretGetRequest request) throws Exception {
+ CredentialStoreService.Client csClient = CredentialStoreClientFactory.createAiravataCSClient("localhost", 8960);
+ String secretId = request.getSecretId();
+ String[] parts = secretId.split(":");
+ String csToken = parts[0];
+ String user = parts[1];
+ String gateway = parts[2];
+ SSHCredential sshCredential = csClient.getSSHCredential(csToken, gateway);
+
+ SCPSecret scpSecret = SCPSecret.newBuilder()
+ .setPrivateKey(sshCredential.getPrivateKey())
+ .setPublicKey(sshCredential.getPublicKey())
+ .setPassphrase(sshCredential.getPassphrase())
+ .setUser(user).build();
+
+ return Optional.of(scpSecret);
+ }
+
+ @Override
+ public SCPSecret createSCPSecret(SCPSecretCreateRequest request) {
+ return null;
+ }
+
+ @Override
+ public boolean updateSCPSecret(SCPSecretUpdateRequest request) {
+ return false;
+ }
+
+ @Override
+ public boolean deleteSCPSecret(SCPSecretDeleteRequest request) {
+ return false;
+ }
+}
diff --git a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
index 26f20fb..d88192a 100644
--- a/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
+++ b/services/secret-service/server/src/main/java/org/apache/airavata/mft/secret/server/handler/SecretServiceHandler.java
@@ -32,12 +32,17 @@ public class SecretServiceHandler extends SecretServiceGrpc.SecretServiceImplBas
@Override
public void getSCPSecret(SCPSecretGetRequest request, StreamObserver<SCPSecret> responseObserver) {
- this.backend.getSCPSecret(request).ifPresentOrElse(secret -> {
- responseObserver.onNext(secret);
- responseObserver.onCompleted();
- }, () -> {
- responseObserver.onError(new Exception("No SCP Secret with id " + request.getSecretId()));
- });
+ try {
+ this.backend.getSCPSecret(request).ifPresentOrElse(secret -> {
+ responseObserver.onNext(secret);
+ responseObserver.onCompleted();
+ }, () -> {
+ responseObserver.onError(new Exception("No SCP Secret with id " + request.getSecretId()));
+ });
+ } catch (Exception e) {
+ e.printStackTrace();
+ responseObserver.onError(new Exception("Error in retrieving Secret with id " + request.getSecretId()));
+ }
}
@Override
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
index 6f95895..b764414 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java
@@ -37,6 +37,9 @@ import org.apache.airavata.mft.secret.service.SCPSecret;
import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.LinkedList;
@@ -55,7 +58,13 @@ public class SCPMetadataCollector implements MetadataCollector {
try (SSHClient sshClient = new SSHClient()) {
sshClient.addHostKeyVerifier((h, p, key) -> true);
- KeyProvider keyProvider = sshClient.loadKeys(scpSecret.getPrivateKey(), scpSecret.getPassphrase());
+
+ File privateKeyFile = File.createTempFile("id_rsa", "");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(privateKeyFile));
+ writer.write(scpSecret.getPrivateKey());
+ writer.close();
+
+ KeyProvider keyProvider = sshClient.loadKeys(privateKeyFile.getPath(), scpSecret.getPassphrase());
final List<AuthMethod> am = new LinkedList<>();
am.add(new AuthPublickey(keyProvider));
am.add(new AuthKeyboardInteractive(new ChallengeResponseProvider() {
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index 141395f..aa48e46 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -29,9 +29,7 @@ import org.apache.airavata.mft.secret.service.SCPSecret;
import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
public class SCPReceiver implements Connector {
private Session session;
@@ -44,11 +42,18 @@ public class SCPReceiver implements Connector {
SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+ File privateKeyFile = File.createTempFile("id_rsa", "");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(privateKeyFile));
+ writer.write(scpSecret.getPrivateKey());
+ writer.close();
+
+ privateKeyFile.deleteOnExit();
+
this.session = SCPTransportUtil.createSession(
scpSecret.getUser(),
scpResource.getScpStorage().getHost(),
scpResource.getScpStorage().getPort(),
- scpSecret.getPrivateKey(),
+ privateKeyFile.getPath(),
scpSecret.getPassphrase());
}
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index 7f6b217..b081adc 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -32,14 +32,12 @@ import org.apache.airavata.mft.secret.service.SCPSecret;
import org.apache.airavata.mft.secret.service.SCPSecretGetRequest;
import org.apache.airavata.mft.secret.service.SecretServiceGrpc;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
public class SCPSender implements Connector {
private Session session;
private SCPResource scpResource;
- public void init(String resourceId, String credentialToken) {
+ public void init(String resourceId, String credentialToken) throws IOException {
ResourceServiceGrpc.ResourceServiceBlockingStub resourceClient = ResourceServiceClient.buildClient("localhost", 7002);
this.scpResource = resourceClient.getSCPResource(SCPResourceGetRequest.newBuilder().setResourceId(resourceId).build());
@@ -47,9 +45,14 @@ public class SCPSender implements Connector {
SecretServiceGrpc.SecretServiceBlockingStub secretClient = SecretServiceClient.buildClient("localhost", 7003);
SCPSecret scpSecret = secretClient.getSCPSecret(SCPSecretGetRequest.newBuilder().setSecretId(credentialToken).build());
+ File privateKeyFile = File.createTempFile("id_rsa", "");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(privateKeyFile));
+ writer.write(scpSecret.getPrivateKey());
+ writer.close();
+
this.session = SCPTransportUtil.createSession(scpSecret.getUser(), scpResource.getScpStorage().getHost(),
scpResource.getScpStorage().getPort(),
- scpSecret.getPrivateKey(),
+ privateKeyFile.getPath(),
scpSecret.getPassphrase());
}
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
index bcb8342..5174cb6 100755
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPTransportUtil.java
@@ -100,7 +100,7 @@ public class SCPTransportUtil {
return session;
} catch (JSchException e) {
- System.out.println(e);
+ e.printStackTrace();
return null;
}
}