You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/13 04:02:25 UTC
[airavata-mft] branch master updated: Using Consul kv store an
watches to submit transfer requests
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
The following commit(s) were added to refs/heads/master by this push:
new db97bb0 Using Consul kv store an watches to submit transfer requests
db97bb0 is described below
commit db97bb03ad0d5e6ac2118ae22b49f54e7817cb40
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Dec 12 23:02:06 2019 -0500
Using Consul kv store an watches to submit transfer requests
---
agent/pom.xml | 5 ++
.../apache/airavata/mft/agent/ConsulTester.java | 26 +++++++
.../org/apache/airavata/mft/agent/MFTAgent.java | 80 ++++++++++++++--------
.../apache/airavata/mft/agent/TransferRequest.java | 21 ++++++
pom.xml | 1 +
.../airavata/mft/transport/scp/SCPReceiver.java | 2 +-
.../airavata/mft/transport/scp/SCPSender.java | 8 +--
7 files changed, 110 insertions(+), 33 deletions(-)
diff --git a/agent/pom.xml b/agent/pom.xml
index a8846f5..24dfbfd 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -43,5 +43,10 @@
<artifactId>mft-local-transport</artifactId>
<version>0.01-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>com.orbitz.consul</groupId>
+ <artifactId>consul-client</artifactId>
+ <version>${consul.client}</version>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
new file mode 100644
index 0000000..589466d
--- /dev/null
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
@@ -0,0 +1,26 @@
+package org.apache.airavata.mft.agent;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.orbitz.consul.Consul;
+import com.orbitz.consul.KeyValueClient;
+import java.util.Collections;
+
+public class ConsulTester {
+ public static void main(String args[]) throws Exception {
+ Consul client = Consul.builder().build();
+ KeyValueClient kvClient = client.keyValueClient();
+ ObjectMapper mapper = new ObjectMapper();
+
+ TransferRequest request = new TransferRequest();
+ request.setSourceId("1");
+ request.setSourceType("SCP");
+ request.setDestinationId("2");
+ request.setDestinationType("SCP");
+ request.setAgentList(Collections.singletonList("agent0"));
+ request.setTransferId("transfer005");
+
+ String asString = mapper.writeValueAsString(request);
+
+ kvClient.putValue("agent0/messages/" + request.getTransferId(), asString);
+ }
+}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index d788995..3c2b9c1 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -1,5 +1,9 @@
package org.apache.airavata.mft.agent;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.orbitz.consul.Consul;
+import com.orbitz.consul.KeyValueClient;
+import com.orbitz.consul.cache.KVCache;
import org.apache.airavata.mft.core.ResourceMetadata;
import org.apache.airavata.mft.core.api.Connector;
import org.apache.airavata.mft.core.api.MetadataCollector;
@@ -10,43 +14,63 @@ import org.apache.airavata.mft.transport.scp.SCPMetadataCollector;
import org.apache.airavata.mft.transport.scp.SCPReceiver;
import org.apache.airavata.mft.transport.scp.SCPSender;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Semaphore;
public class MFTAgent {
- private List<TransferRequest> requests = new ArrayList<>();
private TransportMediator mediator = new TransportMediator();
+ private String agentId = "agent0";
+ private Semaphore mainHold = new Semaphore(0);
- public void acceptRequests() {
- for (TransferRequest request : requests) {
-
- try {
- Connector inConnector = resolveConnector(request.getSourceType(), "IN");
- inConnector.init(request.getSourceId(), request.getSourceToken());
- Connector outConnector = resolveConnector(request.getDestinationType(), "OUT");
- outConnector.init(request.getDestinationId(), request.getDestinationToken());
-
- MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
- ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
- System.out.println("File size " + metadata.getResourceSize());
- String transferId = mediator.transfer(inConnector, outConnector, metadata);
- System.out.println("Submitted transfer " + transferId);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ private void acceptRequests() {
+ Consul client = Consul.builder().build();
+ KeyValueClient kvClient = client.keyValueClient();
+
+ KVCache messageCache = KVCache.newCache(kvClient, agentId + "/messages");
+ messageCache.addListener(newValues -> {
+ // Cache notifies all paths with "foo" the root path
+ // If you want to watch only "foo" value, you must filter other paths
+
+ newValues.values().forEach(value -> {
+
+ // Values are encoded in key/value store, decode it if needed
+ Optional<String> decodedValue = value.getValueAsString();
+ decodedValue.ifPresent(v -> {
+ System.out.println(String.format("Value is: %s", v));
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ TransferRequest request = mapper.readValue(v, TransferRequest.class);
+ System.out.println("Received request " + request.getTransferId());
+
+ Connector inConnector = resolveConnector(request.getSourceType(), "IN");
+ inConnector.init(request.getSourceId(), request.getSourceToken());
+ Connector outConnector = resolveConnector(request.getDestinationType(), "OUT");
+ outConnector.init(request.getDestinationId(), request.getDestinationToken());
+
+ MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
+ ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
+ System.out.println("File size " + metadata.getResourceSize());
+ String transferId = mediator.transfer(inConnector, outConnector, metadata);
+ System.out.println("Submitted transfer " + transferId);
+
+ System.out.println("Deleting key " + value.getKey());
+ kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }); //prints "bar"
+
+ });
+ });
+ messageCache.start();
}
- public static void main(String args[]) {
- TransferRequest request = new TransferRequest();
- request.setSourceId("1");
- request.setSourceType("SCP");
- request.setDestinationId("2");
- request.setDestinationType("SCP");
+
+ public static void main(String args[]) throws InterruptedException {
MFTAgent agent = new MFTAgent();
- agent.requests.add(request);
agent.acceptRequests();
+ agent.mainHold.acquire();
}
// TODO load from reflection to avoid dependencies
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
index 42c4b40..008a55a 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
@@ -1,12 +1,17 @@
package org.apache.airavata.mft.agent;
+import java.util.List;
+
public class TransferRequest {
+
+ private String transferId;
private String sourceId;
private String sourceType;
private String sourceToken;
private String destinationId;
private String destinationType;
private String destinationToken;
+ private List<String> agentList;
public String getSourceId() {
return sourceId;
@@ -55,4 +60,20 @@ public class TransferRequest {
public void setDestinationToken(String destinationToken) {
this.destinationToken = destinationToken;
}
+
+ public List<String> getAgentList() {
+ return agentList;
+ }
+
+ public void setAgentList(List<String> agentList) {
+ this.agentList = agentList;
+ }
+
+ public String getTransferId() {
+ return transferId;
+ }
+
+ public void setTransferId(String transferId) {
+ this.transferId = transferId;
+ }
}
diff --git a/pom.xml b/pom.xml
index 821b297..965c829 100755
--- a/pom.xml
+++ b/pom.xml
@@ -107,6 +107,7 @@
<protobuf.maven.plugin>0.5.1</protobuf.maven.plugin>
<os.maven.plugin>1.5.0.Final</os.maven.plugin>
<javax.annotation>1.3.2</javax.annotation>
+ <consul.client>1.3.8</consul.client>
</properties>
</project>
\ No newline at end of file
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index 497159d..9d5af13 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -115,7 +115,7 @@ public class SCPReceiver implements Connector {
}
if (checkAck(in) != 0) {
- System.exit(0);
+ throw new IOException("Error code found in ack " + (checkAck(in)));
}
// send '\0'
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index 0456801..a767594 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -57,7 +57,7 @@ public class SCPSender implements Connector {
channel.connect();
if (checkAck(in) != 0) {
- System.exit(0);
+ throw new IOException("Error code found in ack " + (checkAck(in)));
}
if (ptimestamp) {
@@ -68,7 +68,7 @@ public class SCPSender implements Connector {
out.write(command.getBytes());
out.flush();
if (checkAck(in) != 0) {
- System.exit(0);
+ throw new IOException("Error code found in ack " + (checkAck(in)));
}
}
@@ -85,7 +85,7 @@ public class SCPSender implements Connector {
out.flush();
if (checkAck(in) != 0) {
- System.exit(0);
+ throw new IOException("Error code found in ack " + (checkAck(in)));
}
// send a content of lfile
@@ -111,7 +111,7 @@ public class SCPSender implements Connector {
out.flush();
if (checkAck(in) != 0) {
- System.exit(0);
+ throw new IOException("Error code found in ack " + (checkAck(in)));
}
out.close();