You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2019/12/13 04:02:25 UTC

[airavata-mft] branch master updated: Using Consul kv store an watches to submit transfer requests

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git


The following commit(s) were added to refs/heads/master by this push:
     new db97bb0  Using Consul kv store an watches to submit transfer requests
db97bb0 is described below

commit db97bb03ad0d5e6ac2118ae22b49f54e7817cb40
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Dec 12 23:02:06 2019 -0500

    Using Consul kv store an watches to submit transfer requests
---
 agent/pom.xml                                      |  5 ++
 .../apache/airavata/mft/agent/ConsulTester.java    | 26 +++++++
 .../org/apache/airavata/mft/agent/MFTAgent.java    | 80 ++++++++++++++--------
 .../apache/airavata/mft/agent/TransferRequest.java | 21 ++++++
 pom.xml                                            |  1 +
 .../airavata/mft/transport/scp/SCPReceiver.java    |  2 +-
 .../airavata/mft/transport/scp/SCPSender.java      |  8 +--
 7 files changed, 110 insertions(+), 33 deletions(-)

diff --git a/agent/pom.xml b/agent/pom.xml
index a8846f5..24dfbfd 100644
--- a/agent/pom.xml
+++ b/agent/pom.xml
@@ -43,5 +43,10 @@
             <artifactId>mft-local-transport</artifactId>
             <version>0.01-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>com.orbitz.consul</groupId>
+            <artifactId>consul-client</artifactId>
+            <version>${consul.client}</version>
+        </dependency>
     </dependencies>
 </project>
\ No newline at end of file
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
new file mode 100644
index 0000000..589466d
--- /dev/null
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/ConsulTester.java
@@ -0,0 +1,26 @@
+package org.apache.airavata.mft.agent;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.orbitz.consul.Consul;
+import com.orbitz.consul.KeyValueClient;
+import java.util.Collections;
+
+public class ConsulTester {
+    public static void main(String args[]) throws Exception {
+        Consul client = Consul.builder().build();
+        KeyValueClient kvClient = client.keyValueClient();
+        ObjectMapper mapper = new ObjectMapper();
+
+        TransferRequest request = new TransferRequest();
+        request.setSourceId("1");
+        request.setSourceType("SCP");
+        request.setDestinationId("2");
+        request.setDestinationType("SCP");
+        request.setAgentList(Collections.singletonList("agent0"));
+        request.setTransferId("transfer005");
+
+        String asString = mapper.writeValueAsString(request);
+
+        kvClient.putValue("agent0/messages/" + request.getTransferId(), asString);
+    }
+}
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
index d788995..3c2b9c1 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/MFTAgent.java
@@ -1,5 +1,9 @@
 package org.apache.airavata.mft.agent;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.orbitz.consul.Consul;
+import com.orbitz.consul.KeyValueClient;
+import com.orbitz.consul.cache.KVCache;
 import org.apache.airavata.mft.core.ResourceMetadata;
 import org.apache.airavata.mft.core.api.Connector;
 import org.apache.airavata.mft.core.api.MetadataCollector;
@@ -10,43 +14,63 @@ import org.apache.airavata.mft.transport.scp.SCPMetadataCollector;
 import org.apache.airavata.mft.transport.scp.SCPReceiver;
 import org.apache.airavata.mft.transport.scp.SCPSender;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Semaphore;
 
 public class MFTAgent {
 
-    private List<TransferRequest> requests = new ArrayList<>();
     private TransportMediator mediator = new TransportMediator();
+    private String agentId = "agent0";
+    private Semaphore mainHold = new Semaphore(0);
 
-    public void acceptRequests() {
-        for (TransferRequest request : requests) {
-
-            try {
-                Connector inConnector = resolveConnector(request.getSourceType(), "IN");
-                inConnector.init(request.getSourceId(), request.getSourceToken());
-                Connector outConnector = resolveConnector(request.getDestinationType(), "OUT");
-                outConnector.init(request.getDestinationId(), request.getDestinationToken());
-
-                MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
-                ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
-                System.out.println("File size " + metadata.getResourceSize());
-                String transferId = mediator.transfer(inConnector, outConnector, metadata);
-                System.out.println("Submitted transfer " + transferId);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-        }
+    private void acceptRequests() {
+        Consul client = Consul.builder().build();
+        KeyValueClient kvClient = client.keyValueClient();
+
+        KVCache messageCache = KVCache.newCache(kvClient, agentId + "/messages");
+        messageCache.addListener(newValues -> {
+            // Cache notifies all paths with "foo" the root path
+            // If you want to watch only "foo" value, you must filter other paths
+
+            newValues.values().forEach(value -> {
+
+                // Values are encoded in key/value store, decode it if needed
+                Optional<String> decodedValue = value.getValueAsString();
+                decodedValue.ifPresent(v -> {
+                    System.out.println(String.format("Value is: %s", v));
+                    ObjectMapper mapper = new ObjectMapper();
+                    try {
+                        TransferRequest request = mapper.readValue(v, TransferRequest.class);
+                        System.out.println("Received request " + request.getTransferId());
+
+                        Connector inConnector = resolveConnector(request.getSourceType(), "IN");
+                        inConnector.init(request.getSourceId(), request.getSourceToken());
+                        Connector outConnector = resolveConnector(request.getDestinationType(), "OUT");
+                        outConnector.init(request.getDestinationId(), request.getDestinationToken());
+
+                        MetadataCollector metadataCollector = resolveMetadataCollector(request.getSourceType());
+                        ResourceMetadata metadata = metadataCollector.getGetResourceMetadata(request.getSourceId(), request.getSourceToken());
+                        System.out.println("File size " + metadata.getResourceSize());
+                        String transferId = mediator.transfer(inConnector, outConnector, metadata);
+                        System.out.println("Submitted transfer " + transferId);
+
+                        System.out.println("Deleting key " + value.getKey());
+                        kvClient.deleteKey(value.getKey()); // Due to bug in consul https://github.com/hashicorp/consul/issues/571
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }); //prints "bar"
+
+            });
+        });
+        messageCache.start();
     }
-    public static void main(String args[]) {
-        TransferRequest request = new TransferRequest();
-        request.setSourceId("1");
-        request.setSourceType("SCP");
-        request.setDestinationId("2");
-        request.setDestinationType("SCP");
 
+
+    public static void main(String args[]) throws InterruptedException {
         MFTAgent agent = new MFTAgent();
-        agent.requests.add(request);
         agent.acceptRequests();
+        agent.mainHold.acquire();
     }
 
     // TODO load from reflection to avoid dependencies
diff --git a/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java b/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
index 42c4b40..008a55a 100644
--- a/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
+++ b/agent/src/main/java/org/apache/airavata/mft/agent/TransferRequest.java
@@ -1,12 +1,17 @@
 package org.apache.airavata.mft.agent;
 
+import java.util.List;
+
 public class TransferRequest {
+
+    private String transferId;
     private String sourceId;
     private String sourceType;
     private String sourceToken;
     private String destinationId;
     private String destinationType;
     private String destinationToken;
+    private List<String> agentList;
 
     public String getSourceId() {
         return sourceId;
@@ -55,4 +60,20 @@ public class TransferRequest {
     public void setDestinationToken(String destinationToken) {
         this.destinationToken = destinationToken;
     }
+
+    public List<String> getAgentList() {
+        return agentList;
+    }
+
+    public void setAgentList(List<String> agentList) {
+        this.agentList = agentList;
+    }
+
+    public String getTransferId() {
+        return transferId;
+    }
+
+    public void setTransferId(String transferId) {
+        this.transferId = transferId;
+    }
 }
diff --git a/pom.xml b/pom.xml
index 821b297..965c829 100755
--- a/pom.xml
+++ b/pom.xml
@@ -107,6 +107,7 @@
         <protobuf.maven.plugin>0.5.1</protobuf.maven.plugin>
         <os.maven.plugin>1.5.0.Final</os.maven.plugin>
         <javax.annotation>1.3.2</javax.annotation>
+        <consul.client>1.3.8</consul.client>
     </properties>
 
 </project>
\ No newline at end of file
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
index 497159d..9d5af13 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPReceiver.java
@@ -115,7 +115,7 @@ public class SCPReceiver implements Connector {
             }
 
             if (checkAck(in) != 0) {
-                System.exit(0);
+                throw new IOException("Error code found in ack " + (checkAck(in)));
             }
 
             // send '\0'
diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
index 0456801..a767594 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPSender.java
@@ -57,7 +57,7 @@ public class SCPSender implements Connector {
         channel.connect();
 
         if (checkAck(in) != 0) {
-            System.exit(0);
+            throw new IOException("Error code found in ack " + (checkAck(in)));
         }
 
         if (ptimestamp) {
@@ -68,7 +68,7 @@ public class SCPSender implements Connector {
             out.write(command.getBytes());
             out.flush();
             if (checkAck(in) != 0) {
-                System.exit(0);
+                throw new IOException("Error code found in ack " + (checkAck(in)));
             }
         }
 
@@ -85,7 +85,7 @@ public class SCPSender implements Connector {
         out.flush();
 
         if (checkAck(in) != 0) {
-            System.exit(0);
+            throw new IOException("Error code found in ack " + (checkAck(in)));
         }
 
         // send a content of lfile
@@ -111,7 +111,7 @@ public class SCPSender implements Connector {
         out.flush();
 
         if (checkAck(in) != 0) {
-            System.exit(0);
+            throw new IOException("Error code found in ack " + (checkAck(in)));
         }
         out.close();