You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2023/06/29 00:15:56 UTC

[airavata-mft] branch master updated (e530d65 -> db231dd)

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git


    from e530d65  grpcio grpcio tools issue 104
     new e649298  Handling scp secrets with no public key
     new e0c5226  Adding local outgoing streaming connector
     new db231dd  Terminating pending transfers if an exception was captured

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../mft/agent/transport/ConnectorResolver.java     |  3 +++
 .../airavata/mft/controller/MFTController.java     | 16 ++++++++++++---
 ...r.java => LocalOutgoingStreamingConnector.java} | 24 +++++++++-------------
 .../mft/transport/scp/SCPIncomingConnector.java    |  4 ++--
 4 files changed, 28 insertions(+), 19 deletions(-)
 copy transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/{LocalIncomingStreamingConnector.java => LocalOutgoingStreamingConnector.java} (70%)


[airavata-mft] 01/03: Handling scp secrets with no public key

Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git

commit e6492987f6e14f668238a2980efb5c43189d712f
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Wed Jun 28 20:14:05 2023 -0400

    Handling scp secrets with no public key
---
 .../org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
index dea4c6e..1157214 100644
--- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
+++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPIncomingConnector.java
@@ -58,7 +58,7 @@ public final class SCPIncomingConnector implements IncomingStreamingConnector {
                 scpStorage.getHost(),
                 scpStorage.getPort(),
                 scpSecret.getPrivateKey().getBytes(),
-                scpSecret.getPublicKey().getBytes(),
+                scpSecret.getPublicKey().equals("")? null : scpSecret.getPublicKey().getBytes(),
                 scpSecret.getPassphrase().equals("")? null : scpSecret.getPassphrase().getBytes());
 
         if (session == null) {
@@ -94,7 +94,7 @@ public final class SCPIncomingConnector implements IncomingStreamingConnector {
 
         while (true) {
             int c = checkAck(in);
-            if (c != 'C') {
+            if (c != 'C' && c != 'c' ) {
                 break;
             }
 


[airavata-mft] 02/03: Adding local outgoing streaming connector

Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git

commit e0c5226b58a94a57f79d1bee3f4d216489301a32
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Wed Jun 28 20:14:56 2023 -0400

    Adding local outgoing streaming connector
---
 .../mft/agent/transport/ConnectorResolver.java     |  3 ++
 .../local/LocalOutgoingStreamingConnector.java     | 53 ++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/agent/service/src/main/java/org/apache/airavata/mft/agent/transport/ConnectorResolver.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/transport/ConnectorResolver.java
index 0dee097..ba2186f 100644
--- a/agent/service/src/main/java/org/apache/airavata/mft/agent/transport/ConnectorResolver.java
+++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/transport/ConnectorResolver.java
@@ -75,6 +75,9 @@ public final class ConnectorResolver {
             case "GCS":
                 className = "org.apache.airavata.mft.transport.gcp.GCSOutgoingStreamingConnector";
                 break;
+            case "LOCAL":
+                className = "org.apache.airavata.mft.transport.local.LocalOutgoingStreamingConnector";
+                break;
 
         }
 
diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalOutgoingStreamingConnector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalOutgoingStreamingConnector.java
new file mode 100644
index 0000000..87d0563
--- /dev/null
+++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalOutgoingStreamingConnector.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.airavata.mft.transport.local;
+
+import org.apache.airavata.mft.core.api.ConnectorConfig;
+import org.apache.airavata.mft.core.api.OutgoingStreamingConnector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.OutputStream;
+
+public class LocalOutgoingStreamingConnector implements OutgoingStreamingConnector {
+
+    private String resourcePath;
+
+    private static final Logger logger = LoggerFactory.getLogger(LocalOutgoingStreamingConnector.class);
+
+    @Override
+    public void init(ConnectorConfig connectorConfig) throws Exception {
+        this.resourcePath = connectorConfig.getResourcePath();
+    }
+
+    @Override
+    public void complete() throws Exception {
+        logger.info("File {} successfully written", this.resourcePath);
+    }
+
+    @Override
+    public void failed() throws Exception {
+        logger.error("Failed while writing file {}", this.resourcePath);
+    }
+
+    @Override
+    public OutputStream fetchOutputStream() throws Exception {
+        return new FileOutputStream(this.resourcePath);
+    }
+}


[airavata-mft] 03/03: Terminating pending transfers if an exception was captured

Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-mft.git

commit db231ddc775e67dbab55d35ebaab539a5a79d3f4
Author: DImuthuUpe <di...@gmail.com>
AuthorDate: Wed Jun 28 20:15:43 2023 -0400

    Terminating pending transfers if an exception was captured
---
 .../apache/airavata/mft/controller/MFTController.java    | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
index eff5d04..110f6d2 100644
--- a/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
+++ b/controller/src/main/java/org/apache/airavata/mft/controller/MFTController.java
@@ -304,17 +304,27 @@ public class MFTController implements CommandLineRunner {
 
             if (value.getValueAsString().isPresent()) {
                 logger.debug("Pending " + value.getKey() + " : " + value.getValueAsString().get());
+                String transferId = value.getKey().substring(value.getKey().lastIndexOf("/") + 1);
                 try {
                     TransferApiRequest.Builder builder = TransferApiRequest.newBuilder();
                     JsonFormat.parser().merge(value.getValueAsString().get(), builder);
                     TransferApiRequest transferRequest = builder.build();
-
-                    String transferId = value.getKey().substring(value.getKey().lastIndexOf("/") + 1);
                     AgentTransferRequest.Builder agentTransferRequest = requestBuilder.prepareAgentTransferRequest(transferRequest);
                     pathOptimizer.handleTransferRequest(transferId, transferRequest, agentTransferRequest, value.getKey());
 
                 } catch (Exception e) {
-                    logger.error("Failed to process pending transfer in key {}", value.getKey(), e);
+                    logger.error("Failed to process pending transfer in key {}. Deleting from queue", value.getKey(), e);
+                    try {
+                        mftConsulClient.saveTransferState(transferId, null, new TransferState()
+                                .setUpdateTimeMils(System.currentTimeMillis())
+                                .setState("FAILED").setPercentage(0)
+                                .setPublisher("controller")
+                                .setDescription("Failed to process pending transfer "));
+                        mftConsulClient.getKvClient().deleteKey(value.getKey());
+                    } catch (Exception ex) {
+                        logger.warn("Failed to update state of transfer {} to FAILED", transferId);
+                        // Ignore
+                    }
                 }
             }
         });