You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/03/09 00:39:40 UTC

[airavata] branch mft-integration updated: Supporting all data staging and archival to go with MFT

This is an automated email from the ASF dual-hosted git repository.

dimuthuupe pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata.git


The following commit(s) were added to refs/heads/mft-integration by this push:
     new 4c522a4  Supporting all data staging and archival to go with MFT
4c522a4 is described below

commit 4c522a4db8d730ee8b3ed8ef2efd6145e13b2227
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Sun Mar 8 20:39:23 2020 -0400

    Supporting all data staging and archival to go with MFT
---
 .../helix/impl/task/staging/ArchiveTask.java       | 10 ++-
 .../helix/impl/task/staging/DataStagingTask.java   | 86 ++++++++++++++++++++++
 .../impl/task/staging/InputDataStagingTask.java    |  9 ++-
 .../impl/task/staging/OutputDataStagingTask.java   | 55 +++-----------
 .../apache/airavata/monitor/AbstractMonitor.java   |  1 +
 5 files changed, 114 insertions(+), 47 deletions(-)

diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
index 2448b29..f97254f 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
@@ -98,7 +98,13 @@ public class ArchiveTask extends DataStagingTask {
                 long maxArchiveSize = Long.parseLong(ServerSettings.getSetting("max.archive.size", MAX_ARCHIVE_SIZE + ""));
 
                 if (fileMetadata.getSize() < maxArchiveSize) {
-                    boolean fileTransferred = transferFileToStorage(tarCreationAbsPath, destFilePath, archiveFileName, adaptor, storageResourceAdaptor);
+
+                    boolean fileTransferred = false;
+                    if (ServerSettings.isAgentTransferEnabled()) {
+                        fileTransferred = transferFileToStorageThroughMFT(tarCreationAbsPath, destFilePath);
+                    } else {
+                        fileTransferred = transferFileToStorage(tarCreationAbsPath, destFilePath, archiveFileName, adaptor, storageResourceAdaptor);
+                    }
                     if (!fileTransferred) {
                         logger.error("Failed to transfer created archive file " + tarCreationAbsPath);
                         throw new TaskOnFailException("Failed to transfer created archive file " + tarCreationAbsPath, false, null);
@@ -106,7 +112,7 @@ public class ArchiveTask extends DataStagingTask {
 
                     String destParent = destFilePath.substring(0, destFilePath.lastIndexOf("/"));
                     final String storageArchiveDir = "ARCHIVE";
-                    String unArchiveTarCommand = "mkdir " + storageArchiveDir + " && tar -xvf " + archiveFileName + " -C "
+                    String unArchiveTarCommand = "rm -rf " + storageArchiveDir + " && mkdir " + storageArchiveDir + " && tar -xvf " + archiveFileName + " -C "
                             + storageArchiveDir + " && rm " + archiveFileName + " && chmod 755 -f -R " + storageArchiveDir + "/*";
                     logger.info("Running Un archiving command on storage resource " + unArchiveTarCommand);
 
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
index 9258dd0..504f850 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
@@ -19,16 +19,20 @@
  */
 package org.apache.airavata.helix.impl.task.staging;
 
+import io.grpc.StatusRuntimeException;
 import org.apache.airavata.agents.api.AgentAdaptor;
 import org.apache.airavata.agents.api.AgentException;
 import org.apache.airavata.agents.api.FileMetadata;
 import org.apache.airavata.agents.api.StorageResourceAdaptor;
 import org.apache.airavata.agents.streaming.TransferResult;
 import org.apache.airavata.agents.streaming.VirtualStreamProducer;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
 import org.apache.airavata.common.utils.ServerSettings;
 import org.apache.airavata.helix.impl.task.AiravataTask;
 import org.apache.airavata.helix.impl.task.TaskOnFailException;
 import org.apache.airavata.helix.task.api.support.AdaptorSupport;
+import org.apache.airavata.mft.api.client.MFTApiClient;
+import org.apache.airavata.mft.api.service.*;
 import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
 import org.apache.airavata.model.task.DataStagingTaskModel;
 import org.apache.commons.io.FileUtils;
@@ -312,6 +316,88 @@ public abstract class DataStagingTask extends AiravataTask {
 
     }
 
+    protected boolean transferFileToComputeResourceThroughMFT(String sourcePath, String destPath) throws ApplicationSettingsException,
+                                                                                            TaskOnFailException {
+
+        destPath = destPath + new File(sourcePath).getName();
+        String sourceId = "STORAGE:" + sourcePath + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
+        String sourceToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
+
+        String destId = "CLUSTER:" + destPath + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
+        String destToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
+
+        return transferThroughMFT(sourceId, sourceToken, destId, destToken);
+    }
+
+    protected boolean transferFileToStorageThroughMFT(String sourcePath, String destPath) throws TaskOnFailException,
+                                                                                            ApplicationSettingsException {
+
+        String sourceId = "CLUSTER:" + sourcePath + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
+        String sourceToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
+
+        String destId = "STORAGE:" + destPath + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
+        String destToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
+
+        return transferThroughMFT(sourceId, sourceToken, destId, destToken);
+    }
+
+    private boolean transferThroughMFT(String sourceId, String sourceToken, String destId, String destToken) throws ApplicationSettingsException, TaskOnFailException {
+        MFTApiServiceGrpc.MFTApiServiceBlockingStub mftClient = MFTApiClient.buildClient(
+                ServerSettings.getSetting("mft.server.host"),
+                Integer.parseInt(ServerSettings.getSetting("mft.server.port")));
+
+        ResourceAvailabilityResponse resourceAvailability = mftClient.getResourceAvailability(ResourceAvailabilityRequest.newBuilder()
+                .setResourceId(sourceId)
+                .setResourceToken(sourceToken)
+                .setResourceType("SCP")
+                .setResourceBackend("AIRAVATA")
+                .setResourceCredentialBackend("AIRAVATA").build());
+
+        if (resourceAvailability.getAvailable()) {
+            TransferApiRequest request = TransferApiRequest.newBuilder()
+                    .setSourceId(sourceId)
+                    .setSourceToken(sourceToken)
+                    .setSourceType("SCP")
+                    .setDestinationId(destId)
+                    .setDestinationToken(destToken)
+                    .setDestinationType("SCP")
+                    .setSourceResourceBackend("AIRAVATA")
+                    .setSourceCredentialBackend("AIRAVATA")
+                    .setDestResourceBackend("AIRAVATA")
+                    .setDestCredentialBackend("AIRAVATA")
+                    .setAffinityTransfer(false).build();
+
+            TransferApiResponse response = mftClient.submitTransfer(request);
+            logger.info("Submitted file transfer with id " + response.getTransferId());
+
+            while (true) {
+                try {
+                    TransferStateApiResponse transferState = mftClient.getTransferState(
+                            TransferStateApiRequest.newBuilder().setTransferId(response.getTransferId()).build());
+                    logger.info("Transfer status of " + response.getTransferId() + " is " + transferState.getState());
+                    if ("COMPLETED".equals(transferState.getState())) {
+                        return true;
+                    } else if ("FAILED".equals(transferState.getState())) {
+                        logger.error("Transfer failed due to " + transferState.getDescription());
+                        throw new TaskOnFailException("Transfer " + response.getTransferId() + " failed", false, null);
+                    }
+                } catch (StatusRuntimeException e) {
+                    logger.error("Error in monitoring transfer {}", response.getTransferId(), e);
+                    throw new TaskOnFailException("Error in monitoring trnasfer " + response.getTransferId(), false, e);
+                }
+                logger.info("Waiting for transfer {} status", response.getTransferId());
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    // Ignore
+                }
+            }
+        } else {
+            logger.warn("Resource " + sourceId + " is not available. So ignoring");
+            return false;
+        }
+    }
+
     protected boolean transferFileToStorage(String sourcePath, String destPath, String fileName, AgentAdaptor adaptor,
                               StorageResourceAdaptor storageResourceAdaptor) throws TaskOnFailException {
 
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
index e7e9010..1b15c7a 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
@@ -21,6 +21,7 @@ package org.apache.airavata.helix.impl.task.staging;
 
 import org.apache.airavata.agents.api.AgentAdaptor;
 import org.apache.airavata.agents.api.StorageResourceAdaptor;
+import org.apache.airavata.common.utils.ServerSettings;
 import org.apache.airavata.helix.impl.task.TaskContext;
 import org.apache.airavata.helix.impl.task.TaskOnFailException;
 import org.apache.airavata.helix.task.api.TaskHelper;
@@ -87,7 +88,13 @@ public class InputDataStagingTask extends DataStagingTask {
                     URI destinationURI = new URI(dataStagingTaskModel.getDestination());
 
                     logger.info("Source file " + sourceURI.getPath() + ", destination uri " + destinationURI.getPath() + " for task " + getTaskId());
-                    transferFileToComputeResource(sourceURI.getPath(), destinationURI.getPath(), adaptor, storageResourceAdaptor);
+                    if (ServerSettings.isAgentTransferEnabled()) {
+                        logger.info("Transferring through MFT");
+                        transferFileToComputeResourceThroughMFT(sourceURI.getPath(), destinationURI.getPath());
+                    } else {
+                        transferFileToComputeResource(sourceURI.getPath(), destinationURI.getPath(), adaptor, storageResourceAdaptor);
+                    }
+
                 }
 
             } catch (URISyntaxException e) {
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
index 902ebab..612af2f 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
@@ -153,7 +153,14 @@ public class OutputDataStagingTask extends DataStagingTask {
                     //Wildcard support is only enabled for output data staging
                     assert processOutput != null;
                     logger.info("Transferring file " + sourceFileName);
-                    boolean transferred = transferFileToStorage(newSourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+                    boolean transferred;
+
+                    if (ServerSettings.isAgentTransferEnabled()) {
+                        transferred = transferFileToStorageThroughMFT(newSourceURI.getPath(), destinationURI.getPath());
+                    } else {
+                        transferred = transferFileToStorage(newSourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+                    }
+
                     if (transferred) {
                         destinationURIs.add(destinationURI);
                     } else {
@@ -181,51 +188,11 @@ public class OutputDataStagingTask extends DataStagingTask {
                 assert processOutput != null;
                 boolean transferred = false;
                 if (ServerSettings.isAgentTransferEnabled()) {
-                    String sourceId = "CLUSTER:" + sourceURI.getPath() + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
-                    String sourceToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
-
-                    String destId = "STORAGE:" + destinationURI.getPath() + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
-                    String destToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
-
-                    TransferApiRequest request = TransferApiRequest.newBuilder()
-                            .setSourceId(sourceId)
-                            .setSourceToken(sourceToken)
-                            .setSourceType("SCP")
-                            .setDestinationId(destId)
-                            .setDestinationToken(destToken)
-                            .setDestinationType("SCP")
-                            .setSourceResourceBackend("AIRAVATA")
-                            .setSourceCredentialBackend("AIRAVATA")
-                            .setDestResourceBackend("AIRAVATA")
-                            .setDestCredentialBackend("AIRAVATA")
-                            .setAffinityTransfer(false).build();
-
-                    MFTApiServiceGrpc.MFTApiServiceBlockingStub mftClient = MFTApiClient.buildClient(
-                            ServerSettings.getSetting("mft.server.host"),
-                            Integer.parseInt(ServerSettings.getSetting("mft.server.port")));
-
-                    TransferApiResponse response = mftClient.submitTransfer(request);
-                    logger.info("Submitted file transfer with id " + response.getTransferId());
-
-                    while (true) {
-                        try {
-                            TransferStateApiResponse transferState = mftClient.getTransferState(
-                                    TransferStateApiRequest.newBuilder().setTransferId(response.getTransferId()).build());
-                            logger.info("Transfer status of " + response.getTransferId() + " is " + transferState.getState());
-                            if ("COMPLETED".equals(transferState.getState())) {
-                                transferred = true;
-                                break;
-                            } else if ("FAILED".equals(transferState.getState())) {
-                                throw new TaskOnFailException("Transfer " + response.getTransferId() + " failed", false, null);
-                            }
-                        } catch (StatusRuntimeException e) {
-                            logger.info("No status for transfer " + response.getTransferId());
-                        }
-                        Thread.sleep(1000);
-                    }
+                    transferred = transferFileToStorageThroughMFT(sourceURI.getPath(), destinationURI.getPath());
 
                 } else {
-                    transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+                    transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(), sourceFileName,
+                                adaptor, storageResourceAdaptor);
                 }
 
                 if (transferred) {
diff --git a/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java b/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
index 69bf883..e4af5ec 100644
--- a/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
+++ b/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
@@ -96,6 +96,7 @@ public class AbstractMonitor {
         try {
             if (validateJobStatus(jobStatusResult)) {
                 messageProducer.submitMessageToQueue(jobStatusResult);
+                log.info("Submitted " + jobStatusResult.getJobName() + " with status " + jobStatusResult.getState().name() + " to broker");
             } else {
                 throw new MonitoringException("Failed to validate job status for job id " + jobStatusResult.getJobId());
             }