You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/03/09 00:39:40 UTC
[airavata] branch mft-integration updated: Supporting all data
staging and archival to go with MFT
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/mft-integration by this push:
new 4c522a4 Supporting all data staging and archival to go with MFT
4c522a4 is described below
commit 4c522a4db8d730ee8b3ed8ef2efd6145e13b2227
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Sun Mar 8 20:39:23 2020 -0400
Supporting all data staging and archival to go with MFT
---
.../helix/impl/task/staging/ArchiveTask.java | 10 ++-
.../helix/impl/task/staging/DataStagingTask.java | 86 ++++++++++++++++++++++
.../impl/task/staging/InputDataStagingTask.java | 9 ++-
.../impl/task/staging/OutputDataStagingTask.java | 55 +++-----------
.../apache/airavata/monitor/AbstractMonitor.java | 1 +
5 files changed, 114 insertions(+), 47 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
index 2448b29..f97254f 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/ArchiveTask.java
@@ -98,7 +98,13 @@ public class ArchiveTask extends DataStagingTask {
long maxArchiveSize = Long.parseLong(ServerSettings.getSetting("max.archive.size", MAX_ARCHIVE_SIZE + ""));
if (fileMetadata.getSize() < maxArchiveSize) {
- boolean fileTransferred = transferFileToStorage(tarCreationAbsPath, destFilePath, archiveFileName, adaptor, storageResourceAdaptor);
+
+ boolean fileTransferred = false;
+ if (ServerSettings.isAgentTransferEnabled()) {
+ fileTransferred = transferFileToStorageThroughMFT(tarCreationAbsPath, destFilePath);
+ } else {
+ fileTransferred = transferFileToStorage(tarCreationAbsPath, destFilePath, archiveFileName, adaptor, storageResourceAdaptor);
+ }
if (!fileTransferred) {
logger.error("Failed to transfer created archive file " + tarCreationAbsPath);
throw new TaskOnFailException("Failed to transfer created archive file " + tarCreationAbsPath, false, null);
@@ -106,7 +112,7 @@ public class ArchiveTask extends DataStagingTask {
String destParent = destFilePath.substring(0, destFilePath.lastIndexOf("/"));
final String storageArchiveDir = "ARCHIVE";
- String unArchiveTarCommand = "mkdir " + storageArchiveDir + " && tar -xvf " + archiveFileName + " -C "
+ String unArchiveTarCommand = "rm -rf " + storageArchiveDir + " && mkdir " + storageArchiveDir + " && tar -xvf " + archiveFileName + " -C "
+ storageArchiveDir + " && rm " + archiveFileName + " && chmod 755 -f -R " + storageArchiveDir + "/*";
logger.info("Running Un archiving command on storage resource " + unArchiveTarCommand);
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
index 9258dd0..504f850 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
@@ -19,16 +19,20 @@
*/
package org.apache.airavata.helix.impl.task.staging;
+import io.grpc.StatusRuntimeException;
import org.apache.airavata.agents.api.AgentAdaptor;
import org.apache.airavata.agents.api.AgentException;
import org.apache.airavata.agents.api.FileMetadata;
import org.apache.airavata.agents.api.StorageResourceAdaptor;
import org.apache.airavata.agents.streaming.TransferResult;
import org.apache.airavata.agents.streaming.VirtualStreamProducer;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.helix.impl.task.AiravataTask;
import org.apache.airavata.helix.impl.task.TaskOnFailException;
import org.apache.airavata.helix.task.api.support.AdaptorSupport;
+import org.apache.airavata.mft.api.client.MFTApiClient;
+import org.apache.airavata.mft.api.service.*;
import org.apache.airavata.model.appcatalog.storageresource.StorageResourceDescription;
import org.apache.airavata.model.task.DataStagingTaskModel;
import org.apache.commons.io.FileUtils;
@@ -312,6 +316,88 @@ public abstract class DataStagingTask extends AiravataTask {
}
+ protected boolean transferFileToComputeResourceThroughMFT(String sourcePath, String destPath) throws ApplicationSettingsException,
+ TaskOnFailException {
+
+ destPath = destPath + new File(sourcePath).getName();
+ String sourceId = "STORAGE:" + sourcePath + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
+ String sourceToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
+
+ String destId = "CLUSTER:" + destPath + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
+ String destToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
+
+ return transferThroughMFT(sourceId, sourceToken, destId, destToken);
+ }
+
+ protected boolean transferFileToStorageThroughMFT(String sourcePath, String destPath) throws TaskOnFailException,
+ ApplicationSettingsException {
+
+ String sourceId = "CLUSTER:" + sourcePath + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
+ String sourceToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
+
+ String destId = "STORAGE:" + destPath + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
+ String destToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
+
+ return transferThroughMFT(sourceId, sourceToken, destId, destToken);
+ }
+
+ private boolean transferThroughMFT(String sourceId, String sourceToken, String destId, String destToken) throws ApplicationSettingsException, TaskOnFailException {
+ MFTApiServiceGrpc.MFTApiServiceBlockingStub mftClient = MFTApiClient.buildClient(
+ ServerSettings.getSetting("mft.server.host"),
+ Integer.parseInt(ServerSettings.getSetting("mft.server.port")));
+
+ ResourceAvailabilityResponse resourceAvailability = mftClient.getResourceAvailability(ResourceAvailabilityRequest.newBuilder()
+ .setResourceId(sourceId)
+ .setResourceToken(sourceToken)
+ .setResourceType("SCP")
+ .setResourceBackend("AIRAVATA")
+ .setResourceCredentialBackend("AIRAVATA").build());
+
+ if (resourceAvailability.getAvailable()) {
+ TransferApiRequest request = TransferApiRequest.newBuilder()
+ .setSourceId(sourceId)
+ .setSourceToken(sourceToken)
+ .setSourceType("SCP")
+ .setDestinationId(destId)
+ .setDestinationToken(destToken)
+ .setDestinationType("SCP")
+ .setSourceResourceBackend("AIRAVATA")
+ .setSourceCredentialBackend("AIRAVATA")
+ .setDestResourceBackend("AIRAVATA")
+ .setDestCredentialBackend("AIRAVATA")
+ .setAffinityTransfer(false).build();
+
+ TransferApiResponse response = mftClient.submitTransfer(request);
+ logger.info("Submitted file transfer with id " + response.getTransferId());
+
+ while (true) {
+ try {
+ TransferStateApiResponse transferState = mftClient.getTransferState(
+ TransferStateApiRequest.newBuilder().setTransferId(response.getTransferId()).build());
+ logger.info("Transfer status of " + response.getTransferId() + " is " + transferState.getState());
+ if ("COMPLETED".equals(transferState.getState())) {
+ return true;
+ } else if ("FAILED".equals(transferState.getState())) {
+ logger.error("Transfer failed due to " + transferState.getDescription());
+ throw new TaskOnFailException("Transfer " + response.getTransferId() + " failed", false, null);
+ }
+ } catch (StatusRuntimeException e) {
+ logger.error("Error in monitoring transfer {}", response.getTransferId(), e);
+ throw new TaskOnFailException("Error in monitoring trnasfer " + response.getTransferId(), false, e);
+ }
+ logger.info("Waiting for transfer {} status", response.getTransferId());
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // Ignore
+ }
+ }
+ } else {
+ logger.warn("Resource " + sourceId + " is not available. So ignoring");
+ return false;
+ }
+ }
+
protected boolean transferFileToStorage(String sourcePath, String destPath, String fileName, AgentAdaptor adaptor,
StorageResourceAdaptor storageResourceAdaptor) throws TaskOnFailException {
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
index e7e9010..1b15c7a 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
@@ -21,6 +21,7 @@ package org.apache.airavata.helix.impl.task.staging;
import org.apache.airavata.agents.api.AgentAdaptor;
import org.apache.airavata.agents.api.StorageResourceAdaptor;
+import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.helix.impl.task.TaskContext;
import org.apache.airavata.helix.impl.task.TaskOnFailException;
import org.apache.airavata.helix.task.api.TaskHelper;
@@ -87,7 +88,13 @@ public class InputDataStagingTask extends DataStagingTask {
URI destinationURI = new URI(dataStagingTaskModel.getDestination());
logger.info("Source file " + sourceURI.getPath() + ", destination uri " + destinationURI.getPath() + " for task " + getTaskId());
- transferFileToComputeResource(sourceURI.getPath(), destinationURI.getPath(), adaptor, storageResourceAdaptor);
+ if (ServerSettings.isAgentTransferEnabled()) {
+ logger.info("Transferring through MFT");
+ transferFileToComputeResourceThroughMFT(sourceURI.getPath(), destinationURI.getPath());
+ } else {
+ transferFileToComputeResource(sourceURI.getPath(), destinationURI.getPath(), adaptor, storageResourceAdaptor);
+ }
+
}
} catch (URISyntaxException e) {
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
index 902ebab..612af2f 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/OutputDataStagingTask.java
@@ -153,7 +153,14 @@ public class OutputDataStagingTask extends DataStagingTask {
//Wildcard support is only enabled for output data staging
assert processOutput != null;
logger.info("Transferring file " + sourceFileName);
- boolean transferred = transferFileToStorage(newSourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+ boolean transferred;
+
+ if (ServerSettings.isAgentTransferEnabled()) {
+ transferred = transferFileToStorageThroughMFT(newSourceURI.getPath(), destinationURI.getPath());
+ } else {
+ transferred = transferFileToStorage(newSourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+ }
+
if (transferred) {
destinationURIs.add(destinationURI);
} else {
@@ -181,51 +188,11 @@ public class OutputDataStagingTask extends DataStagingTask {
assert processOutput != null;
boolean transferred = false;
if (ServerSettings.isAgentTransferEnabled()) {
- String sourceId = "CLUSTER:" + sourceURI.getPath() + ":" + getGatewayId() + ":" + getTaskContext().getComputeResourceId();
- String sourceToken = getTaskContext().getComputeResourceCredentialToken() + ":" + getTaskContext().getComputeResourceLoginUserName() + ":" + getGatewayId();
-
- String destId = "STORAGE:" + destinationURI.getPath() + ":" + getGatewayId() + ":" + getTaskContext().getStorageResourceId();
- String destToken = getTaskContext().getStorageResourceCredentialToken() + ":" + getTaskContext().getStorageResourceLoginUserName() + ":" + getGatewayId();
-
- TransferApiRequest request = TransferApiRequest.newBuilder()
- .setSourceId(sourceId)
- .setSourceToken(sourceToken)
- .setSourceType("SCP")
- .setDestinationId(destId)
- .setDestinationToken(destToken)
- .setDestinationType("SCP")
- .setSourceResourceBackend("AIRAVATA")
- .setSourceCredentialBackend("AIRAVATA")
- .setDestResourceBackend("AIRAVATA")
- .setDestCredentialBackend("AIRAVATA")
- .setAffinityTransfer(false).build();
-
- MFTApiServiceGrpc.MFTApiServiceBlockingStub mftClient = MFTApiClient.buildClient(
- ServerSettings.getSetting("mft.server.host"),
- Integer.parseInt(ServerSettings.getSetting("mft.server.port")));
-
- TransferApiResponse response = mftClient.submitTransfer(request);
- logger.info("Submitted file transfer with id " + response.getTransferId());
-
- while (true) {
- try {
- TransferStateApiResponse transferState = mftClient.getTransferState(
- TransferStateApiRequest.newBuilder().setTransferId(response.getTransferId()).build());
- logger.info("Transfer status of " + response.getTransferId() + " is " + transferState.getState());
- if ("COMPLETED".equals(transferState.getState())) {
- transferred = true;
- break;
- } else if ("FAILED".equals(transferState.getState())) {
- throw new TaskOnFailException("Transfer " + response.getTransferId() + " failed", false, null);
- }
- } catch (StatusRuntimeException e) {
- logger.info("No status for transfer " + response.getTransferId());
- }
- Thread.sleep(1000);
- }
+ transferred = transferFileToStorageThroughMFT(sourceURI.getPath(), destinationURI.getPath());
} else {
- transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(), sourceFileName, adaptor, storageResourceAdaptor);
+ transferred = transferFileToStorage(sourceURI.getPath(), destinationURI.getPath(), sourceFileName,
+ adaptor, storageResourceAdaptor);
}
if (transferred) {
diff --git a/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java b/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
index 69bf883..e4af5ec 100644
--- a/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
+++ b/modules/job-monitor/job-monitor-api/src/main/java/org/apache/airavata/monitor/AbstractMonitor.java
@@ -96,6 +96,7 @@ public class AbstractMonitor {
try {
if (validateJobStatus(jobStatusResult)) {
messageProducer.submitMessageToQueue(jobStatusResult);
+ log.info("Submitted " + jobStatusResult.getJobName() + " with status " + jobStatusResult.getState().name() + " to broker");
} else {
throw new MonitoringException("Failed to validate job status for job id " + jobStatusResult.getJobId());
}