You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2018/04/03 19:49:25 UTC
[airavata] branch develop updated: Deleting temporary files created
in Data staging tasks
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new 1a97414 Deleting temporary files created in Data staging tasks
1a97414 is described below
commit 1a97414fbf0e4cabd8af45a7219edf8ec4f18a10
Author: dimuthu <di...@gmail.com>
AuthorDate: Tue Apr 3 15:49:19 2018 -0400
Deleting temporary files created in Data staging tasks
---
.../helix/impl/task/staging/DataStagingTask.java | 43 +++++++++++++++-------
.../impl/task/staging/InputDataStagingTask.java | 34 ++++++++++-------
2 files changed, 50 insertions(+), 27 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
index ff8d6a1..04cf831 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java
@@ -143,22 +143,39 @@ public abstract class DataStagingTask extends AiravataTask {
String localSourceFilePath = getLocalDataPath(fileName);
try {
- logger.info("Downloading output file " + sourcePath + " to the local path " + localSourceFilePath);
- adaptor.copyFileFrom(sourcePath, localSourceFilePath);
- logger.info("Output file downloaded to " + localSourceFilePath);
- } catch (AgentException e) {
- throw new TaskOnFailException("Failed downloading output file " + sourcePath + " to the local path " +
- localSourceFilePath, true, e);
+ try {
+ logger.info("Downloading output file " + sourcePath + " to the local path " + localSourceFilePath);
+ adaptor.copyFileFrom(sourcePath, localSourceFilePath);
+ logger.info("Output file downloaded to " + localSourceFilePath);
+ } catch (AgentException e) {
+ throw new TaskOnFailException("Failed downloading output file " + sourcePath + " to the local path " +
+ localSourceFilePath, true, e);
+ }
+
+ // Uploading output file to the storage resource
+ try {
+ logger.info("Uploading the output file to " + destPath + " from local path " + localSourceFilePath);
+ storageResourceAdaptor.uploadFile(localSourceFilePath, destPath);
+ logger.info("Output file uploaded to " + destPath);
+ } catch (AgentException e) {
+ throw new TaskOnFailException("Failed uploading the output file to " + destPath + " from local path " +
+ localSourceFilePath, true, e);
+ }
+
+ } finally {
+ logger.info("Deleting temporary file " + localSourceFilePath);
+ deleteTempFile(localSourceFilePath);
}
+ }
- // Uploading output file to the storage resource
+ protected void deleteTempFile(String filePath) {
try {
- logger.info("Uploading the output file to " + destPath + " from local path " + localSourceFilePath);
- storageResourceAdaptor.uploadFile(localSourceFilePath, destPath);
- logger.info("Output file uploaded to " + destPath);
- } catch (AgentException e) {
- throw new TaskOnFailException("Failed uploading the output file to " + destPath + " from local path " +
- localSourceFilePath, true, e);
+ File tobeDeleted = new File(filePath);
+ if (tobeDeleted.exists()) {
+ tobeDeleted.delete();
+ }
+ } catch (Exception e) {
+ logger.warn("Failed to delete temporary file " + filePath);
}
}
}
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
index 04da216..b1c8915 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/InputDataStagingTask.java
@@ -93,23 +93,29 @@ public class InputDataStagingTask extends DataStagingTask {
String localSourceFilePath = getLocalDataPath(sourceFileName);
// Downloading input file from the storage resource
- try {
- logger.info("Downloading input file " + sourceURI.getPath() + " to the local path " + localSourceFilePath);
- storageResourceAdaptor.downloadFile(sourceURI.getPath(), localSourceFilePath);
- logger.info("Input file downloaded to " + localSourceFilePath);
- } catch (AgentException e) {
- throw new TaskOnFailException("Failed downloading input file " + sourceFileName + " to the local path " + localSourceFilePath, true, e);
- }
- // Uploading input file to the compute resource
try {
- logger.info("Uploading the input file to " + destinationURI.getPath() + " from local path " + localSourceFilePath);
- adaptor.copyFileTo(localSourceFilePath, destinationURI.getPath());
- logger.info("Output file uploaded to " + destinationURI.getPath());
- } catch (AgentException e) {
- throw new TaskOnFailException("Failed uploading the input file to " + destinationURI.getPath() + " from local path " + localSourceFilePath, true, e);
- }
+ try {
+ logger.info("Downloading input file " + sourceURI.getPath() + " to the local path " + localSourceFilePath);
+ storageResourceAdaptor.downloadFile(sourceURI.getPath(), localSourceFilePath);
+ logger.info("Input file downloaded to " + localSourceFilePath);
+ } catch (AgentException e) {
+ throw new TaskOnFailException("Failed downloading input file " + sourceFileName + " to the local path " + localSourceFilePath, true, e);
+ }
+ // Uploading input file to the compute resource
+ try {
+ logger.info("Uploading the input file to " + destinationURI.getPath() + " from local path " + localSourceFilePath);
+ adaptor.copyFileTo(localSourceFilePath, destinationURI.getPath());
+ logger.info("Input file uploaded to " + destinationURI.getPath());
+ } catch (AgentException e) {
+ throw new TaskOnFailException("Failed uploading the input file to " + destinationURI.getPath() + " from local path " + localSourceFilePath, true, e);
+ }
+
+ } finally {
+ logger.info("Deleting temporary file " + localSourceFilePath);
+ deleteTempFile(localSourceFilePath);
+ }
return onSuccess("Input data staging task " + getTaskId() + " successfully completed");
} catch (TaskOnFailException e) {
--
To stop receiving notification emails like this one, please contact
dimuthuupe@apache.org.