You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/09/24 16:30:27 UTC
[airavata] branch param-sweep updated: Optimizing input placement
logic for param sweep
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/param-sweep by this push:
new e28ff88 Optimizing input placement logic for param sweep
e28ff88 is described below
commit e28ff882acb0ecc0a99a540752ec1e1f825da131
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Sep 24 12:30:13 2020 -0400
Optimizing input placement logic for param sweep
---
.../helix/impl/task/staging/SweepingInputDataStagingTask.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
index 6f97540..474a6f2 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
@@ -38,6 +38,7 @@ import java.io.File;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -113,6 +114,8 @@ public class SweepingInputDataStagingTask extends DataStagingTask {
adaptor.executeCommand("unzip " + sourceFileName, tempZipDir);
String tempDataPath = Paths.get(tempZipDir, sourceFileName.substring(0, sourceFileName.length() - ".zip".length())).toString();
+ List<String> cpCmds = new ArrayList<>();
+
for (int i : rangeInts) {
String sweepSourceDir = Paths.get(tempDataPath, i +"").toString();
List<String> sweepFiles = adaptor.listDirectory(sweepSourceDir);
@@ -124,10 +127,14 @@ public class SweepingInputDataStagingTask extends DataStagingTask {
String destPath = Paths.get(workingDir, i + "", destFileName).toString();
logger.info("Transferring zipped sweeping input file {} to destination path {} locally", localSourceFile, destPath);
- adaptor.executeCommand("cp " + localSourceFile + " " + destPath, sweepSourceDir);
+ cpCmds.add("cd " + sweepSourceDir + "; cp " + localSourceFile + " " + destPath);
}
}
+ String copyCommands = String.join("; ", cpCmds);
+ logger.info("Running input placement commands : {}", copyCommands);
+ adaptor.executeCommand(copyCommands, null);
+
} else {
// TODO: Optimize here to copy locally
for (int i : rangeInts) {