You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/09/24 18:32:19 UTC
[airavata] branch param-sweep updated: Optimizing the directory
scanning logic in SweepingInputDataStagingTask
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch param-sweep
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/param-sweep by this push:
new 14a746f Optimizing the directory scanning logic in SweepingInputDataStagingTask
14a746f is described below
commit 14a746fbac92bc3723034687a2c5e965cd9d973f
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Sep 24 14:32:05 2020 -0400
Optimizing the directory scanning logic in SweepingInputDataStagingTask
---
.../task/staging/SweepingInputDataStagingTask.java | 37 +++++++++++++---------
1 file changed, 22 insertions(+), 15 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
index c78c29d..2a12055 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/staging/SweepingInputDataStagingTask.java
@@ -114,26 +114,33 @@ public class SweepingInputDataStagingTask extends DataStagingTask {
adaptor.executeCommand("unzip " + sourceFileName, tempZipDir);
String tempDataPath = Paths.get(tempZipDir, sourceFileName.substring(0, sourceFileName.length() - ".zip".length())).toString();
- List<String> cpCmds = new ArrayList<>();
- for (int i : rangeInts) {
- String sweepSourceDir = Paths.get(tempDataPath, i +"").toString();
- List<String> sweepFiles = adaptor.listDirectory(sweepSourceDir);
- for (String sweepFile: sweepFiles) {
- String localSourceFile = Paths.get(sweepSourceDir, sweepFile).toString();
+ if (rangeInts.size() > 0) {
+ List<String> cpCmds = new ArrayList<>();
+
+ // Looking for the fist directory and assuming other directories have the same format
+ String firstInputDir = Paths.get(tempDataPath, rangeInts.get(0) + "").toString();
+ List<String> sweepFiles = adaptor.listDirectory(firstInputDir);
+ logger.info("Found files in the first input directory {}, {}", firstInputDir, String.join(", ", sweepFiles));
+
+ for (int i : rangeInts) {
+ String sweepSourceDir = Paths.get(tempDataPath, i + "").toString();
+ for (String sweepFile : sweepFiles) {
+ String localSourceFile = Paths.get(sweepSourceDir, sweepFile).toString();
- String overrideFileName = dataStagingTaskModel.getProcessInput().getOverrideFilename();
- String destFileName = (overrideFileName != null && !"".equals(overrideFileName)) ? overrideFileName : sweepFile;
- String destPath = Paths.get(workingDir, i + "", destFileName).toString();
+ String overrideFileName = dataStagingTaskModel.getProcessInput().getOverrideFilename();
+ String destFileName = (overrideFileName != null && !"".equals(overrideFileName)) ? overrideFileName : sweepFile;
+ String destPath = Paths.get(workingDir, i + "", destFileName).toString();
- logger.info("Preparing the transfer input file {} to destination path {} locally", localSourceFile, destPath);
- cpCmds.add("cd " + sweepSourceDir + "; cp " + localSourceFile + " " + destPath);
+ logger.info("Preparing the transfer input file {} to destination path {} locally", localSourceFile, destPath);
+ cpCmds.add("cd " + sweepSourceDir + "; cp " + localSourceFile + " " + destPath);
+ }
}
- }
- String copyCommands = String.join("; ", cpCmds);
- logger.info("Running input placement commands : {}", copyCommands);
- adaptor.executeCommand(copyCommands, null);
+ String copyCommands = String.join("; ", cpCmds);
+ logger.info("Running input placement commands : {}", copyCommands);
+ adaptor.executeCommand(copyCommands, null);
+ }
} else {
// TODO: Optimize here to copy locally