You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2015/07/09 19:01:27 UTC
airavata git commit: To fix AIRAVATA-1759. Wild char search for
output file.
Repository: airavata
Updated Branches:
refs/heads/airavata-0.15-release-branch 5621a0825 -> 38893ff21
To fix AIRAVATA-1759. Wild char search for output file.
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/38893ff2
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/38893ff2
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/38893ff2
Branch: refs/heads/airavata-0.15-release-branch
Commit: 38893ff2169e405feff6bd9eebbb532096539b28
Parents: 5621a08
Author: raminder <ra...@apache.org>
Authored: Thu Jul 9 13:01:09 2015 -0400
Committer: raminder <ra...@apache.org>
Committed: Thu Jul 9 13:01:09 2015 -0400
----------------------------------------------------------------------
.../airavata/gfac/ssh/util/HandleOutputs.java | 84 +++++++++++++++++---
.../apache/airavata/gsi/ssh/api/Cluster.java | 7 ++
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 47 ++++++++++-
.../apache/airavata/gsi/ssh/util/SSHUtils.java | 9 ++-
4 files changed, 131 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/38893ff2/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/HandleOutputs.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/HandleOutputs.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/HandleOutputs.java
index 0e055b3..3ee84f9 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/HandleOutputs.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/HandleOutputs.java
@@ -4,6 +4,9 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.ListIterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.gfac.Constants;
@@ -35,30 +38,58 @@ public class HandleOutputs {
(new File(outputDataDir)).mkdirs();
List<OutputDataObjectType> outputs = jobExecutionContext.getTaskData().getApplicationOutputs();
- List<String> outputList = cluster.listDirectory(jobExecutionContext.getWorkingDir());
+ List<String> outputList = cluster.listDirectory(jobExecutionContext.getWorkingDir(),true);
boolean missingOutput = false;
for (OutputDataObjectType output : outputs) {
- // FIXME: Validation of outputs based on required and optional and search based on REGEX provided in search.
+ // FIXME: Validation of outputs based on required and optional
+ // and search based on REGEX provided in search.
if (DataType.URI == output.getType()) {
- // for failed jobs outputs are not generated. So we should not download outputs
- if (GFacUtils.isFailedJob(jobExecutionContext)){
- continue;
- }
+ // for failed jobs outputs are not generated. So we should
+ // not download outputs
+ if (GFacUtils.isFailedJob(jobExecutionContext)) {
+ continue;
+ }
String outputFile = output.getValue();
String fileName = outputFile.substring(outputFile.lastIndexOf(File.separatorChar) + 1, outputFile.length());
-
if (output.getLocation() == null && !outputList.contains(fileName) && output.isIsRequired()) {
missingOutput = true;
- } else {
+ }else {
+ // if user value has any pattern char.
+ Pattern p = Pattern.compile("[*:<>?\\|]", Pattern.CASE_INSENSITIVE);
+ Matcher m = p.matcher(outputFile);
+ // boolean b = m.matches();
+ boolean b = m.find();
+ if (b) {
+ if(fileName.isEmpty() && !output.getSearchQuery().isEmpty()){
+ fileName = output.getSearchQuery();
+ }
+ if(fileName.isEmpty()){
+ throw new GFacHandlerException("Define a output value or search query");
+ }
+ String fileregex = wildcardToRegex(fileName);
+ log.info("Regex for: "+ fileName+ " is: " + fileregex);
cluster.scpFrom(outputFile, outputDataDir);
- String localFile = outputDataDir + File.separator + fileName;
- jobExecutionContext.addOutputFile(localFile);
- output.setValue(localFile);
- outputArray.add(output);
+ ListIterator<String> li = outputList.listIterator();
+ while (li.hasNext()) {
+ String next = li.next();
+ if (Pattern.matches(fileregex, next)) {
+ String localFile = outputDataDir + File.separator + next;
+ jobExecutionContext.addOutputFile(localFile);
+ output.setValue(localFile);
+ outputArray.add(output);
+ break;
+ }
+ }
+ } else {
+ cluster.scpFrom(outputFile, outputDataDir);
+ String localFile = outputDataDir + File.separator + fileName;
+ jobExecutionContext.addOutputFile(localFile);
+ output.setValue(localFile);
+ outputArray.add(output);
+ }
}
-
} else if (DataType.STDOUT == output.getType()) {
String downloadFile = jobExecutionContext.getStandardOutput();
String fileName = downloadFile.substring(downloadFile.lastIndexOf(File.separatorChar) + 1, downloadFile.length());
@@ -100,4 +131,31 @@ public class HandleOutputs {
jobExecutionContext.getTaskData().setApplicationOutputs(outputArray);
return outputArray;
}
+ public static String wildcardToRegex(String wildcard){
+ StringBuffer s = new StringBuffer(wildcard.length());
+ s.append('^');
+ for (int i = 0, is = wildcard.length(); i < is; i++) {
+ char c = wildcard.charAt(i);
+ switch(c) {
+ case '*':
+ s.append(".*");
+ break;
+ case '?':
+ s.append(".");
+ break;
+ // escape special regexp-characters
+ case '(': case ')': case '[': case ']': case '$':
+ case '^': case '.': case '{': case '}': case '|':
+ case '\\':
+ s.append("\\");
+ s.append(c);
+ break;
+ default:
+ s.append(c);
+ break;
+ }
+ }
+ s.append('$');
+ return(s.toString());
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/38893ff2/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
index 34e3b94..f31fc69 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
@@ -138,6 +138,13 @@ public interface Cluster {
* @throws SSHApiException throws during error
*/
public List<String> listDirectory(String directoryPath) throws SSHApiException;
+
+ /**
+ * This will list directories in computing resources
+ * @param directoryPath the full qualified path for the directory user wants to create
+ * @throws SSHApiException throws during error
+ */
+ public List<String> listDirectory(String directoryPath, boolean recursive) throws SSHApiException;
/**
* This method can be used to get created ssh session
http://git-wip-us.apache.org/repos/asf/airavata/blob/38893ff2/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
index f8b46c8..f7b1b8b 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -593,7 +593,52 @@ public class GSISSHAbstractCluster implements Cluster {
session.connect();
}
log.info("Listing directory: " + serverInfo.getHost() + ":" + directoryPath);
- files = SSHUtils.listDirectory(directoryPath, session);
+ files = SSHUtils.listDirectory(directoryPath, session, false);
+ retry=0;
+ } catch (IOException e) {
+ log.error(e.getMessage(), e);
+ retry--;
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e1) {
+ log.error(e1.getMessage(), e1);
+ }
+ reconnect(serverInfo, authenticationInfo);
+ if (retry == 0) {
+ throw new SSHApiException("Failed during listing directory:" + directoryPath + " to remote file ", e);
+ }
+ } catch (JSchException e) {
+ retry--;
+ reconnect(serverInfo, authenticationInfo);
+ if (retry == 0) {
+ throw new SSHApiException("Failed during listing directory :" + directoryPath + " to remote file ", e);
+ }
+ }catch (SSHApiException e) {
+ retry--;
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e1) {
+ log.error(e1.getMessage(), e1);
+ }
+ reconnect(serverInfo, authenticationInfo);
+ if (retry == 0) {
+ throw new SSHApiException("Failed during listing directory :" + directoryPath + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ }
+ }
+ }
+ return files;
+ }
+ public synchronized List<String> listDirectory(String directoryPath, boolean recursive) throws SSHApiException {
+ int retry = 3;
+ List<String> files = null;
+ while (retry > 0) {
+ try {
+ if (!session.isConnected()) {
+ session.connect();
+ }
+ log.info("Listing directory: " + serverInfo.getHost() + ":" + directoryPath);
+ files = SSHUtils.listDirectory(directoryPath, session, recursive);
retry=0;
} catch (IOException e) {
log.error(e.getMessage(), e);
http://git-wip-us.apache.org/repos/asf/airavata/blob/38893ff2/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
index 1741833..73bb049 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
@@ -701,10 +701,15 @@ public class SSHUtils {
channel.disconnect();
}
- public static List<String> listDirectory(String path, Session session) throws IOException, JSchException, SSHApiException {
+ public static List<String> listDirectory(String path, Session session, boolean recursive) throws IOException, JSchException, SSHApiException {
// exec 'scp -t rfile' remotely
- String command = "ls " + path;
+ String command;
+ if(recursive){
+ command = "ls -LR " + path;
+ }else{
+ command = "ls " + path;
+ }
Channel channel = session.openChannel("exec");
StandardOutReader stdOutReader = new StandardOutReader();