You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ra...@apache.org on 2014/03/12 22:39:35 UTC
[1/2] git commit: Fixed output data reading. AIRAVATA-1077
Repository: airavata
Updated Branches:
refs/heads/master 7a8838fdb -> 0a46c0cba
Fixed output data reading. AIRAVATA-1077
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/96431980
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/96431980
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/96431980
Branch: refs/heads/master
Commit: 964319800e6f5b7570d5c8be916976bb76954c44
Parents: 0b6de06
Author: raminder <ra...@apache.org>
Authored: Wed Mar 12 17:36:03 2014 -0400
Committer: raminder <ra...@apache.org>
Committed: Wed Mar 12 17:36:03 2014 -0400
----------------------------------------------------------------------
.../airavata/gfac/handler/SCPOutputHandler.java | 24 +++++++++++++-
.../apache/airavata/gfac/utils/GFacUtils.java | 2 ++
.../apache/airavata/gsi/ssh/api/Cluster.java | 9 +++++
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 14 ++++++++
.../apache/airavata/gsi/ssh/util/SSHUtils.java | 35 ++++++++++++++++++++
5 files changed, 83 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/96431980/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/handler/SCPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/handler/SCPOutputHandler.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/handler/SCPOutputHandler.java
index 07b869c..dc267be 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/handler/SCPOutputHandler.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/handler/SCPOutputHandler.java
@@ -22,8 +22,11 @@ package org.apache.airavata.gfac.handler;
import java.io.File;
import java.io.IOException;
+import java.net.URI;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import net.schmizz.sshj.connection.ConnectionException;
import net.schmizz.sshj.transport.TransportException;
@@ -37,10 +40,12 @@ import org.apache.airavata.gfac.provider.GFacProviderException;
import org.apache.airavata.gfac.utils.GFacUtils;
import org.apache.airavata.gfac.utils.OutputUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
+import org.apache.airavata.gsi.ssh.util.SSHUtils;
import org.apache.airavata.model.workspace.experiment.*;
import org.apache.airavata.persistance.registry.jpa.model.DataTransferDetail;
import org.apache.airavata.registry.cpi.ChildDataType;
import org.apache.airavata.schemas.gfac.ApplicationDeploymentDescriptionType;
+import org.apache.airavata.schemas.gfac.URIParameterType;
import org.apache.xmlbeans.XmlException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -107,7 +112,24 @@ public class SCPOutputHandler extends AbstractHandler{
Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ Set<String> keys = output.keySet();
+ for (String paramName : keys) {
+ ActualParameter actualParameter = (ActualParameter) output.get(paramName);
+ if ("URI".equals(actualParameter.getType().getType().toString())) {
+
+ List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
+ if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ } else {
+ String valueList = outputList.get(0);
+ ((URIParameterType) actualParameter.getType()).setValue(valueList);
+ stringMap = new HashMap<String, ActualParameter>();
+ stringMap.put(paramName, actualParameter);
+ }
+ }else{
+ stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ }
+ }
if (stringMap == null || stringMap.isEmpty()) {
throw new GFacHandlerException(
"Empty Output returned from the Application, Double check the application"
http://git-wip-us.apache.org/repos/asf/airavata/blob/96431980/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/GFacUtils.java
index 0ea841f..9f1e32d 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/GFacUtils.java
@@ -38,6 +38,7 @@ import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.gfac.Constants;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.context.JobExecutionContext;
+import org.apache.airavata.model.workspace.experiment.ActionableGroup;
import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
import org.apache.airavata.model.workspace.experiment.DataObjectType;
import org.apache.airavata.model.workspace.experiment.ErrorCategory;
@@ -734,6 +735,7 @@ public class GFacUtils {
ErrorDetails details = new ErrorDetails();
details.setActualErrorMessage(errorMessage);
details.setCorrectiveAction(action);
+ details.setActionableGroup(ActionableGroup.GATEWAYS_ADMINS);
details.setCreationTime(Calendar.getInstance().getTimeInMillis());
details.setErrorCategory(errorCatogory);
registry.add(ChildDataType.ERROR_DETAIL, details, id);
http://git-wip-us.apache.org/repos/asf/airavata/blob/96431980/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
index ee5dcf7..f8cd543 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/api/Cluster.java
@@ -20,6 +20,8 @@
*/
package org.apache.airavata.gsi.ssh.api;
+import java.util.List;
+
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.impl.JobStatus;
@@ -100,6 +102,13 @@ public interface Cluster {
* @throws SSHApiException throws exception during error
*/
public JobStatus getJobStatus(String jobID) throws SSHApiException;
+
+ /**
+ * This will list directories in computing resources
+ * @param directoryPath the full qualified path for the directory user wants to create
+ * @throws SSHApiException throws during error
+ */
+ public List<String> listDirectory(String directoryPath) throws SSHApiException;
/**
http://git-wip-us.apache.org/repos/asf/airavata/blob/96431980/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
index c6866c0..5c24478 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
@@ -21,6 +21,7 @@
package org.apache.airavata.gsi.ssh.impl;
import com.jcraft.jsch.*;
+
import org.apache.airavata.gsi.ssh.api.*;
import org.apache.airavata.gsi.ssh.api.authentication.*;
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
@@ -39,12 +40,14 @@ import org.slf4j.LoggerFactory;
import javax.xml.transform.*;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;
+
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.security.SecureRandom;
+import java.util.List;
public class GSISSHAbstractCluster implements Cluster {
static {
@@ -376,6 +379,17 @@ public class GSISSHAbstractCluster implements Cluster {
}
}
+ public List<String> listDirectory(String directoryPath) throws SSHApiException {
+ try {
+ return SSHUtils.listDirectory(directoryPath, session);
+ } catch (IOException e) {
+ throw new SSHApiException("Failed during creating directory:" + directoryPath + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ } catch (JSchException e) {
+ throw new SSHApiException("Failed during creating directory :" + directoryPath + " to remote file "
+ + serverInfo.getHost() + ":rFile", e);
+ }
+ }
public ServerInfo getServerInfo() {
return serverInfo;
http://git-wip-us.apache.org/repos/asf/airavata/blob/96431980/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
----------------------------------------------------------------------
diff --git a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
index 5e92c43..02547a0 100644
--- a/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
+++ b/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/util/SSHUtils.java
@@ -21,6 +21,7 @@
package org.apache.airavata.gsi.ssh.util;
import com.jcraft.jsch.*;
+
import org.apache.airavata.gsi.ssh.api.authentication.GSIAuthenticationInfo;
import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.ServerInfo;
@@ -30,6 +31,8 @@ import org.apache.airavata.gsi.ssh.jsch.ExtendedJSch;
import org.slf4j.*;
import java.io.*;
+import java.util.Arrays;
+import java.util.List;
/**
* This class is going to be useful to SCP a file to a remote grid machine using my proxy credentials
@@ -611,6 +614,38 @@ public class SSHUtils {
channel.disconnect();
}
+ public static List<String> listDirectory(String path, Session session) throws IOException, JSchException, SSHApiException {
+
+ // exec 'scp -t rfile' remotely
+ String command = "ls " + path;
+ Channel channel = session.openChannel("exec");
+ StandardOutReader stdOutReader = new StandardOutReader();
+
+ ((ChannelExec) channel).setCommand(command);
+
+
+ ((ChannelExec) channel).setErrStream(stdOutReader.getStandardError());
+ try {
+ channel.connect();
+ } catch (JSchException e) {
+
+ channel.disconnect();
+ session.disconnect();
+
+ throw new SSHApiException("Unable to retrieve command output. Command - " + command +
+ " on server - " + session.getHost() + ":" + session.getPort() +
+ " connecting user name - "
+ + session.getUserName(), e);
+ }
+ stdOutReader.onOutput(channel);
+ stdOutReader.getStdOutputString();
+ if (!stdOutReader.getStdErrorString().equals("")) {
+ throw new SSHApiException(stdOutReader.getStdErrorString());
+ }
+ channel.disconnect();
+ return Arrays.asList(stdOutReader.getStdOutputString().split("\n"));
+ }
+
static int checkAck(InputStream in) throws IOException {
int b = in.read();
if (b == 0) return b;
[2/2] git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Posted by ra...@apache.org.
Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/airavata
Merging the changes for AIRAVATA-1077
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/0a46c0cb
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/0a46c0cb
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/0a46c0cb
Branch: refs/heads/master
Commit: 0a46c0cba6a9a05cc3cc76c95af01e18acb89b5d
Parents: 9643198 7a8838f
Author: raminder <ra...@apache.org>
Authored: Wed Mar 12 17:39:22 2014 -0400
Committer: raminder <ra...@apache.org>
Committed: Wed Mar 12 17:39:22 2014 -0400
----------------------------------------------------------------------
.../server/handler/AiravataServerHandler.java | 37 +++++++++-----------
.../registry/jpa/impl/ExperimentRegistry.java | 4 +++
.../jpa/resources/QosParamResource.java | 8 +++--
.../airavata/gsi/ssh/api/job/OutputParser.java | 9 ++---
.../gsi/ssh/api/job/SlurmOutputParser.java | 12 ++++---
.../gsi/ssh/impl/GSISSHAbstractCluster.java | 1 -
6 files changed, 38 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/0a46c0cb/tools/gsissh/src/main/java/org/apache/airavata/gsi/ssh/impl/GSISSHAbstractCluster.java
----------------------------------------------------------------------