You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2014/05/21 19:56:49 UTC
[35/51] [abbrv] git commit: Fixed output population. AIRAVATA-1185
Fixed output population. AIRAVATA-1185
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/9be225ac
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/9be225ac
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/9be225ac
Branch: refs/heads/temp
Commit: 9be225ac4013ffea801b7d1b516fc09f830740f4
Parents: 0eabb93
Author: raminder <ra...@apache.org>
Authored: Tue May 6 10:27:16 2014 -0400
Committer: raminder <ra...@apache.org>
Committed: Tue May 6 10:27:16 2014 -0400
----------------------------------------------------------------------
.../apache/airavata/gfac/utils/OutputUtils.java | 45 ++++++++++++++++
.../gfac/handler/GSISSHOutputHandler.java | 54 +++++++++++++-------
.../airavata/gfac/handler/SSHOutputHandler.java | 28 ++++++----
3 files changed, 98 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java
index efcbcd5..fa47745 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/utils/OutputUtils.java
@@ -21,6 +21,7 @@
package org.apache.airavata.gfac.utils;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
@@ -30,6 +31,8 @@ import org.apache.airavata.common.utils.StringUtil;
import org.apache.airavata.commons.gfac.type.ActualParameter;
import org.apache.airavata.commons.gfac.type.MappingFactory;
import org.apache.airavata.gfac.handler.GFacHandlerException;
+import org.apache.airavata.model.workspace.experiment.DataObjectType;
+import org.apache.airavata.schemas.gfac.DataType;
import org.apache.airavata.schemas.gfac.StdErrParameterType;
import org.apache.airavata.schemas.gfac.StdOutParameterType;
@@ -71,6 +74,48 @@ public class OutputUtils {
return result;
}
+
+ public static void fillOutputFromStdout1(Map<String, Object> output, String stdout, String stderr,List<DataObjectType> outputArray) throws Exception {
+
+ if (stdout == null || stdout.equals("")){
+ throw new GFacHandlerException("Standard output is empty.");
+ }
+
+ Set<String> keys = output.keySet();
+ for (String paramName : keys) {
+ ActualParameter actual = (ActualParameter) output.get(paramName);
+ // if parameter value is not already set, we let it go
+
+ if (actual == null) {
+ continue;
+ }
+ if ("StdOut".equals(actual.getType().getType().toString())) {
+ DataObjectType out = new DataObjectType();
+ out.setKey(paramName);
+ out.setType(DataType.STD_OUT.toString());
+ out.setValue(stdout);
+ outputArray.add(out);
+ } else if ("StdErr".equals(actual.getType().getType().toString())) {
+ DataObjectType out = new DataObjectType();
+ out.setKey(paramName);
+ out.setType(DataType.STD_ERR.toString());
+ out.setValue(stderr);
+ outputArray.add(out);
+ }else if("URI".equals(actual.getType().getType().toString())){
+ continue;
+ }
+ else {
+ String parseStdout = parseStdout(stdout, paramName);
+ if (parseStdout != null) {
+ DataObjectType out = new DataObjectType();
+ out.setKey(paramName);
+ out.setType(DataType.STRING.toString());
+ out.setValue(parseStdout);
+ outputArray.add(out);
+ }
+ }
+ }
+ }
private static String parseStdout(String stdout, String outParam) throws Exception {
String regex = Pattern.quote(outParam) + regexPattern;
http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
index fd80d56..6eff137 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/handler/GSISSHOutputHandler.java
@@ -20,8 +20,18 @@
*/
package org.apache.airavata.gfac.handler;
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import net.schmizz.sshj.connection.ConnectionException;
import net.schmizz.sshj.transport.TransportException;
+
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.Constants;
import org.apache.airavata.commons.gfac.type.ActualParameter;
@@ -35,7 +45,14 @@ import org.apache.airavata.gfac.utils.GFacUtils;
import org.apache.airavata.gfac.utils.OutputUtils;
import org.apache.airavata.gsi.ssh.api.Cluster;
import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
-import org.apache.airavata.model.workspace.experiment.*;
+import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
+import org.apache.airavata.model.workspace.experiment.DataObjectType;
+import org.apache.airavata.model.workspace.experiment.DataTransferDetails;
+import org.apache.airavata.model.workspace.experiment.ErrorCategory;
+import org.apache.airavata.model.workspace.experiment.JobDetails;
+import org.apache.airavata.model.workspace.experiment.TaskDetails;
+import org.apache.airavata.model.workspace.experiment.TransferState;
+import org.apache.airavata.model.workspace.experiment.TransferStatus;
import org.apache.airavata.registry.cpi.ChildDataType;
import org.apache.airavata.registry.cpi.DataType;
import org.apache.airavata.registry.cpi.RegistryException;
@@ -46,12 +63,8 @@ import org.apache.xmlbeans.XmlException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
public class GSISSHOutputHandler extends AbstractHandler{
private static final Logger log = LoggerFactory.getLogger(GSISSHOutputHandler.class);
@@ -154,7 +167,7 @@ public class GSISSHOutputHandler extends AbstractHandler{
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+ List<DataObjectType> outputArray = new ArrayList<DataObjectType>();
Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
Set<String> keys = output.keySet();
for (String paramName : keys) {
@@ -163,31 +176,36 @@ public class GSISSHOutputHandler extends AbstractHandler{
List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr, outputArray);
+ break;
} else {
String valueList = outputList.get(0);
cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
- ((URIParameterType) actualParameter.getType()).setValue(valueList);
- stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
+ DataObjectType dataObjectType = new DataObjectType();
+ dataObjectType.setValue(valueList);
+ dataObjectType.setKey(paramName);
+ dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString());
+ outputArray.add(dataObjectType);
}
} else {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ OutputUtils.fillOutputFromStdout1(output, stdOutStr,stdErrStr, outputArray);
+ break;
}
}
- if (stringMap == null || stringMap.isEmpty()) {
+ if (outputArray == null || outputArray.isEmpty()) {
throw new GFacHandlerException(
"Empty Output returned from the Application, Double check the application"
+ "and ApplicationDescriptor output Parameter Names");
}
- status.setTransferState(TransferState.DOWNLOAD);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
app.setStandardError(localStdErrFile.getAbsolutePath());
app.setStandardOutput(localStdOutFile.getAbsolutePath());
app.setOutputDataDirectory(outputDataDir);
+ status.setTransferState(TransferState.DOWNLOAD);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription(outputDataDir);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID());
} catch (XmlException e) {
throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
} catch (ConnectionException e) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/9be225ac/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
index 42cebba..c70469f 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/handler/SSHOutputHandler.java
@@ -22,6 +22,7 @@ package org.apache.airavata.gfac.handler;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -150,7 +151,7 @@ public class SSHOutputHandler extends AbstractHandler{
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
- Map<String, ActualParameter> stringMap = new HashMap<String, ActualParameter>();
+ List<DataObjectType> outputArray = new ArrayList<DataObjectType>();
Map<String, Object> output = jobExecutionContext.getOutMessageContext().getParameters();
Set<String> keys = output.keySet();
for (String paramName : keys) {
@@ -159,31 +160,36 @@ public class SSHOutputHandler extends AbstractHandler{
List<String> outputList = cluster.listDirectory(app.getOutputDataDirectory());
if (outputList.size() == 0 || outputList.get(0).isEmpty()) {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray);
+ break;
} else {
String valueList = outputList.get(0);
cluster.scpFrom(app.getOutputDataDirectory() + File.separator + valueList, outputDataDir);
jobExecutionContext.addOutputFile(outputDataDir + File.separator + valueList);
- ((URIParameterType) actualParameter.getType()).setValue(valueList);
- stringMap = new HashMap<String, ActualParameter>();
- stringMap.put(paramName, actualParameter);
+ DataObjectType dataObjectType = new DataObjectType();
+ dataObjectType.setValue(valueList);
+ dataObjectType.setKey(paramName);
+ dataObjectType.setType(org.apache.airavata.schemas.gfac.DataType.URI.toString());
+ outputArray.add(dataObjectType);
}
} else {
- stringMap = OutputUtils.fillOutputFromStdout(output, stdOutStr, stdErrStr);
+ OutputUtils.fillOutputFromStdout1(output, stdOutStr, stdErrStr,outputArray);
}
}
- if (stringMap == null || stringMap.isEmpty()) {
+ if (outputArray == null || outputArray.isEmpty()) {
throw new GFacHandlerException(
"Empty Output returned from the Application, Double check the application"
+ "and ApplicationDescriptor output Parameter Names");
}
- status.setTransferState(TransferState.DOWNLOAD);
- detail.setTransferStatus(status);
- registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
-
app.setStandardError(localStdErrFile.getAbsolutePath());
app.setStandardOutput(localStdOutFile.getAbsolutePath());
app.setOutputDataDirectory(outputDataDir);
+ status.setTransferState(TransferState.DOWNLOAD);
+ detail.setTransferStatus(status);
+ detail.setTransferDescription(outputDataDir);
+ registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
+ registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID());
+
} catch (XmlException e) {
throw new GFacHandlerException("Cannot read output:" + e.getMessage(), e);
} catch (ConnectionException e) {