You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ch...@apache.org on 2015/01/08 20:38:25 UTC
airavata git commit: treating STDOUT and STDERR as outputs
Repository: airavata
Updated Branches:
refs/heads/master 98c9a34ea -> 8ff835fdd
treating STDOUT and STDERR as outputs
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8ff835fd
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8ff835fd
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8ff835fd
Branch: refs/heads/master
Commit: 8ff835fdde359c5edf8e34f426f76ff6f50c1d26
Parents: 98c9a34
Author: Chathuri Wimalasena <ka...@gmail.com>
Authored: Thu Jan 8 14:38:17 2015 -0500
Committer: Chathuri Wimalasena <ka...@gmail.com>
Committed: Thu Jan 8 14:38:17 2015 -0500
----------------------------------------------------------------------
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 34 +++++++++++------
.../apache/airavata/job/GFacConfigXmlTest.java | 6 ---
.../gsissh/handler/GSISSHOutputHandler.java | 40 +++++++++++++++++++-
.../gfac/gsissh/util/GFACGSISSHUtils.java | 4 +-
.../ssh/handler/AdvancedSCPOutputHandler.java | 26 ++++++++++++-
5 files changed, 86 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index 77c9d60..006319c 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -51,9 +51,7 @@ import org.apache.airavata.gfac.core.utils.GFacUtils;
import org.apache.airavata.messaging.core.Publisher;
import org.apache.airavata.messaging.core.PublisherFactory;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
-import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription;
-import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
-import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.*;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
import org.apache.airavata.model.messaging.event.*;
@@ -303,13 +301,6 @@ public class BetterGfacImpl implements GFac,Watcher {
List<InputDataObjectType> taskInputs = taskData.getApplicationInputs();
jobExecutionContext.setInMessageContext(new MessageContext(GFacUtils.getInputParamMap(taskInputs)));
-// List<OutputDataObjectType> outputData = experiment.getExperimentOutputs();
- List<OutputDataObjectType> taskOutputs = taskData.getApplicationOutputs();
- if (taskOutputs == null || taskOutputs.isEmpty() ){
- taskOutputs = applicationInterface.getApplicationOutputs();
- }
- jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOuputParamMap(taskOutputs)));
-
jobExecutionContext.setProperty(Constants.PROP_TOPIC, experimentID);
jobExecutionContext.setGfac(this);
jobExecutionContext.setZk(zk);
@@ -384,6 +375,23 @@ public class BetterGfacImpl implements GFac,Watcher {
} else {
setUpWorkingLocation(jobExecutionContext, applicationInterface, "/tmp");
}
+ List<OutputDataObjectType> taskOutputs = taskData.getApplicationOutputs();
+ if (taskOutputs == null || taskOutputs.isEmpty() ){
+ taskOutputs = applicationInterface.getApplicationOutputs();
+ }
+
+ for (OutputDataObjectType objectType : taskOutputs){
+ if (objectType.getAddedToCommandLine() != null && objectType.getAddedToCommandLine()== CommandLineType.EXPLICIT){
+ objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + objectType.getName());
+ }
+ if (objectType.getType() == DataType.STDOUT){
+ objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + jobExecutionContext.getApplicationName() + ".stdout");
+ }
+ if (objectType.getType() == DataType.STDERR){
+ objectType.setValue(jobExecutionContext.getOutputDir() + File.separator + jobExecutionContext.getApplicationName() + ".stderr");
+ }
+ }
+ jobExecutionContext.setOutMessageContext(new MessageContext(GFacUtils.getOuputParamMap(taskOutputs)));
return jobExecutionContext;
}
@@ -402,8 +410,10 @@ public class BetterGfacImpl implements GFac,Watcher {
/*
* Input and Output Directory
*/
- jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME);
- jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME);
+// jobExecutionContext.setInputDir(workingDir + File.separator + Constants.INPUT_DATA_DIR_VAR_NAME);
+ jobExecutionContext.setInputDir(workingDir);
+// jobExecutionContext.setOutputDir(workingDir + File.separator + Constants.OUTPUT_DATA_DIR_VAR_NAME);
+ jobExecutionContext.setOutputDir(workingDir);
/*
* Stdout and Stderr for Shell
http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java
index 237173f..507ab9e 100644
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/job/GFacConfigXmlTest.java
@@ -21,14 +21,8 @@
package org.apache.airavata.job;
import junit.framework.Assert;
-import org.airavata.appcatalog.cpi.AppCatalog;
-import org.airavata.appcatalog.cpi.AppCatalogException;
-import org.apache.aiaravata.application.catalog.data.impl.AppCatalogFactory;
import org.apache.airavata.common.utils.AiravataUtils;
-import org.apache.airavata.gfac.ExecutionMode;
import org.apache.airavata.gfac.GFacConfiguration;
-import org.apache.airavata.gfac.GFacException;
-import org.apache.airavata.gfac.Scheduler;
import org.apache.airavata.gfac.core.context.ApplicationContext;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
index 107f1fe..a29028d 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
@@ -129,7 +129,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
data.append(oldFiles.get(index++)).append(",");
} else {
int i = 0;
- localStdOutFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stdout");
+ localStdOutFile = new File(outputDataDir + File.separator + jobExecutionContext.getApplicationName() + ".stdout");
while(stdOutStr.isEmpty()){
try {
cluster.scpFrom(jobExecutionContext.getStandardOutput(), localStdOutFile.getAbsolutePath());
@@ -149,7 +149,7 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
localStdErrFile = new File(oldFiles.get(index));
data.append(oldFiles.get(index++)).append(",");
} else {
- localStdErrFile = new File(outputDataDir + File.separator + timeStampedExperimentID + "stderr");
+ localStdErrFile = new File(outputDataDir + File.separator + jobExecutionContext.getApplicationName() + ".stderr");
cluster.scpFrom(jobExecutionContext.getStandardError(), localStdErrFile.getAbsolutePath());
StringBuffer temp = new StringBuffer(data.append(localStdErrFile.getAbsolutePath()).append(",").toString());
GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName());
@@ -216,6 +216,42 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
dataObjectType.setName(key);
dataObjectType.setType(DataType.URI);
outputArray.add(dataObjectType);
+ }else if (DataType.STDOUT == outputDataObjectType1.getType()) {
+ String localFile;
+ if (index < oldIndex) {
+ localFile = oldFiles.get(index);
+ data.append(oldFiles.get(index++)).append(",");
+ } else {
+ String fileName = localStdOutFile.getName();
+ localFile = outputDataDir + File.separator + fileName;
+ StringBuffer temp = new StringBuffer(data.append(localFile).append(",").toString());
+ GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName());
+ }
+ jobExecutionContext.addOutputFile(localFile);
+ outputDataObjectType1.setValue(localFile);
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(localFile);
+ dataObjectType.setName(key);
+ dataObjectType.setType(DataType.STDOUT);
+ outputArray.add(dataObjectType);
+ }else if (DataType.STDERR == outputDataObjectType1.getType()) {
+ String localFile;
+ if (index < oldIndex) {
+ localFile = oldFiles.get(index);
+ data.append(oldFiles.get(index++)).append(",");
+ } else {
+ String fileName = localStdErrFile.getName();
+ localFile = outputDataDir + File.separator + fileName;
+ StringBuffer temp = new StringBuffer(data.append(localFile).append(",").toString());
+ GFacUtils.savePluginData(jobExecutionContext, temp.insert(0, ++index), this.getClass().getName());
+ }
+ jobExecutionContext.addOutputFile(localFile);
+ outputDataObjectType1.setValue(localFile);
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(localFile);
+ dataObjectType.setName(key);
+ dataObjectType.setType(DataType.STDERR);
+ outputArray.add(dataObjectType);
}
}
break;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
index 6ef3395..1e03668 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
@@ -214,7 +214,7 @@ public class GFACGSISSHUtils {
}
for (InputDataObjectType inputDataObjectType : sortedInputSet) {
if (inputDataObjectType.getAddedToCommandLine() != null
- && inputDataObjectType.getAddedToCommandLine() == CommandLineType.EXPLICIT) {
+ && inputDataObjectType.getAddedToCommandLine() == CommandLineType.IMPLICIT) {
continue;
}
if (inputDataObjectType.getApplicationArgument() != null
@@ -233,7 +233,7 @@ public class GFACGSISSHUtils {
if (outputParam instanceof OutputDataObjectType) {
OutputDataObjectType output = (OutputDataObjectType) outputParam;
if (output.getValue() != null && !output.getValue().equals("") && output.getAddedToCommandLine() != null
- && output.getAddedToCommandLine() == CommandLineType.IMPLICIT) {
+ && output.getAddedToCommandLine() == CommandLineType.EXPLICIT) {
inputValues.add(output.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/8ff835fd/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
index 154041f..29091d5 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/AdvancedSCPOutputHandler.java
@@ -34,8 +34,10 @@ import org.apache.airavata.gsi.ssh.api.SSHApiException;
import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPasswordAuthenticationInfo;
import org.apache.airavata.gsi.ssh.impl.authentication.DefaultPublicKeyFileAuthentication;
+import org.apache.airavata.model.appcatalog.appinterface.CommandLineType;
import org.apache.airavata.model.appcatalog.appinterface.DataType;
import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.ValidityType;
import org.apache.airavata.model.workspace.experiment.CorrectiveAction;
import org.apache.airavata.model.workspace.experiment.ErrorCategory;
import org.apache.airavata.registry.cpi.ChildDataType;
@@ -146,9 +148,9 @@ public class AdvancedSCPOutputHandler extends AbstractHandler {
OutputDataObjectType outputDataObjectType = (OutputDataObjectType) output.get(paramName);
if (outputDataObjectType.getType() == DataType.URI) {
String downloadFile = outputDataObjectType.getValue();
- if(downloadFile == null || !(new File(downloadFile).isFile())){
+ if(downloadFile == null || !(new File(downloadFile).isFile())){
GFacUtils.saveErrorDetails(jobExecutionContext, "Empty Output returned from the application", CorrectiveAction.CONTACT_SUPPORT, ErrorCategory.AIRAVATA_INTERNAL_ERROR);
- throw new GFacHandlerException("Empty Output returned from the application");
+ throw new GFacHandlerException("Empty Output returned from the application.." );
}
pbsCluster.scpTo(outputPath, downloadFile);
String fileName = downloadFile.substring(downloadFile.lastIndexOf(File.separatorChar)+1, downloadFile.length());
@@ -157,6 +159,26 @@ public class AdvancedSCPOutputHandler extends AbstractHandler {
dataObjectType.setName(paramName);
dataObjectType.setType(DataType.URI);
outputArray.add(dataObjectType);
+ }else if (outputDataObjectType.getType() == DataType.STDOUT) {
+ pbsCluster.scpTo(outputPath, standardOutput);
+ String fileName = standardOutput.substring(standardOutput.lastIndexOf(File.separatorChar)+1, standardOutput.length());
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(outputPath + File.separatorChar + fileName);
+ dataObjectType.setName(paramName);
+ dataObjectType.setType(DataType.STDOUT);
+ dataObjectType.setValidityType(ValidityType.REQUIRED);
+ dataObjectType.setAddedToCommandLine(CommandLineType.IMPLICIT);
+ outputArray.add(dataObjectType);
+ }else if (outputDataObjectType.getType() == DataType.STDERR) {
+ pbsCluster.scpTo(outputPath, standardError);
+ String fileName = standardError.substring(standardError.lastIndexOf(File.separatorChar)+1, standardError.length());
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(outputPath + File.separatorChar + fileName);
+ dataObjectType.setName(paramName);
+ dataObjectType.setType(DataType.STDERR);
+ dataObjectType.setValidityType(ValidityType.REQUIRED);
+ dataObjectType.setAddedToCommandLine(CommandLineType.IMPLICIT);
+ outputArray.add(dataObjectType);
}
}
registry.add(ChildDataType.EXPERIMENT_OUTPUT, outputArray, jobExecutionContext.getExperimentID());