You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by ch...@apache.org on 2015/01/12 16:12:01 UTC
airavata git commit: adding relative path when job creation,
save file uri to data transfer details
Repository: airavata
Updated Branches:
refs/heads/master e54313c5e -> 97ad96b27
adding relative path when job creation, save file uri to data transfer details
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/97ad96b2
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/97ad96b2
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/97ad96b2
Branch: refs/heads/master
Commit: 97ad96b27c8eddb9a10e20d192b20374857db3e4
Parents: e54313c
Author: Chathuri Wimalasena <ka...@gmail.com>
Authored: Mon Jan 12 10:11:48 2015 -0500
Committer: Chathuri Wimalasena <ka...@gmail.com>
Committed: Mon Jan 12 10:11:48 2015 -0500
----------------------------------------------------------------------
.../gsissh/handler/GSISSHOutputHandler.java | 4 +-
.../gfac/gsissh/util/GFACGSISSHUtils.java | 12 ++-
.../gfac/ssh/handler/SSHOutputHandler.java | 26 +++++-
.../airavata/gfac/ssh/util/GFACSSHUtils.java | 85 ++++++++++++++++++--
4 files changed, 115 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
index a29028d..2d6e387 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/handler/GSISSHOutputHandler.java
@@ -158,12 +158,12 @@ public class GSISSHOutputHandler extends AbstractRecoverableHandler {
String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
status.setTransferState(TransferState.STDOUT_DOWNLOAD);
detail.setTransferStatus(status);
- detail.setTransferDescription("STDOUT:" + stdOutStr);
+ detail.setTransferDescription("STDOUT:" + localStdOutFile.getAbsolutePath());
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
status.setTransferState(TransferState.STDERROR_DOWNLOAD);
detail.setTransferStatus(status);
- detail.setTransferDescription("STDERR:" + stdErrStr);
+ detail.setTransferDescription("STDERR:" + localStdErrFile.getAbsolutePath());
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
//todo this is a mess we have to fix this
http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
index 1e03668..1bc0683 100644
--- a/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
+++ b/modules/gfac/gfac-gsissh/src/main/java/org/apache/airavata/gfac/gsissh/util/GFACGSISSHUtils.java
@@ -41,6 +41,7 @@ import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.util.CommonUtils;
import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
import org.apache.airavata.model.appcatalog.appinterface.CommandLineType;
+import org.apache.airavata.model.appcatalog.appinterface.DataType;
import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
import org.apache.airavata.model.appcatalog.computeresource.*;
@@ -50,6 +51,7 @@ import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.*;
@@ -224,7 +226,15 @@ public class GFACGSISSHUtils {
if (inputDataObjectType.getValue() != null
&& !inputDataObjectType.getValue().equals("")) {
- inputValues.add(inputDataObjectType.getValue());
+ if (inputDataObjectType.getType() == DataType.URI) {
+ // set only the relative path
+ String filePath = inputDataObjectType.getValue();
+ filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length());
+ inputValues.add(filePath);
+ }else {
+ inputValues.add(inputDataObjectType.getValue());
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
index e2469fd..b574540 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/handler/SSHOutputHandler.java
@@ -122,12 +122,12 @@ public class SSHOutputHandler extends AbstractHandler {
String stdErrStr = GFacUtils.readFileToString(localStdErrFile.getAbsolutePath());
status.setTransferState(TransferState.STDOUT_DOWNLOAD);
detail.setTransferStatus(status);
- detail.setTransferDescription("STDOUT:" + stdOutStr);
+ detail.setTransferDescription("STDOUT:" + localStdOutFile.getAbsolutePath());
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
status.setTransferState(TransferState.STDERROR_DOWNLOAD);
detail.setTransferStatus(status);
- detail.setTransferDescription("STDERR:" + stdErrStr);
+ detail.setTransferDescription("STDERR:" + localStdErrFile.getAbsolutePath());
registry.add(ChildDataType.DATA_TRANSFER_DETAIL, detail, jobExecutionContext.getTaskData().getTaskID());
@@ -166,9 +166,28 @@ public class SSHOutputHandler extends AbstractHandler {
dataObjectType.setName(key);
dataObjectType.setType(DataType.URI);
outputArray.add(dataObjectType);
+ }else if (DataType.STDOUT == actualParameter.getType()) {
+ String fileName = localStdOutFile.getName();
+ String localFile = outputDataDir + File.separator + fileName;
+ jobExecutionContext.addOutputFile(localFile);
+ actualParameter.setValue(localFile);
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(localFile);
+ dataObjectType.setName(key);
+ dataObjectType.setType(DataType.STDOUT);
+ outputArray.add(dataObjectType);
+ }else if (DataType.STDERR == actualParameter.getType()) {
+ String fileName = localStdErrFile.getName();
+ String localFile = outputDataDir + File.separator + fileName;
+ jobExecutionContext.addOutputFile(localFile);
+ actualParameter.setValue(localFile);
+ OutputDataObjectType dataObjectType = new OutputDataObjectType();
+ dataObjectType.setValue(localFile);
+ dataObjectType.setName(key);
+ dataObjectType.setType(DataType.STDERR);
+ outputArray.add(dataObjectType);
}
}
-
break;
} else if (outputList.size() == 1) {//FIXME: Ultrascan case
String valueList = outputList.get(0);
@@ -194,7 +213,6 @@ public class SSHOutputHandler extends AbstractHandler {
+ "and ApplicationDescriptor output Parameter Names");
}
}
- // FIXME: why we set standard error ouput and outputDirectory again ?
jobExecutionContext.setStandardError(localStdErrFile.getAbsolutePath());
jobExecutionContext.setStandardOutput(localStdOutFile.getAbsolutePath());
jobExecutionContext.setOutputDir(outputDataDir);
http://git-wip-us.apache.org/repos/asf/airavata/blob/97ad96b2/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
index ab698ff..d649ade 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/util/GFACSSHUtils.java
@@ -41,7 +41,11 @@ import org.apache.airavata.gsi.ssh.api.job.JobDescriptor;
import org.apache.airavata.gsi.ssh.impl.GSISSHAbstractCluster;
import org.apache.airavata.gsi.ssh.impl.PBSCluster;
import org.apache.airavata.gsi.ssh.util.CommonUtils;
+import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription;
+import org.apache.airavata.model.appcatalog.appinterface.CommandLineType;
+import org.apache.airavata.model.appcatalog.appinterface.DataType;
import org.apache.airavata.model.appcatalog.appinterface.InputDataObjectType;
+import org.apache.airavata.model.appcatalog.appinterface.OutputDataObjectType;
import org.apache.airavata.model.appcatalog.computeresource.*;
import org.apache.airavata.model.appcatalog.gatewayprofile.ComputeResourcePreference;
import org.apache.airavata.model.workspace.experiment.ComputationalResourceScheduling;
@@ -51,6 +55,7 @@ import org.apache.airavata.model.workspace.experiment.TaskDetails;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.*;
public class GFACSSHUtils {
@@ -248,14 +253,58 @@ public class GFACSSHUtils {
int i = random.nextInt(Integer.MAX_VALUE);
jobDescriptor.setJobName(String.valueOf(i + 99999999));
jobDescriptor.setWorkingDirectory(jobExecutionContext.getWorkingDir());
+
List<String> inputValues = new ArrayList<String>();
MessageContext input = jobExecutionContext.getInMessageContext();
- Map<String, Object> inputs = input.getParameters();
- Set<String> keys = inputs.keySet();
- for (String paramName : keys) {
- InputDataObjectType inputDataObjectType = (InputDataObjectType) inputs.get(paramName);
- inputValues.add(inputDataObjectType.getValue());
+
+ // sort the inputs first and then build the command List
+ Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() {
+ @Override
+ public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) {
+ return inputDataObjectType.getInputOrder() - t1.getInputOrder();
+ }
+ };
+ Set<InputDataObjectType> sortedInputSet = new TreeSet<InputDataObjectType>(inputOrderComparator);
+ for (Object object : input.getParameters().values()) {
+ if (object instanceof InputDataObjectType) {
+ InputDataObjectType inputDOT = (InputDataObjectType) object;
+ sortedInputSet.add(inputDOT);
+ }
+ }
+ for (InputDataObjectType inputDataObjectType : sortedInputSet) {
+ if (inputDataObjectType.getAddedToCommandLine() != null
+ && inputDataObjectType.getAddedToCommandLine() == CommandLineType.IMPLICIT) {
+ continue;
+ }
+ if (inputDataObjectType.getApplicationArgument() != null
+ && !inputDataObjectType.getApplicationArgument().equals("")) {
+ inputValues.add(inputDataObjectType.getApplicationArgument());
+ }
+
+ if (inputDataObjectType.getValue() != null
+ && !inputDataObjectType.getValue().equals("")) {
+ if (inputDataObjectType.getType() == DataType.URI) {
+ // set only the relative path
+ String filePath = inputDataObjectType.getValue();
+ filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length());
+ inputValues.add(filePath);
+ }else {
+ inputValues.add(inputDataObjectType.getValue());
+ }
+
+ }
}
+ Map<String, Object> outputParams = jobExecutionContext.getOutMessageContext().getParameters();
+ for (Object outputParam : outputParams.values()) {
+ if (outputParam instanceof OutputDataObjectType) {
+ OutputDataObjectType output = (OutputDataObjectType) outputParam;
+ if (output.getValue() != null && !output.getValue().equals("") && output.getAddedToCommandLine() != null
+ && output.getAddedToCommandLine() == CommandLineType.EXPLICIT) {
+ inputValues.add(output.getValue());
+ }
+ }
+ }
+
jobDescriptor.setInputValues(inputValues);
jobDescriptor.setUserName(((GSISSHAbstractCluster) cluster).getServerInfo().getUserName());
jobDescriptor.setShellName("/bin/bash");
@@ -296,9 +345,35 @@ public class GFACSSHUtils {
} else {
logger.error("Task scheduling cannot be null at this point..");
}
+ ApplicationDeploymentDescription appDepDescription = jobExecutionContext.getApplicationContext().getApplicationDeploymentDescription();
+ List<String> moduleCmds = appDepDescription.getModuleLoadCmds();
+ if (moduleCmds != null) {
+ for (String moduleCmd : moduleCmds) {
+ jobDescriptor.addModuleLoadCommands(moduleCmd);
+ }
+ }
+ List<String> preJobCommands = appDepDescription.getPreJobCommands();
+ if (preJobCommands != null) {
+ for (String preJobCommand : preJobCommands) {
+ jobDescriptor.addPreJobCommand(parseCommand(preJobCommand, jobExecutionContext));
+ }
+ }
+
+ List<String> postJobCommands = appDepDescription.getPostJobCommands();
+ if (postJobCommands != null) {
+ for (String postJobCommand : postJobCommands) {
+ jobDescriptor.addPostJobCommand(parseCommand(postJobCommand, jobExecutionContext));
+ }
+ }
return jobDescriptor;
}
+ private static String parseCommand(String value, JobExecutionContext jobExecutionContext) {
+ String parsedValue = value.replaceAll("\\$workingDir", jobExecutionContext.getWorkingDir());
+ parsedValue = parsedValue.replaceAll("\\$inputDir", jobExecutionContext.getInputDir());
+ parsedValue = parsedValue.replaceAll("\\$outputDir", jobExecutionContext.getOutputDir());
+ return parsedValue;
+ }
/**
* This method can be used to set the Security Context if its not set and later use it in other places
* @param jobExecutionContext