You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2016/10/17 21:15:51 UTC
airavata git commit: Added template language to pre and post commands
Repository: airavata
Updated Branches:
refs/heads/develop e04785a4c -> 8bc2dcc4e
Added template language to pre and post commands
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/8bc2dcc4
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/8bc2dcc4
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/8bc2dcc4
Branch: refs/heads/develop
Commit: 8bc2dcc4eb3a45c09fa32903c38bd97e349b4599
Parents: e04785a
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Mon Oct 17 17:15:06 2016 -0400
Committer: Shameera Rathnayaka <sh...@gmail.com>
Committed: Mon Oct 17 17:15:41 2016 -0400
----------------------------------------------------------------------
.../apache/airavata/gfac/core/GFacUtils.java | 48 ++++++++++++--------
.../apache/airavata/gfac/core/GroovyMap.java | 9 +++-
.../org/apache/airavata/gfac/core/Script.java | 4 ++
.../gfac/core/context/ProcessContext.java | 20 +++++---
.../airavata/gfac/core/GFacUtilsTest.java | 13 ++++++
.../impl/task/DefaultJobSubmissionTask.java | 2 +-
.../gfac/impl/task/ForkJobSubmissionTask.java | 2 +-
.../gfac/impl/task/LocalJobSubmissionTask.java | 2 +-
8 files changed, 69 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index fe9fd01..b69764e 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -21,6 +21,7 @@ package org.apache.airavata.gfac.core;
import groovy.lang.Writable;
import groovy.text.GStringTemplateEngine;
+import groovy.text.SimpleTemplateEngine;
import groovy.text.TemplateEngine;
import org.apache.airavata.common.exception.ApplicationSettingsException;
import org.apache.airavata.common.utils.*;
@@ -64,9 +65,6 @@ import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
-import javax.xml.transform.*;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
import javax.xml.xpath.*;
import java.io.*;
import java.net.InetAddress;
@@ -416,8 +414,11 @@ public class GFacUtils {
public static String getZKGfacServersParentPath() {
return ZKPaths.makePath(ZkConstants.ZOOKEEPER_SERVERS_NODE, ZkConstants.ZOOKEEPER_GFAC_SERVER_NODE);
}
-
- public static GroovyMap creatGroovyMap(ProcessContext processContext, TaskContext taskContext)
+ public static GroovyMap crateGroovyMap(ProcessContext processContext)
+ throws ApplicationSettingsException, AppCatalogException, GFacException {
+ return createGroovyMap(processContext, null);
+ }
+ public static GroovyMap createGroovyMap(ProcessContext processContext, TaskContext taskContext)
throws GFacException, AppCatalogException, ApplicationSettingsException {
GroovyMap groovyMap = new GroovyMap();
@@ -430,6 +431,10 @@ public class GFacUtils {
groovyMap.add(Script.EXECUTABLE_PATH, processContext.getApplicationDeploymentDescription().getExecutablePath());
groovyMap.add(Script.STANDARD_OUT_FILE, processContext.getStdoutLocation());
groovyMap.add(Script.STANDARD_ERROR_FILE, processContext.getStderrLocation());
+ groovyMap.add(Script.SCRATCH_LOCATION, processContext.getScratchLocation());
+ groovyMap.add(Script.GATEWAY_ID, processContext.getGatewayId());
+ groovyMap.add(Script.GATEWAY_USER_NAME, processContext.getProcessModel().getUserName());
+ groovyMap.add(Script.APPLICATION_NAME, processContext.getApplicationInterfaceDescription().getApplicationName());
ComputeResourcePreference crp = getComputeResourcePreference(processContext);
if (isValid(crp.getAllocationProjectNumber())) {
@@ -448,14 +453,16 @@ public class GFacUtils {
groovyMap.add(Script.USER_NAME, processContext.getJobSubmissionRemoteCluster().getServerInfo().getUserName());
groovyMap.add(Script.SHELL_NAME, "/bin/bash");
// get walltime
- try {
- JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
- if (jobSubmissionTaskModel.getWallTime() > 0) {
- groovyMap.add(Script.MAX_WALL_TIME,
- GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+ if (taskContext != null) {
+ try {
+ JobSubmissionTaskModel jobSubmissionTaskModel = ((JobSubmissionTaskModel) taskContext.getSubTaskModel());
+ if (jobSubmissionTaskModel.getWallTime() > 0) {
+ groovyMap.add(Script.MAX_WALL_TIME,
+ GFacUtils.maxWallTimeCalculator(jobSubmissionTaskModel.getWallTime()));
+ }
+ } catch (TException e) {
+ log.error("Error while getting job submission sub task model", e);
}
- } catch (TException e) {
- log.error("Error while getting job submissiont sub task model", e);
}
// NOTE: Give precedence to data comes with experiment
@@ -522,7 +529,7 @@ public class GFacUtils {
if (preJobCommands != null) {
List<String> preJobCmdCollect = preJobCommands.stream()
.sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder())
- .map(map -> map.getCommand())
+ .map(map -> parseCommands(map.getCommand(), groovyMap))
.collect(Collectors.toList());
groovyMap.add(Script.PRE_JOB_COMMANDS, preJobCmdCollect);
}
@@ -531,7 +538,7 @@ public class GFacUtils {
if (postJobCommands != null) {
List<String> postJobCmdCollect = postJobCommands.stream()
.sorted((e1, e2) -> e1.getCommandOrder() - e2.getCommandOrder())
- .map(map -> map.getCommand())
+ .map(map -> parseCommands(map.getCommand(), groovyMap))
.collect(Collectors.toList());
groovyMap.add(Script.POST_JOB_COMMANDS, postJobCmdCollect);
}
@@ -703,11 +710,14 @@ public class GFacUtils {
return i;
}
- private static String parseCommand(String value, ProcessContext context) {
- String parsedValue = value.replaceAll("\\$workingDir", context.getWorkingDir());
- parsedValue = parsedValue.replaceAll("\\$inputDir", context.getInputDir());
- parsedValue = parsedValue.replaceAll("\\$outputDir", context.getOutputDir());
- return parsedValue;
+ static String parseCommands(String value, GroovyMap bindMap) {
+ TemplateEngine templateEngine = new GStringTemplateEngine();
+ try {
+ return templateEngine.createTemplate(value).make(bindMap).toString();
+ } catch (ClassNotFoundException | IOException e) {
+ throw new IllegalArgumentException("Error while parsing command " + value
+ + " , Invalid command or incomplete bind map");
+ }
}
public static ResourceJobManager getResourceJobManager(ProcessContext processContext) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
index d618d40..1abc878 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GroovyMap.java
@@ -45,7 +45,8 @@ public class GroovyMap extends HashMap<String, Object> {
public GroovyMap() {
super();
- addDefaultValues(); // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding
+ // to mitigate groovy exception groovy.lang.MissingPropertyException: No such property: <name> for class: groovy.lang.Binding
+ addDefaultValues();
}
public GroovyMap add(Script name, Object value){
@@ -77,6 +78,7 @@ public class GroovyMap extends HashMap<String, Object> {
.add(Script.RESERVATION, null)
.add(Script.EXPORTS, null)
.add(Script.MODULE_COMMANDS, null)
+ .add(Script.SCRATCH_LOCATION, null)
.add(Script.WORKING_DIR, null)
.add(Script.PRE_JOB_COMMANDS, null)
.add(Script.JOB_SUBMITTER_COMMAND, null)
@@ -88,7 +90,10 @@ public class GroovyMap extends HashMap<String, Object> {
.add(Script.CHASSIS_NAME, null)
.add(Script.INPUT_DIR, null)
.add(Script.OUTPUT_DIR, null)
- .add(Script.USER_NAME, null);
+ .add(Script.USER_NAME, null)
+ .add(Script.GATEWAY_ID, null)
+ .add(Script.GATEWAY_USER_NAME, null)
+ .add(Script.APPLICATION_NAME, null);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
index 1e3ecd9..3ac5797 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/Script.java
@@ -54,6 +54,7 @@ public enum Script {
RESERVATION("reservation"),
EXPORTS("exports"),
MODULE_COMMANDS("moduleCommands"),
+ SCRATCH_LOCATION("scratchLocation"),
WORKING_DIR("workingDirectory"),
PRE_JOB_COMMANDS("preJobCommands"),
JOB_SUBMITTER_COMMAND("jobSubmitterCommand"),
@@ -66,6 +67,9 @@ public enum Script {
INPUT_DIR("inputDir"),
OUTPUT_DIR("outputDir"),
USER_NAME("userName"),
+ GATEWAY_ID("gatewayId"),
+ GATEWAY_USER_NAME("gatewayUserName"),
+ APPLICATION_NAME("applicationName"),
;
String name;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
index 8a2cc4e..8079e2a 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/context/ProcessContext.java
@@ -64,6 +64,7 @@ public class ProcessContext {
private final String tokenId;
private ProcessModel processModel;
private String workingDir;
+ private String scratchLocation;
private String inputDir;
private String outputDir;
private String localWorkingDir;
@@ -166,13 +167,7 @@ public class ProcessContext {
if (processModel.getProcessResourceSchedule().getStaticWorkingDir() != null){
workingDir = processModel.getProcessResourceSchedule().getStaticWorkingDir();
}else {
- String scratchLocation = null;
- String overrideScratchLocation = processModel.getProcessResourceSchedule().getOverrideScratchLocation();
- if (overrideScratchLocation != null && !overrideScratchLocation.equals("")) {
- scratchLocation = overrideScratchLocation;
- } else {
- scratchLocation = computeResourcePreference.getScratchLocation();
- }
+ String scratchLocation = getScratchLocation();
workingDir = (scratchLocation.endsWith("/") ? scratchLocation + processId : scratchLocation + "/" +
processId);
}
@@ -180,6 +175,16 @@ public class ProcessContext {
return workingDir;
}
+ public String getScratchLocation() {
+ if (scratchLocation == null) {
+ scratchLocation = processModel.getProcessResourceSchedule().getOverrideScratchLocation();
+ if(scratchLocation == null || scratchLocation.isEmpty()){
+ scratchLocation = computeResourcePreference.getScratchLocation();
+ }
+ }
+ return scratchLocation;
+ }
+
public void setWorkingDir(String workingDir) {
this.workingDir = workingDir;
}
@@ -524,5 +529,6 @@ public class ProcessContext {
this.recoveryWithCancel = recoveryWithCancel;
}
+
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
index 3262f86..cf56780 100644
--- a/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
+++ b/modules/gfac/gfac-core/src/test/java/org/apache/airavata/gfac/core/GFacUtilsTest.java
@@ -46,4 +46,17 @@ public class GFacUtilsTest {
String shared = GFacUtils.getQoS(qos, "compute");
Assert.assertNull(shared);
}
+
+ @Test
+ public void parserCommandTest() throws Exception {
+ String command = "mkdir -p $scratchLocation/$gatewayId/$gatewayUserName/$applicationName";
+ GroovyMap groovyMap = new GroovyMap();
+ groovyMap.add(Script.SCRATCH_LOCATION, "/my/scratch");
+ groovyMap.add(Script.GATEWAY_ID, "seagrid");
+ groovyMap.add(Script.GATEWAY_USER_NAME, "John");
+ groovyMap.add(Script.APPLICATION_NAME, "gaussian");
+ String value = GFacUtils.parseCommands(command, groovyMap);
+ Assert.assertNotNull(value);
+ Assert.assertEquals("mkdir -p /my/scratch/seagrid/John/gaussian", value);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
index cabb77f..657de00 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/DefaultJobSubmissionTask.java
@@ -72,7 +72,7 @@ public class DefaultJobSubmissionTask implements JobSubmissionTask {
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
- GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext,taskContext);
+ GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
index e231cb9..1b43116 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/ForkJobSubmissionTask.java
@@ -64,7 +64,7 @@ public class ForkJobSubmissionTask implements JobSubmissionTask {
JobModel jobModel = processContext.getJobModel();
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
- GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext, taskContext);
+ GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext, taskContext);
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());
ResourceJobManager resourceJobManager = GFacUtils.getResourceJobManager(processContext);
JobManagerConfiguration jConfig = null;
http://git-wip-us.apache.org/repos/asf/airavata/blob/8bc2dcc4/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
index a8b8e39..7e989e4 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/LocalJobSubmissionTask.java
@@ -68,7 +68,7 @@ public class LocalJobSubmissionTask implements JobSubmissionTask{
jobModel.setTaskId(taskContext.getTaskId());
RemoteCluster remoteCluster = processContext.getJobSubmissionRemoteCluster();
- GroovyMap groovyMap = GFacUtils.creatGroovyMap(processContext,taskContext);
+ GroovyMap groovyMap = GFacUtils.createGroovyMap(processContext,taskContext);
String jobId = AiravataUtils.getId("JOB_ID_");
jobModel.setJobName(groovyMap.get(Script.JOB_NAME).toString());