You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/07/08 22:29:52 UTC
airavata git commit: Fixed yaml configuration parsing issues
Repository: airavata
Updated Branches:
refs/heads/master c4c03f3fb -> eaf941ae4
Fixed yaml configuration parsing issues
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/eaf941ae
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/eaf941ae
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/eaf941ae
Branch: refs/heads/master
Commit: eaf941ae4112da96a5a15ed2dfa076de4002ea2d
Parents: c4c03f3
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Wed Jul 8 16:29:44 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Wed Jul 8 16:29:44 2015 -0400
----------------------------------------------------------------------
.../server/src/main/resources/gfac-config.yaml | 106 ++++++++++---------
.../core/config/DataTransferTaskConfig.java | 17 +++
.../gfac/core/config/GFacYamlConfigruation.java | 35 ++++--
.../core/config/JobSubmitterTaskConfig.java | 16 +++
.../org/apache/airavata/gfac/impl/Factory.java | 35 ++++--
5 files changed, 144 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/configuration/server/src/main/resources/gfac-config.yaml
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/gfac-config.yaml b/modules/configuration/server/src/main/resources/gfac-config.yaml
index 5530547..46ece9c 100644
--- a/modules/configuration/server/src/main/resources/gfac-config.yaml
+++ b/modules/configuration/server/src/main/resources/gfac-config.yaml
@@ -18,62 +18,72 @@
#
################################################################
-config:
- jobSubmitters:
- - submissionProtocol: SSH
- taskClass: org.apache.airavata.task.adapters.SSHJobSubmissionTask
- properties:
- - password: pwd123
- - passPhrase: test
- - privateKey: key
- - publicKey: pubkey
+jobSubmitters:
+ - submissionProtocol: SSH
+ taskClass: org.apache.airavata.gfac.impl.task.SSHJobSubmissionTask
+ properties:
+ - password: pwd123
+ passPhrase: test
+ privateKey: key
+ publicKey: pubk
+ # - submissionProtocol: GSISSH
+ # taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
- - submissionProtocol: GSISSH
- taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
+ - submissionProtocol: LOCAL
+ taskClass: org.apache.airavata.gfac.impl.task.LocalJobSubmissionTask
- commonTasks:
- - type: SETUP
- taskClass: org.apache.airavata.task.common.SetupTask
+commonTasks:
+ - type: SETUP
+ taskClass: org.apache.airavata.task.common.SetupTask
- - type: CLEANUP
- taskClass: org.apache.airavata.task.common.CleanupTask
+ - type: CLEANUP
+ taskClass: org.apache.airavata.task.common.CleanupTask
- fileTransferTasks:
- - transferProtocol: SCP
- taskClass: org.apache.airavata.task.adapters.SCPFileTransferTask
- properties:
- - password: pwd123
- - passPhrase: test
- - privateKey: key
- - publicKey: pubkey
+fileTransferTasks:
+ - transferProtocol: SCP
+ taskClass: org.apache.airavata.gfac.impl.task.SCPInputDataStageTask
+ properties:
+ - password: pwd123
+ passPhrase: test
+ privateKey: key
+ publicKey: pubkey
- - transferProtocol: HTTP
- taskClass: org.apache.airavata.task.adapters.HTTPFileTransferTask
+ #- transferProtocol: SFTP
+ # taskClass: org.apache.airavata.task.adapters.SFTPFileTransferTask
- - transferProtocol: GRIDFTP
- taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
+ #- transferProtocol: GRIDFTP
+ # taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
- resources:
- - jobManagerType: PBS
- commandOutputParser: org.apache.airavata.commmand.PBSOutputParser
- emailParser: org.apache.airavata.email.parser.PBSEmailParser
- resourceEmailAddresses:
- - pbsconsult@sdsc.edu # gordon
- - adm@trident.bigred2.uits.iu.edu # Bigred2
- - root <ad...@trident.bigred2.uits.iu.edu> # Bigred2
- - root <ad...@scyld.localdomain> # alamo
+ #- transferProtocol: LOCAL
+ # taskClass: org.apache.airavata.task.adapters.LocalFileTransferTask
- - jobManagerType: SLURM
- commandOutputParser: org.apache.airavata.commmand.SLURMOutputParser
- emailParser: org.apache.airavata.email.parser.SLURMEmailParser
- resourceEmailAddresses:
- - SDSC Admin <sl...@comet-fe3.sdsc.edu> # comet
- - slurm@batch1.stampede.tacc.utexas.edu # stampede
+resources:
+ - jobManagerType: PBS
+ commandOutputParser: org.apache.airavata.gfac.impl.job.PBSOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser
+ resourceEmailAddresses:
+ - pbsconsult@sdsc.edu # gordon
+ - adm@trident.bigred2.uits.iu.edu # Bigred2
+ - root <ad...@trident.bigred2.uits.iu.edu> # Bigred2
+ - root <ad...@scyld.localdomain> # alamo
- - jobManagerType: UGE
- commandOutputParser: org.apache.airavata.commmand.UGEOutputParser
- emailParser: org.apache.airavata.email.parser.UGEEmailParser
- resourceEmailAddresses:
- - ls4.tacc.utexas.edu # contain Lonestar
+ - jobManagerType: SLURM
+ commandOutputParser: org.apache.airavata.gfac.impl.job.SlurmOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser
+ resourceEmailAddresses:
+ - SDSC Admin <sl...@comet-fe3.sdsc.edu> # comet
+ - slurm@batch1.stampede.tacc.utexas.edu # stampede
+
+ - jobManagerType: UGE
+ commandOutputParser: org.apache.airavata.gfac.impl.job.UGEOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.UGEEmailParser
+ resourceEmailAddresses:
+ - ls4.tacc.utexas.edu # contain Lonestar
+
+ - jobManagerType: LSF
+ commandOutputParser: org.apache.airavata.gfac.impl.job.LSFOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.LSFEmailParser
+ resourceEmailAddresses:
+ - iu.xsede.edu # test resource mail address
http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
index a4817fb..76728f0 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/DataTransferTaskConfig.java
@@ -22,9 +22,14 @@ package org.apache.airavata.gfac.core.config;
import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
+import java.util.HashMap;
+import java.util.Map;
+
public class DataTransferTaskConfig {
private DataMovementProtocol transferProtocol;
private String taskClass;
+ private Map<String,String> properties = new HashMap<>();
+
public DataMovementProtocol getTransferProtocol() {
return transferProtocol;
@@ -41,4 +46,16 @@ public class DataTransferTaskConfig {
public void setTaskClass(String taskClass) {
this.taskClass = taskClass;
}
+
+ public void addProperty(String key, String value) {
+ properties.put(key, value);
+ }
+
+ public void addProperties(Map<String, String> propMap) {
+ propMap.forEach(properties::put);
+ }
+
+ public Map<String,String> getProperties(){
+ return properties;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
index 5dbad87..fa8ce32 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/GFacYamlConfigruation.java
@@ -46,6 +46,7 @@ public class GFacYamlConfigruation {
private static final String COMMAND_OUTPUT_PARSER = "commandOutputParser";
private static final String EMAIL_PARSER = "emailParser";
private static final String RESOURCE_EMAIL_ADDRESS = "resourceEmailAddress";
+ private static final String PROPERTIES = "properties";
private List<JobSubmitterTaskConfig> jobSubmitters = new ArrayList<>();
private List<DataTransferTaskConfig> fileTransferTasks = new ArrayList<>();
@@ -71,26 +72,44 @@ public class GFacYamlConfigruation {
if (load instanceof Map) {
Map<String, Object> loadMap = (Map<String, Object>) load;
String identifier;
- List<Map<String, String>> jobSubYamls = (List<Map<String, String>>) loadMap.get(JOB_SUBMITTERS);
+ List<Map<String,Object >> jobSubYamls = (List<Map<String, Object>>) loadMap.get(JOB_SUBMITTERS);
JobSubmitterTaskConfig jobSubmitterTaskConfig;
if (jobSubYamls != null) {
- for (Map<String, String> jobSub : jobSubYamls) {
+ for (Map<String, Object> jobSub : jobSubYamls) {
jobSubmitterTaskConfig = new JobSubmitterTaskConfig();
- identifier = jobSub.get (SUBMISSIO_PROTOCOL);
+ identifier = ((String) jobSub.get(SUBMISSIO_PROTOCOL));
jobSubmitterTaskConfig.setSubmissionProtocol(JobSubmissionProtocol.valueOf(identifier));
- jobSubmitterTaskConfig.setTaskClass(jobSub.get(TASK_CLASS));
+ jobSubmitterTaskConfig.setTaskClass(((String) jobSub.get(TASK_CLASS)));
+ Object propertiesObj = jobSub.get(PROPERTIES);
+ List propertiesList;
+ if (propertiesObj instanceof List) {
+ propertiesList = ((List) propertiesObj);
+ if (propertiesList.size() > 0) {
+ Map<String, String> props = (Map<String, String>) propertiesList.get(0);
+ jobSubmitterTaskConfig.addProperties(props);
+ }
+ }
jobSubmitters.add(jobSubmitterTaskConfig);
}
}
- List<Map<String, String>> fileTransYamls = (List<Map<String, String>>) loadMap.get(FILE_TRANSFER_TASKS);
+ List<Map<String, Object>> fileTransYamls = (List<Map<String, Object>>) loadMap.get(FILE_TRANSFER_TASKS);
DataTransferTaskConfig dataTransferTaskConfig;
if (fileTransYamls != null) {
- for (Map<String, String> fileTransConfig : fileTransYamls) {
+ for (Map<String, Object> fileTransConfig : fileTransYamls) {
dataTransferTaskConfig = new DataTransferTaskConfig();
- identifier = fileTransConfig.get (TRANSFER_PROTOCOL);
+ identifier = ((String) fileTransConfig.get(TRANSFER_PROTOCOL));
dataTransferTaskConfig.setTransferProtocol(DataMovementProtocol.valueOf(identifier));
- dataTransferTaskConfig.setTaskClass(fileTransConfig.get(TASK_CLASS));
+ dataTransferTaskConfig.setTaskClass(((String) fileTransConfig.get(TASK_CLASS)));
+ Object propertiesObj = fileTransConfig.get(PROPERTIES);
+ List propertiesList;
+ if (propertiesObj instanceof List) {
+ propertiesList = (List) propertiesObj;
+ if (propertiesList.size() > 0) {
+ Map<String, String> props = (Map<String, String>) propertiesList.get(0);
+ dataTransferTaskConfig.addProperties(props);
+ }
+ }
fileTransferTasks.add(dataTransferTaskConfig);
}
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
index daea282..6377f88 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/config/JobSubmitterTaskConfig.java
@@ -22,9 +22,13 @@ package org.apache.airavata.gfac.core.config;
import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+import java.util.HashMap;
+import java.util.Map;
+
public class JobSubmitterTaskConfig {
private JobSubmissionProtocol submissionProtocol;
private String taskClass;
+ private Map<String,String> properties = new HashMap<>();
public JobSubmissionProtocol getSubmissionProtocol() {
return submissionProtocol;
@@ -41,4 +45,16 @@ public class JobSubmitterTaskConfig {
public void setTaskClass(String taskClass) {
this.taskClass = taskClass;
}
+
+ public void addProperty(String key, String value) {
+ properties.put(key, value);
+ }
+
+ public void addProperties(Map<String, String> propMap) {
+ propMap.forEach(properties::put);
+ }
+
+ public Map<String,String> getProperties(){
+ return properties;
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/eaf941ae/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
index 2b629bb..fbb8f2f 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/Factory.java
@@ -40,6 +40,7 @@ import org.apache.airavata.gfac.core.monitor.JobMonitor;
import org.apache.airavata.gfac.core.scheduler.HostScheduler;
import org.apache.airavata.gfac.core.task.JobSubmissionTask;
import org.apache.airavata.gfac.core.task.Task;
+import org.apache.airavata.gfac.core.task.TaskException;
import org.apache.airavata.gfac.impl.job.LSFJobConfiguration;
import org.apache.airavata.gfac.impl.job.LSFOutputParser;
import org.apache.airavata.gfac.impl.job.PBSJobConfiguration;
@@ -67,6 +68,8 @@ import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
@@ -259,18 +262,32 @@ public abstract class Factory {
private static void loadConfiguration() throws GFacException {
GFacYamlConfigruation config = new GFacYamlConfigruation();
- for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
- jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), null);
- }
+ try {
+ for (JobSubmitterTaskConfig jobSubmitterTaskConfig : config.getJobSbumitters()) {
+ String taskClass = jobSubmitterTaskConfig.getTaskClass();
+ Class<?> aClass = Class.forName(taskClass);
+ Constructor<?> constructor = aClass.getConstructor();
+ JobSubmissionTask task = (JobSubmissionTask) constructor.newInstance();
+ task.init(jobSubmitterTaskConfig.getProperties());
+ jobSubmissionTask.put(jobSubmitterTaskConfig.getSubmissionProtocol(), task);
+ }
- for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
- dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), null);
- }
+ for (DataTransferTaskConfig dataTransferTaskConfig : config.getFileTransferTasks()) {
+ String taskClass = dataTransferTaskConfig.getTaskClass();
+ Class<?> aClass = Class.forName(taskClass);
+ Constructor<?> constructor = aClass.getConstructor();
+ Task task = (Task) constructor.newInstance();
+ task.init(dataTransferTaskConfig.getProperties());
+ dataMovementTask.put(dataTransferTaskConfig.getTransferProtocol(), task);
+ }
- for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
- resources.put(resourceConfig.getJobManagerType(), resourceConfig);
+ for (ResourceConfig resourceConfig : config.getResourceConfiguration()) {
+ resources.put(resourceConfig.getJobManagerType(), resourceConfig);
+ }
+ readConfig = true;
+ } catch (Exception e) {
+ throw new GFacException("Gfac config issue", e);
}
- readConfig = true;
}