You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/07/14 16:54:23 UTC
[2/3] airavata git commit: Registered job management commands with
resource. Fixed issues with Slurm output parser
Registered job management commands with resource. Fixed issues with Slurm output parser
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/706edf84
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/706edf84
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/706edf84
Branch: refs/heads/master
Commit: 706edf8465ced1280e0940f5b73fa027958cd369
Parents: cce7be7
Author: Shameera Rathanyaka <sh...@gmail.com>
Authored: Tue Jul 14 10:51:57 2015 -0400
Committer: Shameera Rathanyaka <sh...@gmail.com>
Committed: Tue Jul 14 10:51:57 2015 -0400
----------------------------------------------------------------------
.../tools/RegisterSampleApplications.java | 20 +++++--
.../apache/airavata/gfac/core/GFacUtils.java | 12 +---
.../airavata/gfac/impl/HPCRemoteCluster.java | 1 -
.../apache/airavata/gfac/impl/job/JobUtil.java | 59 ++++++++++++++++++++
.../gfac/impl/job/SlurmOutputParser.java | 21 +++----
5 files changed, 84 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
----------------------------------------------------------------------
diff --git a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
index 67a84cb..7294bbb 100644
--- a/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
+++ b/airavata-api/airavata-client-sdks/java-client-samples/src/main/java/org/apache/airavata/client/tools/RegisterSampleApplications.java
@@ -1372,11 +1372,21 @@ public class RegisterSampleApplications {
ResourceJobManager resourceJobManager = RegisterSampleApplicationsUtils.
createResourceJobManager(resourceJobManagerType, monitoringEndPoint, jobMangerBinPath, null);
- if (jobManagerCommand != null) {
- Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>();
- jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, jobManagerCommand);
- resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap);
- }
+ Map<JobManagerCommand, String> jobManagerCommandStringMap = new HashMap<JobManagerCommand, String>();
+ if (resourceJobManagerType == ResourceJobManagerType.SLURM) {
+ jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, "sbatch");
+ jobManagerCommandStringMap.put(JobManagerCommand.JOB_MONITORING, "squeue");
+ jobManagerCommandStringMap.put(JobManagerCommand.DELETION, "scancel");
+ resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap);
+ } else if (resourceJobManagerType == ResourceJobManagerType.PBS) {
+ jobManagerCommandStringMap.put(JobManagerCommand.SUBMISSION, "qsub");
+ jobManagerCommandStringMap.put(JobManagerCommand.JOB_MONITORING, "qstat");
+ jobManagerCommandStringMap.put(JobManagerCommand.DELETION, "qdel");
+ resourceJobManager.setJobManagerCommands(jobManagerCommandStringMap);
+ }
+ // TODO - set job manage commands for UGE and LSF type compute resources.
+
+ // TODO - set parallelism command
SSHJobSubmission sshJobSubmission = new SSHJobSubmission();
sshJobSubmission.setResourceJobManager(resourceJobManager);
http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
index 1b58730..b1321d6 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java
@@ -950,17 +950,7 @@ public class GFacUtils {
ApplicationParallelismType parallelism = appDepDescription.getParallelism();
if (parallelism != null) {
if (parallelism == ApplicationParallelismType.MPI || parallelism == ApplicationParallelismType.OPENMP || parallelism == ApplicationParallelismType.OPENMP_MPI) {
- if (resourceJobManager != null) {
- Map<JobManagerCommand, String> jobManagerCommands = resourceJobManager.getJobManagerCommands();
- if (jobManagerCommands != null && !jobManagerCommands.isEmpty()) {
- for (JobManagerCommand command : jobManagerCommands.keySet()) {
- if (command == JobManagerCommand.SUBMISSION) {
- String commandVal = jobManagerCommands.get(command);
- jobDescriptor.setJobSubmitter(commandVal);
- }
- }
- }
- }
+ jobDescriptor.setJobSubmitter("ibrun");
}
}
return jobDescriptor;
http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
index 69cf03e..fa73898 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/HPCRemoteCluster.java
@@ -33,7 +33,6 @@ import org.apache.airavata.gfac.core.authentication.SSHKeyAuthentication;
import org.apache.airavata.gfac.core.cluster.AbstractRemoteCluster;
import org.apache.airavata.gfac.core.cluster.CommandInfo;
import org.apache.airavata.gfac.core.cluster.CommandOutput;
-import org.apache.airavata.gfac.core.cluster.OutputParser;
import org.apache.airavata.gfac.core.cluster.RawCommandInfo;
import org.apache.airavata.gfac.core.cluster.ServerInfo;
import org.apache.airavata.model.status.JobStatus;
http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java
new file mode 100644
index 0000000..0cb49a9
--- /dev/null
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/JobUtil.java
@@ -0,0 +1,59 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.airavata.gfac.impl.job;
+
+import org.apache.airavata.model.status.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JobUtil {
+ private static final Logger log = LoggerFactory.getLogger(JobUtil.class);
+
+ public static JobState getJobState(String status) {
+ log.info("parsing the job status returned : " + status);
+ if (status != null) {
+ if ("C".equals(status) || "CD".equals(status) || "E".equals(status) || "CG".equals(status) || "DONE".equals(status)) {
+ return JobState.COMPLETE;
+// } else if ("H".equals(status) || "h".equals(status)) {
+// return JobState.HELD;
+ } else if ("Q".equals(status) || "qw".equals(status) || "PEND".equals(status)) {
+ return JobState.QUEUED;
+ } else if ("R".equals(status) || "CF".equals(status) || "r".equals(status) || "RUN".equals(status)) {
+ return JobState.ACTIVE;
+// } else if ("T".equals(status)) {
+// return JobState.HELD;
+ } else if ("W".equals(status) || "PD".equals(status)) {
+ return JobState.QUEUED;
+ } else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) {
+ return JobState.SUSPENDED;
+ } else if ("CA".equals(status)) {
+ return JobState.CANCELED;
+ } else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) {
+ return JobState.FAILED;
+ } else if ("PR".equals(status) || "Er".equals(status)) {
+ return JobState.FAILED;
+ } else if ("U".equals(status) || ("UNKWN".equals(status))) {
+ return JobState.UNKNOWN;
+ }
+ }
+ return JobState.UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/706edf84/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java
index 1974843..fecb5e7 100644
--- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java
+++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/job/SlurmOutputParser.java
@@ -38,6 +38,7 @@ public class SlurmOutputParser implements OutputParser {
private static final Logger log = LoggerFactory.getLogger(SlurmOutputParser.class);
public static final int JOB_NAME_OUTPUT_LENGTH = 8;
public static final String STATUS = "status";
+ public static final String JOBID = "jobId";
public void parseSingleJob(JobDescriptor descriptor, String rawOutput) throws SSHApiException {
log.info(rawOutput);
@@ -105,17 +106,13 @@ public class SlurmOutputParser implements OutputParser {
* @return
*/
public String parseJobSubmission(String rawOutput) throws SSHApiException {
- // FIXME : use regex to match correct jobId;
- log.info(rawOutput);
- String[] info = rawOutput.split("\n");
- for (String anInfo : info) {
- if (anInfo.contains("Submitted batch job")) {
- String[] split = anInfo.split("Submitted batch job");
- return split[1].trim();
- }
- }
- return "";
-// throw new SSHApiException(rawOutput); //todo//To change body of implemented methods use File | Settings | File Templates.
+ log.info(rawOutput);
+ Pattern pattern = Pattern.compile("Submitted batch job (?<" + JOBID + ">[^\\s]*)");
+ Matcher matcher = pattern.matcher(rawOutput);
+ if (matcher.find()) {
+ return matcher.group(JOBID);
+ }
+ return "";
}
public JobStatus parseJobStatus(String jobID, String rawOutput) throws SSHApiException {
@@ -123,7 +120,7 @@ public class SlurmOutputParser implements OutputParser {
Pattern pattern = Pattern.compile(jobID + "(?=\\s+\\S+\\s+\\S+\\s+\\S+\\s+(?<" + STATUS + ">\\w+))");
Matcher matcher = pattern.matcher(rawOutput);
if (matcher.find()) {
- return new JobStatus(JobState.valueOf(matcher.group(STATUS)));
+ return new JobStatus(JobUtil.getJobState(matcher.group(STATUS)));
}
return null;
}