You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2022/01/16 21:29:58 UTC
[airavata] branch develop updated: Relevant changes from htcondor branch
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new 887e1fc Relevant changes from htcondor branch
887e1fc is described below
commit 887e1fce400230594acc31c052a43fec48d9b414
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Jan 13 20:45:20 2022 -0500
Relevant changes from htcondor branch
---
.../computeresource/ResourceJobManagerType.java | 5 +-
.../templates/email-monitor/email-config.yaml.j2 | 4 +
.../task/submission/config/GroovyMapBuilder.java | 57 ++++++-
.../impl/task/submission/config/GroovyMapData.java | 12 ++
.../impl/task/submission/config/JobFactory.java | 5 +
.../config/app/HTCondorJobConfiguration.java | 117 ++++++++++++++
.../impl/task/submission/config/app/JobUtil.java | 4 +-
.../config/app/parser/HTCondorOutputParser.java | 169 +++++++++++++++++++++
.../src/main/resources/HTCONDOR_Groovy.template | 24 +++
.../resources/email-monitor/conf/email-config.yaml | 6 +-
.../src/main/resources/email-config.yaml | 6 +-
.../monitor/email/parser/HTCondorEmailParser.java | 129 ++++++++++++++++
.../src/main/resources/email-config.yaml | 6 +-
.../compute_resource_model.thrift | 3 +-
14 files changed, 539 insertions(+), 8 deletions(-)
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
index e1a336c..f7b73a2 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
@@ -54,7 +54,8 @@ public enum ResourceJobManagerType implements org.apache.thrift.TEnum {
LSF(3),
UGE(4),
CLOUD(5),
- AIRAVATA_CUSTOM(6);
+ AIRAVATA_CUSTOM(6),
+ HTCONDOR(7);
private final int value;
@@ -89,6 +90,8 @@ public enum ResourceJobManagerType implements org.apache.thrift.TEnum {
return CLOUD;
case 6:
return AIRAVATA_CUSTOM;
+ case 7:
+ return HTCONDOR;
default:
return null;
}
diff --git a/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2 b/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
index 4f9ce90..9b4cf55 100644
--- a/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
+++ b/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
@@ -93,3 +93,7 @@ config:
resourceEmailAddresses:
- iu.xsede.edu # test resource mail address
- tcs.tulsahpc.org #Tandy
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
index dfca629..ed5dd45 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
@@ -76,7 +76,10 @@ public class GroovyMapBuilder {
mapData.setQueueSpecificMacros(taskContext.getQueueSpecificMacros());
mapData.setAccountString(taskContext.getAllocationProjectNumber());
mapData.setReservation(taskContext.getReservation());
- mapData.setJobName("A" + String.valueOf(generateJobName()));
+ if(this.taskContext.getResourceJobManager().getResourceJobManagerType() == ResourceJobManagerType.HTCONDOR)
+ mapData.setJobName("HTCondor");
+ else
+ mapData.setJobName("A" + String.valueOf(generateJobName()));
mapData.setWorkingDirectory(taskContext.getWorkingDir());
mapData.setTaskId(taskContext.getTaskId());
mapData.setExperimentDataDir(taskContext.getProcessModel().getExperimentDataDir());
@@ -94,6 +97,9 @@ public class GroovyMapBuilder {
inputValues.addAll(getProcessOutputValues(taskContext.getProcessModel().getProcessOutputs(), true));
mapData.setInputs(inputValues);
+ List<String> inputFiles = getProcessInputFiles(taskContext.getProcessModel().getProcessInputs(), false);
+ mapData.setInputFiles(inputFiles);
+
List<String> inputValuesAll = getProcessInputValues(taskContext.getProcessModel().getProcessInputs(), false);
inputValuesAll.addAll(getProcessOutputValues(taskContext.getProcessModel().getProcessOutputs(), false));
mapData.setInputsAll(inputValuesAll);
@@ -305,6 +311,55 @@ public class GroovyMapBuilder {
return inputValues;
}
+ private static List<String> getProcessInputFiles(List<InputDataObjectType> processInputs, boolean commandLineOnly) {
+ List<String> inputFiles = new ArrayList<String>();
+ if (processInputs != null) {
+
+ // sort the inputs first and then build the command ListR
+ Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() {
+ @Override
+ public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) {
+ return inputDataObjectType.getInputOrder() - t1.getInputOrder();
+ }
+ };
+ Set<InputDataObjectType> sortedInputSet = new TreeSet<InputDataObjectType>(inputOrderComparator);
+ for (InputDataObjectType input : processInputs) {
+ sortedInputSet.add(input);
+ }
+ for (InputDataObjectType inputDataObjectType : sortedInputSet) {
+ if (!inputDataObjectType.isIsRequired() &&
+ (inputDataObjectType.getValue() == null || "".equals(inputDataObjectType.getValue()))) {
+ // For URI/ Collection non required inputs, if the value is empty, ignore it. Fix for airavata-3276
+ continue;
+ }
+
+ if (inputDataObjectType.getValue() != null
+ && !inputDataObjectType.getValue().equals("")) {
+ if (inputDataObjectType.getType() == DataType.URI) {
+ if (inputDataObjectType.getOverrideFilename() != null) {
+ inputFiles.add(inputDataObjectType.getOverrideFilename());
+ } else {
+ // set only the relative path
+ String filePath = inputDataObjectType.getValue();
+ filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length());
+ inputFiles.add(filePath);
+ }
+ } else if (inputDataObjectType.getType() == DataType.URI_COLLECTION) {
+ String filePaths = inputDataObjectType.getValue();
+ String[] paths = filePaths.split(MULTIPLE_INPUTS_SPLITTER);
+
+ for (int i = 0; i < paths.length; i++) {
+ paths[i] = paths[i].substring(paths[i].lastIndexOf(File.separatorChar) + 1);
+ }
+
+ inputFiles.add(String.join(" ", paths));
+ }
+ }
+ }
+ }
+ return inputFiles;
+ }
+
private static List<String> getProcessOutputValues(List<OutputDataObjectType> processOutputs, boolean commandLineOnly) {
List<String> inputValues = new ArrayList<>();
if (processOutputs != null) {
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
index 45c9e9d..24e02b4 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
@@ -89,6 +89,9 @@ public class GroovyMapData {
@ScriptTag(name = "inputs")
private List<String> inputs;
+ @ScriptTag(name = "inputFiles")
+ private List<String> inputFiles;
+
@ScriptTag(name = "inputsAll")
private List<String> inputsAll;
@@ -317,6 +320,15 @@ public class GroovyMapData {
return this;
}
+ public List<String> getInputFiles() {
+ return inputFiles;
+ }
+
+ public GroovyMapData setInputFiles(List<String> inputFiles) {
+ this.inputFiles = inputFiles;
+ return this;
+ }
+
public List<String> getInputsAll() {
return inputsAll;
}
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
index e9099e4..5deb964 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
@@ -45,6 +45,8 @@ public class JobFactory {
return "LSF_Groovy.template";
case CLOUD:
return "CLOUD_Groovy.template";
+ case HTCONDOR:
+ return "HTCONDOR_Groovy.template";
default:
return null;
}
@@ -119,6 +121,9 @@ public class JobFactory {
case FORK:
return new ForkJobConfiguration(templateFileName, ".sh", resourceJobManager.getJobManagerBinPath(),
resourceJobManager.getJobManagerCommands(), new ForkOutputParser());
+ case HTCONDOR:
+ return new HTCondorJobConfiguration(templateFileName, ".submit", resourceJobManager
+ .getJobManagerBinPath(), resourceJobManager.getJobManagerCommands(), new HTCondorOutputParser());
// We don't have a job configuration manager for CLOUD type
default:
throw new Exception("Could not find a job manager configuration for job manager type "
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java
new file mode 100644
index 0000000..034b887
--- /dev/null
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java
@@ -0,0 +1,117 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.helix.impl.task.submission.config.app;
+
+import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration;
+import org.apache.airavata.helix.impl.task.submission.config.OutputParser;
+import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+import java.util.Map;
+
+public class HTCondorJobConfiguration implements JobManagerConfiguration {
+ private final Map<JobManagerCommand, String> jMCommands;
+ private String jobDescriptionTemplateName;
+ private String scriptExtension;
+ private String installedPath;
+ private OutputParser parser;
+
+ public HTCondorJobConfiguration(String jobDescriptionTemplateName,
+ String scriptExtension, String installedPath, Map<JobManagerCommand, String>
+ jobManagerCommands, OutputParser parser) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ this.scriptExtension = scriptExtension;
+ this.parser = parser;
+ installedPath = installedPath.trim();
+ if (installedPath.endsWith("/")) {
+ this.installedPath = installedPath;
+ } else {
+ this.installedPath = installedPath + "/";
+ }
+ this.jMCommands = jobManagerCommands;
+ }
+
+ public RawCommandInfo getCancelCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.DELETION).trim() + " " + jobID);
+ }
+
+ public String getJobDescriptionTemplateName() {
+ return jobDescriptionTemplateName;
+ }
+
+ public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ }
+
+ public RawCommandInfo getMonitorCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " " + jobID + " -nobatch");
+ }
+
+ public String getScriptExtension() {
+ return scriptExtension;
+ }
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
+ workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
+ }
+
+ public String getInstalledPath() {
+ return installedPath;
+ }
+
+ public void setInstalledPath(String installedPath) {
+ this.installedPath = installedPath;
+ }
+
+ public OutputParser getParser() {
+ return parser;
+ }
+
+ public void setParser(OutputParser parser) {
+ this.parser = parser;
+ }
+
+ public RawCommandInfo getUserBasedMonitorCommand(String userName) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " -submitter " + userName);
+ }
+
+ @Override
+ public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " " + jobName + " -submitter " + userName);
+ }
+
+ @Override
+ public String getBaseCancelCommand() {
+ return jMCommands.get(JobManagerCommand.DELETION).trim();
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return jMCommands.get(JobManagerCommand.JOB_MONITORING).trim();
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return jMCommands.get(JobManagerCommand.SUBMISSION).trim();
+ }
+}
\ No newline at end of file
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
index 650cf53..0ee5c7b 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
@@ -39,11 +39,11 @@ public class JobUtil {
return JobState.ACTIVE;
// } else if ("T".equals(status)) {
// return JobState.HELD;
- } else if ("W".equals(status) || "PD".equals(status)) {
+ } else if ("W".equals(status) || "PD".equals(status) || "I".equals(status)) {
return JobState.QUEUED;
} else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) {
return JobState.SUSPENDED;
- } else if ("CA".equals(status)) {
+ } else if ("CA".equals(status) || "X".equals(status)) {
return JobState.CANCELED;
} else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) {
return JobState.FAILED;
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java
new file mode 100644
index 0000000..5a6dfec
--- /dev/null
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java
@@ -0,0 +1,169 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.helix.impl.task.submission.config.app.parser;
+
+import org.apache.airavata.helix.impl.task.submission.config.OutputParser;
+import org.apache.airavata.helix.impl.task.submission.config.app.JobUtil;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.registry.core.utils.DBConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HTCondorOutputParser implements OutputParser {
+ private static final Logger log = LoggerFactory.getLogger(HTCondorOutputParser.class);
+ public static final int JOB_NAME_OUTPUT_LENGTH = 8;
+ public static final String STATUS = "status";
+ public static final String JOBID = "jobId";
+
+ /**
+ * This can be used to parseSingleJob the result of a job submission to get the JobID
+ * @param rawOutput
+ * @return the job id as a String, or null if no job id found
+ */
+ public String parseJobSubmission(String rawOutput) throws Exception {
+ log.info(rawOutput);
+ if (rawOutput != null && !rawOutput.isEmpty()) {
+
+ Pattern pattern = Pattern.compile("\\d+ job\\(s\\) submitted to cluster (?<" + JOBID + ">\\d+)");
+ Matcher matcher = pattern.matcher(rawOutput);
+
+ if (matcher.find()) {
+ return matcher.group(JOBID);
+ }
+ }
+ return "";
+ }
+
+
+ /**
+ * Parse output return by job submission task and identify jobSubmission failures.
+ * @param rawOutput
+ * @return true if job submission has been failed, false otherwise.
+ */
+ public boolean isJobSubmissionFailed(String rawOutput) {
+ Pattern pattern = Pattern.compile("failed");
+ Matcher matcher = pattern.matcher(rawOutput);
+ return matcher.find();
+ }
+
+
+ /**
+ * This can be used to get the job status from the output
+ * @param jobID
+ * @param rawOutput
+ */
+ public JobStatus parseJobStatus(String jobID, String rawOutput) throws Exception {
+ log.info(rawOutput);
+ if (rawOutput != null && !rawOutput.isEmpty()) {
+ Pattern pattern = Pattern.compile("\\s+" + jobID + ".\\d+(?=\\s+\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+(?<" + STATUS + ">\\w+))");
+ Matcher matcher = pattern.matcher(rawOutput);
+
+ if (matcher.find()) {
+ if (matcher.group(STATUS).equals("E")) {
+ log.info("parsing the job status returned : " + STATUS);
+ return new JobStatus(JobState.FAILED);
+ }
+ return new JobStatus(JobUtil.getJobState(matcher.group(STATUS)));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This can be used to parseSingleJob a big output and get multipleJob statuses
+ * @param statusMap list of status map will return and key will be the job ID
+ * @param rawOutput
+ */
+ public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws Exception {
+ log.debug(rawOutput);
+
+ String[] info = rawOutput.split("\n");
+ String lastString = info[info.length - 1];
+
+ if (lastString.contains("ID") || lastString.contains("OWNER")) {
+ log.info("There are no jobs with this username ... ");
+ return;
+ }
+
+ for (String jobID : statusMap.keySet()) {
+ String jobId = jobID.split(",")[0];
+ String ownerName = jobID.split(",")[1];
+ boolean found = false;
+
+ for (int i = 1; i < info.length; i++) {
+ if (info[i].contains(ownerName)) {
+ // now starts processing this line
+ log.info(info[i]);
+ String correctLine = info[i];
+ String[] columns = correctLine.split(" ");
+ List<String> columnList = new ArrayList<String>();
+ for (String s : columns) {
+ if (!"".equals(s)) {
+ columnList.add(s);
+ }
+ }
+ if ("E".equals(columnList.get(4))) {
+ columnList.set(4, "Er");
+ }
+ try {
+ statusMap.put(jobID, new JobStatus(JobState.valueOf(columnList.get(4))));
+ } catch (IndexOutOfBoundsException e) {
+ statusMap.put(jobID, new JobStatus(JobState.valueOf("U")));
+ }
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ log.error("Couldn't find the status of the Job with Owner: " + ownerName + "Job Id: " + jobId);
+ }
+ }
+ }
+
+ @Override
+ public String parseJobId(String jobName, String rawOutput) throws Exception {
+ String regJobId = "jobId";
+ if (jobName == null) {
+ return null;
+ } else if(jobName.length() > JOB_NAME_OUTPUT_LENGTH) {
+ jobName = jobName.substring(0, JOB_NAME_OUTPUT_LENGTH);
+ }
+ Pattern pattern = Pattern.compile("(?=(?<" + regJobId + ">\\d+)\\s+\\w+\\s+" + jobName + ")"); // regex - look ahead and match
+ if (rawOutput != null) {
+ Matcher matcher = pattern.matcher(rawOutput);
+ if (matcher.find()) {
+ return matcher.group(regJobId);
+ } else {
+ log.error("No match is found for JobName");
+ return null;
+ }
+ } else {
+ log.error("Error: RawOutput shouldn't be null");
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template b/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template
new file mode 100644
index 0000000..a5bfbe0
--- /dev/null
+++ b/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template
@@ -0,0 +1,24 @@
+# HTCondor job submission script generated by Apache Airavata
+<%
+ if (executablePath != null && executablePath != "") out.print 'executable = ' + executablePath + '\n'
+ if (inputs != null && inputs.size() > 0) out.print 'arguments = \"'
+ if (inputs != null && inputs.size() > 0) for(input in inputs) out.print input + ' '
+ if (inputs != null && inputs.size() > 0) out.print '\"\n'
+ if (exports != null && exports.size() > 0) out.print 'environment = '
+ if (exports != null && exports.size() > 0) for(com in exports) out.print com + ';'
+ if (exports != null && exports.size() > 0) out.print '\n'
+
+ if (qualityOfService != null && qualityOfService != "") out.print 'priority = ' + qualityOfService + '\n'
+ if (cpuCount != null && cpuCount != "") out.print 'request_cpus = ' + cpuCount + '\n'
+ if (usedMem != null && usedMem != "") out.print 'request_memory = ' + usedMem + '\n'
+ if (mailAddress != null && mailAddress != "") out.print 'notification = Always\nnotify_user = ' + mailAddress + '\n'
+
+ if (workingDirectory != null && workingDirectory != "") out.print 'initialdir = ' + workingDirectory + '\n'
+ if (standardOutFile != null && standardOutFile != "") out.print 'output = ' + standardOutFile + '\n'
+ if (standardErrorFile != null && standardErrorFile != "") out.print 'error = ' + standardErrorFile + '\n'
+ out.print 'should_transfer_files = Yes\nwhen_to_transfer_output = ON_EXIT\n'
+ if (inputFiles != null && inputFiles.size() > 0) out.print 'transfer_input_files = '
+ if (inputFiles != null && inputFiles.size() > 0) for(file in inputFiles) out.print file + ', '
+ if (inputFiles != null && inputFiles.size() > 0) out.print '\n'
+ out.print 'queue\n'
+%>
\ No newline at end of file
diff --git a/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml b/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
index 0c86bfb..02d03b3 100644
--- a/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
+++ b/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
@@ -45,4 +45,8 @@ config:
- root <ro...@local> # USD HPC Cluster
- root <li...@siu.edu> # SIU Little Dog
- sge@bigdog.research.siu.edu # SIU Big Dog
- - root <ro...@legacy.usd.edu> # USD HPC Cluster
\ No newline at end of file
+ - root <ro...@legacy.usd.edu> # USD HPC Cluster
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/modules/ide-integration/src/main/resources/email-config.yaml b/modules/ide-integration/src/main/resources/email-config.yaml
index e6c6a85..9d701bb 100644
--- a/modules/ide-integration/src/main/resources/email-config.yaml
+++ b/modules/ide-integration/src/main/resources/email-config.yaml
@@ -45,4 +45,8 @@ config:
- root <ro...@local> # USD HPC Cluster
- root <li...@siu.edu> # SIU Little Dog
- sge@bigdog.research.siu.edu # SIU Big Dog
- - root <ro...@legacy.usd.edu> # USD HPC Cluster
\ No newline at end of file
+ - root <ro...@legacy.usd.edu> # USD HPC Cluster
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
new file mode 100644
index 0000000..7158ec6
--- /dev/null
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
@@ -0,0 +1,129 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.monitor.JobStatusResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HTCondorEmailParser implements EmailParser {
+ private static final Logger log = LoggerFactory.getLogger(HTCondorEmailParser.class);
+
+ // Group matterns to match against
+ private static final String JOBID = "jobid";
+ private static final String STATUS = "status";
+
+ // Regex used to match desired information
+ private static final String JOBID_REGEX = "(?<" + JOBID + ">\\d+)\\.\\d+"; // Regex pattern to match a Job ID from an HTCondor email
+ private static final String CONTENTS_REGEX = "\\s*(?=exited\\s+\\S+\\s+with status\\s+(?<" + STATUS + ">-?\\d+\\.?\\d*))"; // Regex pattern to match a Job Status
+
+ // Regex Patterns
+ private static final Pattern jobIdPattern = Pattern.compile(JOBID_REGEX);
+ private static final Pattern statusPattern = Pattern.compile(CONTENTS_REGEX);
+
+
+ /*
+ * Name : JobStatusResult
+ * Params : Message message : The email message that was received
+ * Returns : JobStatusResult
+ * Purpose : Responsible for parsing the email to access an HTCondor job status
+ */
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ // Job Status Results
+ JobStatusResult jobStatusResult = new JobStatusResult();
+
+ try {
+ // Parse the Subject Line to get the job ID
+ parseSubject(message.getSubject(), jobStatusResult);
+
+ // Parse the email contents to get the job state
+ parseJobState((String) message.getContent(), jobStatusResult);
+ } catch (IOException e) {
+ throw new AiravataException("[EJM]: There was an error while parsing the content of the HTCondor email -> " + e);
+ }
+
+ return jobStatusResult;
+ }
+
+
+ /*
+ * Name : parseSubject
+ * Params : String subject : The email's subject line
+ * JobStatusResult jobStatusResult : The JobStatusResult to fill out
+ * Returns : None
+ * Purpose : To parse the HTCondor email subject line for the job ID
+ */
+ private void parseSubject(String subject, JobStatusResult jobStatusResult) {
+ // Create a new Matcher object to use for parsing the subject line
+ Matcher matcher = jobIdPattern.matcher(subject);
+
+ // Parse the job ID if the Job ID is available in the subject line
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ jobStatusResult.setJobName("HTCondor");
+ } else {
+ log.error("[EJM]: The Job ID was not found in the HTCondor email subject -> " + subject);
+ }
+ }
+
+
+ /*
+ * Name : parseJobState
+ * Params : String content : The email's message content
+ * JobStatusResult jobStatusResult : The JobStatusResult to fill out
+ * Returns : None
+ * Purpose : To parse the HTCondor email for the job status.
+ * [NOTE] Due to the limited information available in the HTCondor status emails, the only
+ * statuses that may be parsed are FAILURE and COMPLETE
+ */
+ private void parseJobState(String content, JobStatusResult jobStatusResult) {
+ // Split message content into an array of lines
+// String[] messageArray = content.split("\n");
+
+ // Access the line of the email with the status result
+// String statusLine = messageArray[5];
+
+ // Match the job status in the status line
+ Matcher matcher = statusPattern.matcher(content);
+
+ // Determine the state that the job is in
+ if(matcher.find()) {
+ String status = matcher.group(STATUS);
+
+ if (status.equals("0")) {
+ jobStatusResult.setState(JobState.COMPLETE);
+ }else if (!status.isEmpty()) {
+ jobStatusResult.setState(JobState.FAILED);
+ } else {
+ log.error("[EJM] An unknown job status result was found in the content of the HTCondor email. Status found: " + status);
+ }
+ }else{
+ log.error("[EJM]: The Job Status was not found in the content of the HTCondor email.");
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml b/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
index 4c8f147..cb99eda 100644
--- a/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
+++ b/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
@@ -18,4 +18,8 @@ config:
- jobManagerType: UGE
emailParser: org.apache.airavata.monitor.email.parser.UGEEmailParser
resourceEmailAddresses:
- - ls4.tacc.utexas.edu # contain Lonestar
\ No newline at end of file
+ - ls4.tacc.utexas.edu # contain Lonestar
+
+ - jobManagerType: HTCONDOR
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
index c4d0137..1d64c00 100644
--- a/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
+++ b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
@@ -54,7 +54,8 @@ enum ResourceJobManagerType {
LSF,
UGE,
CLOUD,
- AIRAVATA_CUSTOM
+ AIRAVATA_CUSTOM,
+ HTCONDOR
}
/**