You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2022/01/14 01:45:59 UTC
[airavata] branch htcondor-new created (now a37ae49)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch htcondor-new
in repository https://gitbox.apache.org/repos/asf/airavata.git.
at a37ae49 Relevant changes from htcondor branch
This branch includes the following new commits:
new a37ae49 Relevant changes from htcondor branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airavata] 01/01: Relevant changes from htcondor branch
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch htcondor-new
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit a37ae49ef4959c731dd220830711bd790c7e1635
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Jan 13 20:45:20 2022 -0500
Relevant changes from htcondor branch
---
.../computeresource/ResourceJobManagerType.java | 5 +-
.../templates/email-monitor/email-config.yaml.j2 | 4 +
.../task/submission/config/GroovyMapBuilder.java | 57 ++++++-
.../impl/task/submission/config/GroovyMapData.java | 12 ++
.../impl/task/submission/config/JobFactory.java | 5 +
.../config/app/HTCondorJobConfiguration.java | 117 ++++++++++++++
.../impl/task/submission/config/app/JobUtil.java | 4 +-
.../config/app/parser/HTCondorOutputParser.java | 169 +++++++++++++++++++++
.../src/main/resources/HTCONDOR_Groovy.template | 24 +++
.../resources/email-monitor/conf/email-config.yaml | 6 +-
.../src/main/resources/email-config.yaml | 6 +-
.../monitor/email/parser/HTCondorEmailParser.java | 129 ++++++++++++++++
.../src/main/resources/email-config.yaml | 6 +-
.../compute_resource_model.thrift | 3 +-
14 files changed, 539 insertions(+), 8 deletions(-)
diff --git a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
index e1a336c..f7b73a2 100644
--- a/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
+++ b/airavata-api/airavata-data-models/src/main/java/org/apache/airavata/model/appcatalog/computeresource/ResourceJobManagerType.java
@@ -54,7 +54,8 @@ public enum ResourceJobManagerType implements org.apache.thrift.TEnum {
LSF(3),
UGE(4),
CLOUD(5),
- AIRAVATA_CUSTOM(6);
+ AIRAVATA_CUSTOM(6),
+ HTCONDOR(7);
private final int value;
@@ -89,6 +90,8 @@ public enum ResourceJobManagerType implements org.apache.thrift.TEnum {
return CLOUD;
case 6:
return AIRAVATA_CUSTOM;
+ case 7:
+ return HTCONDOR;
default:
return null;
}
diff --git a/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2 b/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
index 4f9ce90..9b4cf55 100644
--- a/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
+++ b/dev-tools/ansible/roles/job_monitor/templates/email-monitor/email-config.yaml.j2
@@ -93,3 +93,7 @@ config:
resourceEmailAddresses:
- iu.xsede.edu # test resource mail address
- tcs.tulsahpc.org #Tandy
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
index dfca629..ed5dd45 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
@@ -76,7 +76,10 @@ public class GroovyMapBuilder {
mapData.setQueueSpecificMacros(taskContext.getQueueSpecificMacros());
mapData.setAccountString(taskContext.getAllocationProjectNumber());
mapData.setReservation(taskContext.getReservation());
- mapData.setJobName("A" + String.valueOf(generateJobName()));
+ if(this.taskContext.getResourceJobManager().getResourceJobManagerType() == ResourceJobManagerType.HTCONDOR)
+ mapData.setJobName("HTCondor");
+ else
+ mapData.setJobName("A" + String.valueOf(generateJobName()));
mapData.setWorkingDirectory(taskContext.getWorkingDir());
mapData.setTaskId(taskContext.getTaskId());
mapData.setExperimentDataDir(taskContext.getProcessModel().getExperimentDataDir());
@@ -94,6 +97,9 @@ public class GroovyMapBuilder {
inputValues.addAll(getProcessOutputValues(taskContext.getProcessModel().getProcessOutputs(), true));
mapData.setInputs(inputValues);
+ List<String> inputFiles = getProcessInputFiles(taskContext.getProcessModel().getProcessInputs(), false);
+ mapData.setInputFiles(inputFiles);
+
List<String> inputValuesAll = getProcessInputValues(taskContext.getProcessModel().getProcessInputs(), false);
inputValuesAll.addAll(getProcessOutputValues(taskContext.getProcessModel().getProcessOutputs(), false));
mapData.setInputsAll(inputValuesAll);
@@ -305,6 +311,55 @@ public class GroovyMapBuilder {
return inputValues;
}
+ private static List<String> getProcessInputFiles(List<InputDataObjectType> processInputs, boolean commandLineOnly) {
+ List<String> inputFiles = new ArrayList<String>();
+ if (processInputs != null) {
+
+ // sort the inputs first and then build the command ListR
+ Comparator<InputDataObjectType> inputOrderComparator = new Comparator<InputDataObjectType>() {
+ @Override
+ public int compare(InputDataObjectType inputDataObjectType, InputDataObjectType t1) {
+ return inputDataObjectType.getInputOrder() - t1.getInputOrder();
+ }
+ };
+ Set<InputDataObjectType> sortedInputSet = new TreeSet<InputDataObjectType>(inputOrderComparator);
+ for (InputDataObjectType input : processInputs) {
+ sortedInputSet.add(input);
+ }
+ for (InputDataObjectType inputDataObjectType : sortedInputSet) {
+ if (!inputDataObjectType.isIsRequired() &&
+ (inputDataObjectType.getValue() == null || "".equals(inputDataObjectType.getValue()))) {
+ // For URI/ Collection non required inputs, if the value is empty, ignore it. Fix for airavata-3276
+ continue;
+ }
+
+ if (inputDataObjectType.getValue() != null
+ && !inputDataObjectType.getValue().equals("")) {
+ if (inputDataObjectType.getType() == DataType.URI) {
+ if (inputDataObjectType.getOverrideFilename() != null) {
+ inputFiles.add(inputDataObjectType.getOverrideFilename());
+ } else {
+ // set only the relative path
+ String filePath = inputDataObjectType.getValue();
+ filePath = filePath.substring(filePath.lastIndexOf(File.separatorChar) + 1, filePath.length());
+ inputFiles.add(filePath);
+ }
+ } else if (inputDataObjectType.getType() == DataType.URI_COLLECTION) {
+ String filePaths = inputDataObjectType.getValue();
+ String[] paths = filePaths.split(MULTIPLE_INPUTS_SPLITTER);
+
+ for (int i = 0; i < paths.length; i++) {
+ paths[i] = paths[i].substring(paths[i].lastIndexOf(File.separatorChar) + 1);
+ }
+
+ inputFiles.add(String.join(" ", paths));
+ }
+ }
+ }
+ }
+ return inputFiles;
+ }
+
private static List<String> getProcessOutputValues(List<OutputDataObjectType> processOutputs, boolean commandLineOnly) {
List<String> inputValues = new ArrayList<>();
if (processOutputs != null) {
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
index 45c9e9d..24e02b4 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java
@@ -89,6 +89,9 @@ public class GroovyMapData {
@ScriptTag(name = "inputs")
private List<String> inputs;
+ @ScriptTag(name = "inputFiles")
+ private List<String> inputFiles;
+
@ScriptTag(name = "inputsAll")
private List<String> inputsAll;
@@ -317,6 +320,15 @@ public class GroovyMapData {
return this;
}
+ public List<String> getInputFiles() {
+ return inputFiles;
+ }
+
+ public GroovyMapData setInputFiles(List<String> inputFiles) {
+ this.inputFiles = inputFiles;
+ return this;
+ }
+
public List<String> getInputsAll() {
return inputsAll;
}
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
index e9099e4..5deb964 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/JobFactory.java
@@ -45,6 +45,8 @@ public class JobFactory {
return "LSF_Groovy.template";
case CLOUD:
return "CLOUD_Groovy.template";
+ case HTCONDOR:
+ return "HTCONDOR_Groovy.template";
default:
return null;
}
@@ -119,6 +121,9 @@ public class JobFactory {
case FORK:
return new ForkJobConfiguration(templateFileName, ".sh", resourceJobManager.getJobManagerBinPath(),
resourceJobManager.getJobManagerCommands(), new ForkOutputParser());
+ case HTCONDOR:
+ return new HTCondorJobConfiguration(templateFileName, ".submit", resourceJobManager
+ .getJobManagerBinPath(), resourceJobManager.getJobManagerCommands(), new HTCondorOutputParser());
// We don't have a job configuration manager for CLOUD type
default:
throw new Exception("Could not find a job manager configuration for job manager type "
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java
new file mode 100644
index 0000000..034b887
--- /dev/null
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java
@@ -0,0 +1,117 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.helix.impl.task.submission.config.app;
+
+import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration;
+import org.apache.airavata.helix.impl.task.submission.config.OutputParser;
+import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo;
+import org.apache.airavata.model.appcatalog.computeresource.JobManagerCommand;
+import org.apache.commons.io.FilenameUtils;
+
+import java.io.File;
+import java.util.Map;
+
+public class HTCondorJobConfiguration implements JobManagerConfiguration {
+ private final Map<JobManagerCommand, String> jMCommands;
+ private String jobDescriptionTemplateName;
+ private String scriptExtension;
+ private String installedPath;
+ private OutputParser parser;
+
+ public HTCondorJobConfiguration(String jobDescriptionTemplateName,
+ String scriptExtension, String installedPath, Map<JobManagerCommand, String>
+ jobManagerCommands, OutputParser parser) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ this.scriptExtension = scriptExtension;
+ this.parser = parser;
+ installedPath = installedPath.trim();
+ if (installedPath.endsWith("/")) {
+ this.installedPath = installedPath;
+ } else {
+ this.installedPath = installedPath + "/";
+ }
+ this.jMCommands = jobManagerCommands;
+ }
+
+ public RawCommandInfo getCancelCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.DELETION).trim() + " " + jobID);
+ }
+
+ public String getJobDescriptionTemplateName() {
+ return jobDescriptionTemplateName;
+ }
+
+ public void setJobDescriptionTemplateName(String jobDescriptionTemplateName) {
+ this.jobDescriptionTemplateName = jobDescriptionTemplateName;
+ }
+
+ public RawCommandInfo getMonitorCommand(String jobID) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " " + jobID + " -nobatch");
+ }
+
+ public String getScriptExtension() {
+ return scriptExtension;
+ }
+
+ public RawCommandInfo getSubmitCommand(String workingDirectory,String pbsFilePath) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION).trim() + " " +
+ workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath));
+ }
+
+ public String getInstalledPath() {
+ return installedPath;
+ }
+
+ public void setInstalledPath(String installedPath) {
+ this.installedPath = installedPath;
+ }
+
+ public OutputParser getParser() {
+ return parser;
+ }
+
+ public void setParser(OutputParser parser) {
+ this.parser = parser;
+ }
+
+ public RawCommandInfo getUserBasedMonitorCommand(String userName) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " -submitter " + userName);
+ }
+
+ @Override
+ public RawCommandInfo getJobIdMonitorCommand(String jobName, String userName) {
+ return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.JOB_MONITORING).trim() + " " + jobName + " -submitter " + userName);
+ }
+
+ @Override
+ public String getBaseCancelCommand() {
+ return jMCommands.get(JobManagerCommand.DELETION).trim();
+ }
+
+ @Override
+ public String getBaseMonitorCommand() {
+ return jMCommands.get(JobManagerCommand.JOB_MONITORING).trim();
+ }
+
+ @Override
+ public String getBaseSubmitCommand() {
+ return jMCommands.get(JobManagerCommand.SUBMISSION).trim();
+ }
+}
\ No newline at end of file
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
index 650cf53..0ee5c7b 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/JobUtil.java
@@ -39,11 +39,11 @@ public class JobUtil {
return JobState.ACTIVE;
// } else if ("T".equals(status)) {
// return JobState.HELD;
- } else if ("W".equals(status) || "PD".equals(status)) {
+ } else if ("W".equals(status) || "PD".equals(status) || "I".equals(status)) {
return JobState.QUEUED;
} else if ("S".equals(status) || "PSUSP".equals(status) || "USUSP".equals(status) || "SSUSP".equals(status)) {
return JobState.SUSPENDED;
- } else if ("CA".equals(status)) {
+ } else if ("CA".equals(status) || "X".equals(status)) {
return JobState.CANCELED;
} else if ("F".equals(status) || "NF".equals(status) || "TO".equals(status) || "EXIT".equals(status)) {
return JobState.FAILED;
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java
new file mode 100644
index 0000000..5a6dfec
--- /dev/null
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/parser/HTCondorOutputParser.java
@@ -0,0 +1,169 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.helix.impl.task.submission.config.app.parser;
+
+import org.apache.airavata.helix.impl.task.submission.config.OutputParser;
+import org.apache.airavata.helix.impl.task.submission.config.app.JobUtil;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.model.status.JobStatus;
+import org.apache.airavata.registry.core.utils.DBConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HTCondorOutputParser implements OutputParser {
+ private static final Logger log = LoggerFactory.getLogger(HTCondorOutputParser.class);
+ public static final int JOB_NAME_OUTPUT_LENGTH = 8;
+ public static final String STATUS = "status";
+ public static final String JOBID = "jobId";
+
+ /**
+ * This can be used to parseSingleJob the result of a job submission to get the JobID
+ * @param rawOutput
+ * @return the job id as a String, or null if no job id found
+ */
+ public String parseJobSubmission(String rawOutput) throws Exception {
+ log.info(rawOutput);
+ if (rawOutput != null && !rawOutput.isEmpty()) {
+
+ Pattern pattern = Pattern.compile("\\d+ job\\(s\\) submitted to cluster (?<" + JOBID + ">\\d+)");
+ Matcher matcher = pattern.matcher(rawOutput);
+
+ if (matcher.find()) {
+ return matcher.group(JOBID);
+ }
+ }
+ return "";
+ }
+
+
+ /**
+ * Parse output return by job submission task and identify jobSubmission failures.
+ * @param rawOutput
+ * @return true if job submission has been failed, false otherwise.
+ */
+ public boolean isJobSubmissionFailed(String rawOutput) {
+ Pattern pattern = Pattern.compile("failed");
+ Matcher matcher = pattern.matcher(rawOutput);
+ return matcher.find();
+ }
+
+
+ /**
+ * This can be used to get the job status from the output
+ * @param jobID
+ * @param rawOutput
+ */
+ public JobStatus parseJobStatus(String jobID, String rawOutput) throws Exception {
+ log.info(rawOutput);
+ if (rawOutput != null && !rawOutput.isEmpty()) {
+ Pattern pattern = Pattern.compile("\\s+" + jobID + ".\\d+(?=\\s+\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+(?<" + STATUS + ">\\w+))");
+ Matcher matcher = pattern.matcher(rawOutput);
+
+ if (matcher.find()) {
+ if (matcher.group(STATUS).equals("E")) {
+ log.info("parsing the job status returned : " + STATUS);
+ return new JobStatus(JobState.FAILED);
+ }
+ return new JobStatus(JobUtil.getJobState(matcher.group(STATUS)));
+ }
+ }
+ return null;
+ }
+
+ /**
+ * This can be used to parseSingleJob a big output and get multipleJob statuses
+ * @param statusMap list of status map will return and key will be the job ID
+ * @param rawOutput
+ */
+ public void parseJobStatuses(String userName, Map<String, JobStatus> statusMap, String rawOutput) throws Exception {
+ log.debug(rawOutput);
+
+ String[] info = rawOutput.split("\n");
+ String lastString = info[info.length - 1];
+
+ if (lastString.contains("ID") || lastString.contains("OWNER")) {
+ log.info("There are no jobs with this username ... ");
+ return;
+ }
+
+ for (String jobID : statusMap.keySet()) {
+ String jobId = jobID.split(",")[0];
+ String ownerName = jobID.split(",")[1];
+ boolean found = false;
+
+ for (int i = 1; i < info.length; i++) {
+ if (info[i].contains(ownerName)) {
+ // now starts processing this line
+ log.info(info[i]);
+ String correctLine = info[i];
+ String[] columns = correctLine.split(" ");
+ List<String> columnList = new ArrayList<String>();
+ for (String s : columns) {
+ if (!"".equals(s)) {
+ columnList.add(s);
+ }
+ }
+ if ("E".equals(columnList.get(4))) {
+ columnList.set(4, "Er");
+ }
+ try {
+ statusMap.put(jobID, new JobStatus(JobState.valueOf(columnList.get(4))));
+ } catch (IndexOutOfBoundsException e) {
+ statusMap.put(jobID, new JobStatus(JobState.valueOf("U")));
+ }
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ log.error("Couldn't find the status of the Job with Owner: " + ownerName + "Job Id: " + jobId);
+ }
+ }
+ }
+
+ @Override
+ public String parseJobId(String jobName, String rawOutput) throws Exception {
+ String regJobId = "jobId";
+ if (jobName == null) {
+ return null;
+ } else if(jobName.length() > JOB_NAME_OUTPUT_LENGTH) {
+ jobName = jobName.substring(0, JOB_NAME_OUTPUT_LENGTH);
+ }
+ Pattern pattern = Pattern.compile("(?=(?<" + regJobId + ">\\d+)\\s+\\w+\\s+" + jobName + ")"); // regex - look ahead and match
+ if (rawOutput != null) {
+ Matcher matcher = pattern.matcher(rawOutput);
+ if (matcher.find()) {
+ return matcher.group(regJobId);
+ } else {
+ log.error("No match is found for JobName");
+ return null;
+ }
+ } else {
+ log.error("Error: RawOutput shouldn't be null");
+ return null;
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template b/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template
new file mode 100644
index 0000000..a5bfbe0
--- /dev/null
+++ b/modules/configuration/server/src/main/resources/HTCONDOR_Groovy.template
@@ -0,0 +1,24 @@
+# HTCondor job submission script generated by Apache Airavata
+<%
+ if (executablePath != null && executablePath != "") out.print 'executable = ' + executablePath + '\n'
+ if (inputs != null && inputs.size() > 0) out.print 'arguments = \"'
+ if (inputs != null && inputs.size() > 0) for(input in inputs) out.print input + ' '
+ if (inputs != null && inputs.size() > 0) out.print '\"\n'
+ if (exports != null && exports.size() > 0) out.print 'environment = '
+ if (exports != null && exports.size() > 0) for(com in exports) out.print com + ';'
+ if (exports != null && exports.size() > 0) out.print '\n'
+
+ if (qualityOfService != null && qualityOfService != "") out.print 'priority = ' + qualityOfService + '\n'
+ if (cpuCount != null && cpuCount != "") out.print 'request_cpus = ' + cpuCount + '\n'
+ if (usedMem != null && usedMem != "") out.print 'request_memory = ' + usedMem + '\n'
+ if (mailAddress != null && mailAddress != "") out.print 'notification = Always\nnotify_user = ' + mailAddress + '\n'
+
+ if (workingDirectory != null && workingDirectory != "") out.print 'initialdir = ' + workingDirectory + '\n'
+ if (standardOutFile != null && standardOutFile != "") out.print 'output = ' + standardOutFile + '\n'
+ if (standardErrorFile != null && standardErrorFile != "") out.print 'error = ' + standardErrorFile + '\n'
+ out.print 'should_transfer_files = Yes\nwhen_to_transfer_output = ON_EXIT\n'
+ if (inputFiles != null && inputFiles.size() > 0) out.print 'transfer_input_files = '
+ if (inputFiles != null && inputFiles.size() > 0) for(file in inputFiles) out.print file + ', '
+ if (inputFiles != null && inputFiles.size() > 0) out.print '\n'
+ out.print 'queue\n'
+%>
\ No newline at end of file
diff --git a/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml b/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
index 0c86bfb..02d03b3 100644
--- a/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
+++ b/modules/distribution/src/main/resources/email-monitor/conf/email-config.yaml
@@ -45,4 +45,8 @@ config:
- root <ro...@local> # USD HPC Cluster
- root <li...@siu.edu> # SIU Little Dog
- sge@bigdog.research.siu.edu # SIU Big Dog
- - root <ro...@legacy.usd.edu> # USD HPC Cluster
\ No newline at end of file
+ - root <ro...@legacy.usd.edu> # USD HPC Cluster
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/modules/ide-integration/src/main/resources/email-config.yaml b/modules/ide-integration/src/main/resources/email-config.yaml
index e6c6a85..9d701bb 100644
--- a/modules/ide-integration/src/main/resources/email-config.yaml
+++ b/modules/ide-integration/src/main/resources/email-config.yaml
@@ -45,4 +45,8 @@ config:
- root <ro...@local> # USD HPC Cluster
- root <li...@siu.edu> # SIU Little Dog
- sge@bigdog.research.siu.edu # SIU Big Dog
- - root <ro...@legacy.usd.edu> # USD HPC Cluster
\ No newline at end of file
+ - root <ro...@legacy.usd.edu> # USD HPC Cluster
+
+ - jobManagerType: HTCondor
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
new file mode 100644
index 0000000..7158ec6
--- /dev/null
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
@@ -0,0 +1,129 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.airavata.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.model.status.JobState;
+import org.apache.airavata.monitor.JobStatusResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HTCondorEmailParser implements EmailParser {
+ private static final Logger log = LoggerFactory.getLogger(HTCondorEmailParser.class);
+
+ // Group matterns to match against
+ private static final String JOBID = "jobid";
+ private static final String STATUS = "status";
+
+ // Regex used to match desired information
+ private static final String JOBID_REGEX = "(?<" + JOBID + ">\\d+)\\.\\d+"; // Regex pattern to match a Job ID from an HTCondor email
+ private static final String CONTENTS_REGEX = "\\s*(?=exited\\s+\\S+\\s+with status\\s+(?<" + STATUS + ">-?\\d+\\.?\\d*))"; // Regex pattern to match a Job Status
+
+ // Regex Patterns
+ private static final Pattern jobIdPattern = Pattern.compile(JOBID_REGEX);
+ private static final Pattern statusPattern = Pattern.compile(CONTENTS_REGEX);
+
+
+ /*
+ * Name : JobStatusResult
+ * Params : Message message : The email message that was received
+ * Returns : JobStatusResult
+ * Purpose : Responsible for parsing the email to access an HTCondor job status
+ */
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ // Job Status Results
+ JobStatusResult jobStatusResult = new JobStatusResult();
+
+ try {
+ // Parse the Subject Line to get the job ID
+ parseSubject(message.getSubject(), jobStatusResult);
+
+ // Parse the email contents to get the job state
+ parseJobState((String) message.getContent(), jobStatusResult);
+ } catch (IOException e) {
+ throw new AiravataException("[EJM]: There was an error while parsing the content of the HTCondor email -> " + e);
+ }
+
+ return jobStatusResult;
+ }
+
+
+ /*
+ * Name : parseSubject
+ * Params : String subject : The email's subject line
+ * JobStatusResult jobStatusResult : The JobStatusResult to fill out
+ * Returns : None
+ * Purpose : To parse the HTCondor email subject line for the job ID
+ */
+ private void parseSubject(String subject, JobStatusResult jobStatusResult) {
+ // Create a new Matcher object to use for parsing the subject line
+ Matcher matcher = jobIdPattern.matcher(subject);
+
+ // Parse the job ID if the Job ID is available in the subject line
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ jobStatusResult.setJobName("HTCondor");
+ } else {
+ log.error("[EJM]: The Job ID was not found in the HTCondor email subject -> " + subject);
+ }
+ }
+
+
+ /*
+ * Name : parseJobState
+ * Params : String content : The email's message content
+ * JobStatusResult jobStatusResult : The JobStatusResult to fill out
+ * Returns : None
+ * Purpose : To parse the HTCondor email for the job status.
+ * [NOTE] Due to the limited information available in the HTCondor status emails, the only
+ * statuses that may be parsed are FAILURE and COMPLETE
+ */
+ private void parseJobState(String content, JobStatusResult jobStatusResult) {
+ // Split message content into an array of lines
+// String[] messageArray = content.split("\n");
+
+ // Access the line of the email with the status result
+// String statusLine = messageArray[5];
+
+ // Match the job status in the status line
+ Matcher matcher = statusPattern.matcher(content);
+
+ // Determine the state that the job is in
+ if(matcher.find()) {
+ String status = matcher.group(STATUS);
+
+ if (status.equals("0")) {
+ jobStatusResult.setState(JobState.COMPLETE);
+ }else if (!status.isEmpty()) {
+ jobStatusResult.setState(JobState.FAILED);
+ } else {
+ log.error("[EJM] An unknown job status result was found in the content of the HTCondor email. Status found: " + status);
+ }
+ }else{
+ log.error("[EJM]: The Job Status was not found in the content of the HTCondor email.");
+ }
+ }
+}
\ No newline at end of file
diff --git a/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml b/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
index 4c8f147..cb99eda 100644
--- a/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
+++ b/modules/job-monitor/email-monitor/src/main/resources/email-config.yaml
@@ -18,4 +18,8 @@ config:
- jobManagerType: UGE
emailParser: org.apache.airavata.monitor.email.parser.UGEEmailParser
resourceEmailAddresses:
- - ls4.tacc.utexas.edu # contain Lonestar
\ No newline at end of file
+ - ls4.tacc.utexas.edu # contain Lonestar
+
+ - jobManagerType: HTCONDOR
+ emailParser: org.apache.airavata.monitor.email.parser.HTCondorEmailParser
+ resourceEmailAddresses:
\ No newline at end of file
diff --git a/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
index c4d0137..1d64c00 100644
--- a/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
+++ b/thrift-interface-descriptions/data-models/resource-catalog-models/compute_resource_model.thrift
@@ -54,7 +54,8 @@ enum ResourceJobManagerType {
LSF,
UGE,
CLOUD,
- AIRAVATA_CUSTOM
+ AIRAVATA_CUSTOM,
+ HTCONDOR
}
/**