You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2022/09/08 14:31:54 UTC
[airavata] branch HTCondor-JobName-Fix created (now 8fd9abd4a4)
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a change to branch HTCondor-JobName-Fix
in repository https://gitbox.apache.org/repos/asf/airavata.git
at 8fd9abd4a4 HtCondor JobName Fix
This branch includes the following new commits:
new 8fd9abd4a4 HtCondor JobName Fix
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[airavata] 01/01: HtCondor JobName Fix
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch HTCondor-JobName-Fix
in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 8fd9abd4a4b7bfd53adbbbaae80698e03d802861
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Sep 8 10:31:38 2022 -0400
HtCondor JobName Fix
---
.../task/submission/config/GroovyMapBuilder.java | 5 +---
.../airavata/monitor/email/EmailBasedMonitor.java | 16 +++++++++---
.../email/parser/AiravataCustomMailParser.java | 3 ++-
.../airavata/monitor/email/parser/EmailParser.java | 3 ++-
.../monitor/email/parser/HTCondorEmailParser.java | 29 +++++++++++++++++++---
.../monitor/email/parser/LSFEmailParser.java | 3 ++-
.../monitor/email/parser/PBSEmailParser.java | 3 ++-
.../monitor/email/parser/SLURMEmailParser.java | 3 ++-
.../monitor/email/parser/UGEEmailParser.java | 3 ++-
9 files changed, 52 insertions(+), 16 deletions(-)
diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
index ed5dd45751..b4f2814aac 100644
--- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
+++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapBuilder.java
@@ -76,10 +76,7 @@ public class GroovyMapBuilder {
mapData.setQueueSpecificMacros(taskContext.getQueueSpecificMacros());
mapData.setAccountString(taskContext.getAllocationProjectNumber());
mapData.setReservation(taskContext.getReservation());
- if(this.taskContext.getResourceJobManager().getResourceJobManagerType() == ResourceJobManagerType.HTCONDOR)
- mapData.setJobName("HTCondor");
- else
- mapData.setJobName("A" + String.valueOf(generateJobName()));
+ mapData.setJobName("A" + String.valueOf(generateJobName()));
mapData.setWorkingDirectory(taskContext.getWorkingDir());
mapData.setTaskId(taskContext.getTaskId());
mapData.setExperimentDataDir(taskContext.getProcessModel().getExperimentDataDir());
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/EmailBasedMonitor.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/EmailBasedMonitor.java
index ddfd951898..63f9e47ee5 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/EmailBasedMonitor.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/EmailBasedMonitor.java
@@ -28,6 +28,7 @@ import org.apache.airavata.monitor.kafka.MessageProducer;
import org.apache.airavata.monitor.email.parser.EmailParser;
import org.apache.airavata.monitor.email.parser.ResourceConfig;
import org.apache.airavata.model.appcatalog.computeresource.ResourceJobManagerType;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -151,9 +152,18 @@ public class EmailBasedMonitor extends AbstractMonitor implements Runnable {
throw new AiravataException("[EJM]: Un-handle resource job manager type: " + jobMonitorType
.toString() + " for email monitoring --> " + addressStr);
}
- JobStatusResult jobStatusResult = emailParser.parseEmail(message);
- jobStatusResult.setPublisherName(ServerSettings.getSetting("job.monitor.broker.publisher.id"));
- return jobStatusResult;
+ RegistryService.Client regClient = getRegistryClientPool().getResource();
+
+ try {
+ JobStatusResult jobStatusResult = emailParser.parseEmail(message, regClient);
+ jobStatusResult.setPublisherName(ServerSettings.getSetting("job.monitor.broker.publisher.id"));
+ return jobStatusResult;
+ } catch (Exception e) {
+ getRegistryClientPool().returnBrokenResource(regClient);
+ throw e;
+ } finally {
+ getRegistryClientPool().returnResource(regClient);
+ }
}
private ResourceJobManagerType getJobMonitorType(String addressStr) throws AiravataException {
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/AiravataCustomMailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/AiravataCustomMailParser.java
index 1a56c145b2..ae0cecb900 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/AiravataCustomMailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/AiravataCustomMailParser.java
@@ -22,6 +22,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,7 +42,7 @@ public class AiravataCustomMailParser implements EmailParser {
private static final Pattern pattern = Pattern.compile(REGEX);
@Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException {
JobStatusResult jobStatusResult = new JobStatusResult();
parseSubject(message.getSubject(), jobStatusResult);
return jobStatusResult;
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/EmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/EmailParser.java
index a94dc1a352..d76f733c19 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/EmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/EmailParser.java
@@ -21,6 +21,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import javax.mail.Message;
import javax.mail.MessagingException;
@@ -31,5 +32,5 @@ public interface EmailParser {
static final String JOBNAME = "jobName";
static final String EXIT_STATUS = "exitStatus";
- JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
+ JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException;
}
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
index 7158ec6f76..e91b0153ec 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/HTCondorEmailParser.java
@@ -20,14 +20,18 @@
package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.model.job.JobModel;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.mail.Message;
import javax.mail.MessagingException;
import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -53,7 +57,7 @@ public class HTCondorEmailParser implements EmailParser {
* Returns : JobStatusResult
* Purpose : Responsible for parsing the email to access an HTCondor job status
*/
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException{
// Job Status Results
JobStatusResult jobStatusResult = new JobStatusResult();
@@ -63,7 +67,17 @@ public class HTCondorEmailParser implements EmailParser {
// Parse the email contents to get the job state
parseJobState((String) message.getContent(), jobStatusResult);
- } catch (IOException e) {
+
+ String processId = fetchProcessId((String)message.getContent());
+ List<JobModel> jobs = registryClient.getJobs("processId", processId);
+ Optional<JobModel> firstJob = jobs.stream().filter(job -> job.getJobId().equals(jobStatusResult.getJobId())).findFirst();
+ if (firstJob.isPresent()) {
+ jobStatusResult.setJobName(firstJob.get().getJobName());
+ } else {
+ throw new Exception("No job found matching job id " + jobStatusResult.getJobId() + " for HTCondor");
+ }
+
+ } catch (Exception e) {
throw new AiravataException("[EJM]: There was an error while parsing the content of the HTCondor email -> " + e);
}
@@ -85,7 +99,6 @@ public class HTCondorEmailParser implements EmailParser {
// Parse the job ID if the Job ID is available in the subject line
if (matcher.find()) {
jobStatusResult.setJobId(matcher.group(JOBID));
- jobStatusResult.setJobName("HTCondor");
} else {
log.error("[EJM]: The Job ID was not found in the HTCondor email subject -> " + subject);
}
@@ -101,6 +114,16 @@ public class HTCondorEmailParser implements EmailParser {
* [NOTE] Due to the limited information available in the HTCondor status emails, the only
* statuses that may be parsed are FAILURE and COMPLETE
*/
+
+ private String fetchProcessId(String content) {
+ int start = content.indexOf("submitted from directory");
+ int end = content.indexOf("\n", start);
+ String path = content.substring(start + "submitted from directory".length() + 1, end-1);
+ int pathSeperatorIndex = path.lastIndexOf("/");
+ String processId = path.substring(pathSeperatorIndex + 1);
+ return processId;
+ }
+
private void parseJobState(String content, JobStatusResult jobStatusResult) {
// Split message content into an array of lines
// String[] messageArray = content.split("\n");
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/LSFEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/LSFEmailParser.java
index 755b72153a..99660c2ae8 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/LSFEmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/LSFEmailParser.java
@@ -22,6 +22,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +40,7 @@ public class LSFEmailParser implements EmailParser {
public static final String FAILED = "Exited";
@Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException {
JobStatusResult jobStatusResult = new JobStatusResult();
parseContent(message, jobStatusResult);
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/PBSEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/PBSEmailParser.java
index 14a44975df..b0c22988e7 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/PBSEmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/PBSEmailParser.java
@@ -22,6 +22,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,7 +45,7 @@ public class PBSEmailParser implements EmailParser {
private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
@Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException {
JobStatusResult jobStatusResult = new JobStatusResult();
// log.info("Parsing -> " + message.getSubject());
try {
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/SLURMEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/SLURMEmailParser.java
index e3f2637e31..eaae04071d 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/SLURMEmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/SLURMEmailParser.java
@@ -22,6 +22,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,7 @@ public class SLURMEmailParser implements EmailParser {
private static final Pattern pattern = Pattern.compile(REGEX);
@Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException{
JobStatusResult jobStatusResult = new JobStatusResult();
parseSubject(message.getSubject(), jobStatusResult);
return jobStatusResult;
diff --git a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/UGEEmailParser.java b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/UGEEmailParser.java
index 1d7a76463b..ebaf84d90e 100644
--- a/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/UGEEmailParser.java
+++ b/modules/job-monitor/email-monitor/src/main/java/org/apache/airavata/monitor/email/parser/UGEEmailParser.java
@@ -22,6 +22,7 @@ package org.apache.airavata.monitor.email.parser;
import org.apache.airavata.common.exception.AiravataException;
import org.apache.airavata.model.status.JobState;
import org.apache.airavata.monitor.JobStatusResult;
+import org.apache.airavata.registry.api.RegistryService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +46,7 @@ public class UGEEmailParser implements EmailParser {
@Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ public JobStatusResult parseEmail(Message message, RegistryService.Client registryClient) throws MessagingException, AiravataException {
JobStatusResult jobStatusResult = new JobStatusResult();
parseContent(message, jobStatusResult);