You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/04/07 18:34:09 UTC
airavata git commit: Added Lonestar email parser and refactored
parser code
Repository: airavata
Updated Branches:
refs/heads/emailMonitoring b110ff250 -> 9f1fa2d35
Added Lonestar email parser and refactored parser code
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/9f1fa2d3
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/9f1fa2d3
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/9f1fa2d3
Branch: refs/heads/emailMonitoring
Commit: 9f1fa2d35ebe35652a40db9d04c5e96e77b5eb1c
Parents: b110ff2
Author: shamrath <sh...@gmail.com>
Authored: Tue Apr 7 12:34:54 2015 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Tue Apr 7 12:34:54 2015 -0400
----------------------------------------------------------------------
.../airavata/gfac/core/cpi/BetterGfacImpl.java | 1 -
.../core/monitor/mail/EmailBasedMonitor.java | 1 +
.../core/monitor/mail/parser/EmailParser.java | 3 ++
.../mail/parser/LonestarEmailParser.java | 52 +++++++++++++++++++-
.../monitor/mail/parser/PBSEmailParser.java | 8 ++-
.../monitor/mail/parser/SLURMEmailParser.java | 4 +-
.../gfac/ssh/provider/impl/SSHProvider.java | 50 ++++++++++---------
7 files changed, 88 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
index 1d9008b..8d09a09 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/cpi/BetterGfacImpl.java
@@ -305,7 +305,6 @@ public class BetterGfacImpl implements GFac,Watcher {
jobExecutionContext.setGfac(this);
jobExecutionContext.setZk(zk);
jobExecutionContext.setCredentialStoreToken(AiravataZKUtils.getExpTokenId(zk, experimentID, taskID));
- jobExecutionContext.setProperty("MonitorPublisher", monitorPublisher);
// handle job submission protocol
List<JobSubmissionInterface> jobSubmissionInterfaces = computeResource.getJobSubmissionInterfaces();
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
index 8adb860..f15e582 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
@@ -102,6 +102,7 @@ public class EmailBasedMonitor implements Runnable{
}
public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) {
+ log.info("Added Job Id : " + jobId + " to email based monitor map");
jobMonitorMap.put(jobId, jobExecutionContext);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
index d35e6b7..ad8e4a3 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
@@ -27,6 +27,9 @@ import javax.mail.Message;
import javax.mail.MessagingException;
public interface EmailParser {
+ static final String STATUS = "status";
+ static final String JOBID = "jobId";
+ static final String EXIT_STATUS = "exitStatus";
JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
index 6adde4e..59651da 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
@@ -20,5 +20,55 @@
*/
package org.apache.airavata.gfac.core.monitor.mail.parser;
-public class LonestarEmailParser {
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LonestarEmailParser implements EmailParser {
+ private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class);
+ //root@c312-206.ls4.tacc.utexas.edu
+ private static final String SIGNAL = "signal";
+ private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS
+ + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(LONESTAR_REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ String status = matcher.group(STATUS);
+ jobStatusResult.setState(getJobState(status, content));
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for content => \n" + content);
+ }
+ } catch (IOException e) {
+ throw new AiravataException("Error while reading content of the email message");
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String status, String content) {
+ switch (status) {
+ case "Aborted":
+ return JobState.FAILED;
+ case "Scucess":
+ return JobState.COMPLETE;
+ default:
+ return JobState.UNKNOWN;
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
index 37d2317..cd6142c 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
@@ -36,20 +36,18 @@ public class PBSEmailParser implements EmailParser {
private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class);
- private static final String STATUS = "status";
- private static final String JOBID = "jobId";
- private static final String EXIT_STATUS = "exitStatus";
+
private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<"
+ STATUS + ">[a-zA-Z\\ ]*)";
private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
@Override
public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
try {
String content = ((String) message.getContent());
Pattern pattern = Pattern.compile(REGEX);
Matcher matcher = pattern.matcher(content);
- JobStatusResult jobStatusResult = new JobStatusResult();
if (matcher.find()) {
jobStatusResult.setJobId(matcher.group(JOBID));
String statusLine = matcher.group(STATUS);
@@ -62,7 +60,7 @@ public class PBSEmailParser implements EmailParser {
} catch (IOException e) {
throw new AiravataException("Error while reading content of the email message");
}
- return null;
+ return jobStatusResult;
}
private JobState getJobState(String statusLine, String content) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
index e5af721..ceb0013 100644
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
+++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
@@ -42,10 +42,10 @@ public class SLURMEmailParser implements EmailParser {
@Override
public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ JobStatusResult jobStatusResult = new JobStatusResult();
String subject = message.getSubject();
Pattern pattern = Pattern.compile(REGEX);
Matcher matcher = pattern.matcher(subject);
- JobStatusResult jobStatusResult = new JobStatusResult();
if (matcher.find()) {
jobStatusResult.setJobId(matcher.group(JOBID));
jobStatusResult.setState(getJobState(matcher.group(STATUS)));
@@ -57,7 +57,7 @@ public class SLURMEmailParser implements EmailParser {
} else {
log.error("No matched found for subject -> " + subject);
}
- return null;
+ return jobStatusResult;
}
private JobState getJobState(String state) {
http://git-wip-us.apache.org/repos/asf/airavata/blob/9f1fa2d3/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
index 0012ce1..32be550 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
@@ -22,12 +22,15 @@
package org.apache.airavata.gfac.ssh.provider.impl;
import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.common.utils.ServerSettings;
import org.apache.airavata.gfac.Constants;
+import org.apache.airavata.gfac.ExecutionMode;
import org.apache.airavata.gfac.GFacException;
import org.apache.airavata.gfac.core.context.JobExecutionContext;
import org.apache.airavata.gfac.core.context.MessageContext;
+import org.apache.airavata.gfac.core.cpi.BetterGfacImpl;
import org.apache.airavata.gfac.core.handler.GFacHandlerException;
+import org.apache.airavata.gfac.core.handler.ThreadedHandler;
import org.apache.airavata.gfac.core.notification.events.StartExecutionEvent;
import org.apache.airavata.gfac.core.provider.AbstractProvider;
import org.apache.airavata.gfac.core.provider.GFacProviderException;
@@ -367,28 +370,31 @@ public class SSHProvider extends AbstractProvider {
}
public void delegateToMonitorHandlers(JobExecutionContext jobExecutionContext) throws GFacHandlerException {
- /*List<ThreadedHandler> daemonHandlers = BetterGfacImpl.getDaemonHandlers();
- if (daemonHandlers == null) {
- daemonHandlers = BetterGfacImpl.getDaemonHandlers();
- }
- ThreadedHandler pullMonitorHandler = null;
- ThreadedHandler pushMonitorHandler = null;
- for (ThreadedHandler threadedHandler : daemonHandlers) {
- if ("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())) {
- pullMonitorHandler = threadedHandler;
- pullMonitorHandler.invoke(jobExecutionContext);
+ if (ServerSettings.isEmailBasedNotificationEnable()) {
+ try {
+ EmailBasedMonitor emailBasedMonitor = EmailBasedMonitor.getInstant(BetterGfacImpl.getMonitorPublisher());
+ emailBasedMonitor.addToJobMonitorMap(jobExecutionContext);
+ } catch (ApplicationSettingsException e) {
+ throw new GFacHandlerException("Error while delegating job execution context to email based monitor");
+ }
+ } else {
+ List<ThreadedHandler> daemonHandlers = BetterGfacImpl.getDaemonHandlers();
+ if (daemonHandlers == null) {
+ daemonHandlers = BetterGfacImpl.getDaemonHandlers();
+ }
+ ThreadedHandler pullMonitorHandler = null;
+ ThreadedHandler pushMonitorHandler = null;
+ for (ThreadedHandler threadedHandler : daemonHandlers) {
+ if ("org.apache.airavata.gfac.monitor.handlers.GridPullMonitorHandler".equals(threadedHandler.getClass().getName())) {
+ pullMonitorHandler = threadedHandler;
+ pullMonitorHandler.invoke(jobExecutionContext);
+ }
+ // have to handle the GridPushMonitorHandler logic
+ }
+ if (pullMonitorHandler == null && pushMonitorHandler == null && ExecutionMode.ASYNCHRONOUS.equals(jobExecutionContext.getGFacConfiguration().getExecutionMode())) {
+ log.error("No Daemon handler is configured in gfac-config.xml, either pull or push, so monitoring will not invoked" +
+ ", execution is configured as asynchronous, so Outhandler will not be invoked");
}
- // have to handle the GridPushMonitorHandler logic
- }
- if (pullMonitorHandler == null && pushMonitorHandler == null && ExecutionMode.ASYNCHRONOUS.equals(jobExecutionContext.getGFacConfiguration().getExecutionMode())) {
- log.error("No Daemon handler is configured in gfac-config.xml, either pull or push, so monitoring will not invoked" +
- ", execution is configured as asynchronous, so Outhandler will not be invoked");
- }*/
- try {
- EmailBasedMonitor emailBasedMonitor = EmailBasedMonitor.getInstant(((MonitorPublisher) jobExecutionContext.getProperty("MonitorPublisher")));
- emailBasedMonitor.addToJobMonitorMap(jobExecutionContext);
- } catch (ApplicationSettingsException e) {
- throw new GFacHandlerException("Error while delegating job execution context to email based monitor");
}
}