You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/04/13 23:05:34 UTC
airavata git commit: Fixed AIRAVATA-1666
Repository: airavata
Updated Branches:
refs/heads/master 11c01c6af -> 4f95a7908
Fixed AIRAVATA-1666
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/4f95a790
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/4f95a790
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/4f95a790
Branch: refs/heads/master
Commit: 4f95a790825fab6ebb6749fe0e731734c520f338
Parents: 11c01c6
Author: shamrath <sh...@gmail.com>
Authored: Mon Apr 13 17:05:32 2015 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Mon Apr 13 17:05:32 2015 -0400
----------------------------------------------------------------------
.../airavata/common/utils/ServerSettings.java | 29 ++------------
.../main/resources/airavata-server.properties | 9 +----
.../gfac/monitor/email/EmailBasedMonitor.java | 18 ++++++---
.../gfac/monitor/email/EmailMonitorFactory.java | 18 +++++----
.../monitor/email/parser/PBSEmailParser.java | 42 --------------------
.../monitor/email/parser/SLURMEmailParser.java | 25 +-----------
.../gfac/ssh/provider/impl/SSHProvider.java | 3 +-
7 files changed, 31 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
----------------------------------------------------------------------
diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
index 09360d4..e37ea42 100644
--- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
+++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java
@@ -68,12 +68,7 @@ public class ServerSettings extends ApplicationSettings {
private static final String WORKFLOW_PARSER = "workflow.parser";
// email based monitoring configurations
- private static final String EMAIL_BASED_MONITOR_HOST = "email.based.monitor.host";
- private static final String EMAIL_BASED_MONITOR_ADDRESS = "email.based.monitor.address";
- private static final String EMAIL_BASED_MONITOR_PASSWORD = "email.based.monitor.password";
- private static final String EMAIL_BASED_MONITOR_FOLDER_NAME = "email.based.monitor.folder.name";
- private static final String EMAIL_BASED_MONITOR_STORE_PROTOCOL = "email.based.monitor.store.protocol";
- private static final String ENABLE_EMAIL_BASED_MONITORING = "enable.email.based.monitoring";
+ private static final String EMAIL_BASED_MONITORING_PERIOD = "email.based.monitoring.period";
private static boolean stopAllThreads = false;
@@ -222,28 +217,10 @@ public class ServerSettings extends ApplicationSettings {
return getSetting(WORKFLOW_PARSER);
}
- public static String getEmailBasedMonitorHost() throws ApplicationSettingsException {
- return getSetting(EMAIL_BASED_MONITOR_HOST);
- }
-
- public static String getEmailBasedMonitorAddress() throws ApplicationSettingsException {
- return getSetting(EMAIL_BASED_MONITOR_ADDRESS);
- }
-
- public static String getEmailBasedMonitorPassword() throws ApplicationSettingsException {
- return getSetting(EMAIL_BASED_MONITOR_PASSWORD);
- }
-
- public static String getEmailBasedMonitorFolderName() throws ApplicationSettingsException {
- return getSetting(EMAIL_BASED_MONITOR_FOLDER_NAME);
- }
+ public static int getEmailMonitorPeriod() throws ApplicationSettingsException {
+ return Integer.valueOf(getSetting(EMAIL_BASED_MONITORING_PERIOD, "100000"));
- public static String getEmailBasedMonitorStoreProtocol() throws ApplicationSettingsException {
- return getSetting(EMAIL_BASED_MONITOR_STORE_PROTOCOL);
}
- public static boolean isEmailBasedNotificationEnable() {
- return Boolean.valueOf(getSetting(ENABLE_EMAIL_BASED_MONITORING, "false"));
- }
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/configuration/server/src/main/resources/airavata-server.properties
----------------------------------------------------------------------
diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties
index 36f562c..7f18e3d 100644
--- a/modules/configuration/server/src/main/resources/airavata-server.properties
+++ b/modules/configuration/server/src/main/resources/airavata-server.properties
@@ -195,13 +195,8 @@ host.scheduler=org.apache.airavata.gfac.core.scheduler.impl.SimpleHostScheduler
#mechanisms and one would be able to start a monitor
monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
-#These properties will used to enable email base monitoring
-enable.email.based.monitoring=false
-email.based.monitor.host=imap.gmail.com
-email.based.monitor.address=test@scigap.com
-email.based.monitor.password=changeme
-email.based.monitor.folder.name=INBOX
-email.based.monitor.store.protocol=imaps
+#These property will used to query the email server periodically. value in milliseconds(ms).
+email.based.monitoring.period=10000
###########################################################################
# AMQP Notification Configuration
###########################################################################
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
index affe156..631e1a4 100644
--- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
@@ -137,11 +137,12 @@ public class EmailBasedMonitor implements Runnable{
if (!store.isConnected()) {
store.connect();
}
- Thread.sleep(2000);
+ Thread.sleep(ServerSettings.getEmailMonitorPeriod());
emailFolder = store.getFolder(emailMonitorProperty.getFolderName());
emailFolder.open(Folder.READ_WRITE);
Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
List<Message> processedMessages = new ArrayList<>();
+ List<Message> unreadMessages = new ArrayList<>();
for (Message message : searchMessages) {
try {
JobStatusResult jobStatusResult = parse(message);
@@ -152,7 +153,8 @@ public class EmailBasedMonitor implements Runnable{
} else {
// we can get JobExecutionContext null in multiple Gfac instances environment,
// where this job is not submitted by this Gfac instance hence we ignore this message.
- log.info("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
+ unreadMessages.add(message);
+// log.info("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
}
} catch (AiravataException e) {
log.error("Error parsing email message =====================================>", e);
@@ -164,9 +166,15 @@ public class EmailBasedMonitor implements Runnable{
}
}
if (!processedMessages.isEmpty()) {
- Message[] prosMessages = new Message[processedMessages.size()];
- processedMessages.toArray(prosMessages);
- emailFolder.setFlags(prosMessages, new Flags(Flags.Flag.SEEN), true);
+ Message[] seenMessages = new Message[processedMessages.size()];
+ processedMessages.toArray(seenMessages);
+ emailFolder.setFlags(seenMessages, new Flags(Flags.Flag.SEEN), true);
+
+ }
+ if (!unreadMessages.isEmpty()) {
+ Message[] unseenMessages = new Message[unreadMessages.size()];
+ unreadMessages.toArray(unseenMessages);
+ emailFolder.setFlags(unseenMessages, new Flags(Flags.Flag.SEEN), false);
}
emailFolder.close(false);
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java
index 2c24973..5807460 100644
--- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailMonitorFactory.java
@@ -27,27 +27,31 @@ import java.util.Map;
public class EmailMonitorFactory {
- private Map<String, EmailBasedMonitor> emailMonitors = new HashMap<String, EmailBasedMonitor>();
+ private static Map<String, EmailBasedMonitor> emailMonitors = new HashMap<String, EmailBasedMonitor>();
- public synchronized EmailBasedMonitor getEmailBasedMonitor(EmailMonitorProperty emailMonitorProp) {
+ public static EmailBasedMonitor getEmailBasedMonitor(EmailMonitorProperty emailMonitorProp) {
String key = getKey(emailMonitorProp);
EmailBasedMonitor monitor = emailMonitors.get(key);
if (monitor == null) {
- monitor = new EmailBasedMonitor(emailMonitorProp);
- emailMonitors.put(key, monitor);
- new Thread(monitor).start();
+ synchronized (emailMonitors){
+ if (monitor == null) {
+ monitor = new EmailBasedMonitor(emailMonitorProp);
+ emailMonitors.put(key, monitor);
+ new Thread(monitor).start();
+ }
+ }
}
return monitor;
}
- public void stopAllMonitors() {
+ public static void stopAllMonitors() {
for (EmailBasedMonitor emailBasedMonitor : emailMonitors.values()) {
emailBasedMonitor.stopMonitoring();
}
}
- private String getKey(EmailMonitorProperty emailMonitorProp) {
+ private static String getKey(EmailMonitorProperty emailMonitorProp) {
StringBuffer sb = new StringBuffer(emailMonitorProp.getHost().trim());
sb.append("_").append(emailMonitorProp.getStoreProtocol().name());
sb.append("_").append(emailMonitorProp.getEmailAddress().trim());
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
index b748704..6cb961a 100644
--- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
@@ -94,46 +94,4 @@ public class PBSEmailParser implements EmailParser {
return -1;
}
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Begun execution
- */
-/*
- -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Execution terminated
- Exit_status=0
- resources_used.cput=00:14:31
- resources_used.mem=124712kb
- resources_used.vmem=3504116kb
- resources_used.walltime=00:04:10
- Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
- Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
- */
-
-
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
index 643265e..327caa6 100644
--- a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
@@ -52,7 +52,7 @@ public class SLURMEmailParser implements EmailParser {
// TODO remove following test lines
String jobId = matcher.group(JOBID);
String status = matcher.group(STATUS);
- log.info("SLURM " + status + " message received -> " + jobId);
+// log.info("SLURM " + status + " message received -> " + jobId);
return jobStatusResult;
} else {
log.error("No matched found for subject -> " + subject);
@@ -75,27 +75,4 @@ public class SLURMEmailParser implements EmailParser {
}
}
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
}
http://git-wip-us.apache.org/repos/asf/airavata/blob/4f95a790/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
index 4dff9e1..b2fbcd2 100644
--- a/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
+++ b/modules/gfac/gfac-ssh/src/main/java/org/apache/airavata/gfac/ssh/provider/impl/SSHProvider.java
@@ -382,8 +382,7 @@ public class SSHProvider extends AbstractProvider {
if (monitorMode != null && monitorMode == MonitorMode.JOB_EMAIL_NOTIFICATION_MONITOR) {
EmailMonitorProperty emailMonitorProp = sshJobSubmission.getEmailMonitorProperty();
if (emailMonitorProp != null) {
- EmailMonitorFactory emailMonitorFactory = new EmailMonitorFactory();
- EmailBasedMonitor emailBasedMonitor = emailMonitorFactory.getEmailBasedMonitor(emailMonitorProp);
+ EmailBasedMonitor emailBasedMonitor = EmailMonitorFactory.getEmailBasedMonitor(emailMonitorProp);
emailBasedMonitor.addToJobMonitorMap(jobExecutionContext);
return;
}