You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/04/03 17:22:16 UTC
airavata git commit: Added Email based monitoring classes
Repository: airavata
Updated Branches:
refs/heads/emailMonitoring [created] fb6e3b6e9
Added Email based monitoring classes
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/fb6e3b6e
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/fb6e3b6e
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/fb6e3b6e
Branch: refs/heads/emailMonitoring
Commit: fb6e3b6e97ea5a4958015d55af698811ab8be8cb
Parents: 17b8740
Author: shamrath <sh...@gmail.com>
Authored: Fri Apr 3 11:22:55 2015 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Fri Apr 3 11:22:55 2015 -0400
----------------------------------------------------------------------
.../monitor/impl/mail/EmailBasedMonitor.java | 88 ++++++++++++++++
.../monitor/impl/mail/parser/EmailParser.java | 29 ++++++
.../impl/mail/parser/PBSEmailParser.java | 102 +++++++++++++++++++
.../impl/mail/parser/SlurmEmailParser.java | 72 +++++++++++++
4 files changed, 291 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/fb6e3b6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/EmailBasedMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/EmailBasedMonitor.java
new file mode 100644
index 0000000..740b087
--- /dev/null
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/EmailBasedMonitor.java
@@ -0,0 +1,88 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.mail;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.monitor.core.Monitor;
+import org.apache.airavata.gfac.monitor.impl.mail.parser.EmailParser;
+import org.apache.airavata.gfac.monitor.impl.mail.parser.PBSEmailParser;
+import org.apache.airavata.gfac.monitor.impl.mail.parser.SLURMEmailParser;
+
+import javax.mail.Address;
+import javax.mail.Flags;
+import javax.mail.Folder;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.NoSuchProviderException;
+import javax.mail.Session;
+import javax.mail.Store;
+import javax.mail.search.FlagTerm;
+import java.util.Properties;
+
+public class EmailBasedMonitor implements Monitor {
+
+ private static final String PBS_CONSULT_SDSC_EDU = "pbsconsult@sdsc.edu";
+ private static final String SLURM_BATCH_STAMPEDE = "slurm@batch1.stampede.tacc.utexas.edu";
+
+ private Session session ;
+ private Store store;
+ private Folder emailFolder;
+
+ public void monitorEmails(String host, String username, String password ) throws MessagingException,
+ InterruptedException, AiravataException {
+ Properties properties = new Properties();
+ properties.put("mail.store.protocol", "imaps");
+ session = Session.getDefaultInstance(properties);
+ store = session.getStore("imaps");
+ store.connect(host, username, password);
+ while (!ServerSettings.isStopAllThreads()) {
+ Thread.sleep(2000);
+ emailFolder = store.getFolder("TEST");
+ emailFolder.open(Folder.READ_WRITE);
+ Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
+ for (Message message : searchMessages) {
+ parse(message);
+ }
+ emailFolder.setFlags(searchMessages, new Flags(Flags.Flag.SEEN), true);
+ emailFolder.close(false);
+ }
+ store.close();
+ }
+
+ private void parse(Message message) throws MessagingException, AiravataException {
+ Address fromAddress = message.getFrom()[0];
+ EmailParser emailParser;
+ String addressStr = fromAddress.toString();
+ switch (addressStr) {
+ case PBS_CONSULT_SDSC_EDU:
+ emailParser = new PBSEmailParser();
+ break;
+ case SLURM_BATCH_STAMPEDE:
+ emailParser = new SLURMEmailParser();
+ break;
+ default:
+ throw new AiravataException("Un-handle address type for email monitoring --> " + addressStr);
+ }
+ emailParser.parseEmail(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/fb6e3b6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/EmailParser.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/EmailParser.java
new file mode 100644
index 0000000..d0789c4
--- /dev/null
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/EmailParser.java
@@ -0,0 +1,29 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.mail.parser;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+
+public interface EmailParser {
+
+ void parseEmail(Message message) throws MessagingException;
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/fb6e3b6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/PBSEmailParser.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/PBSEmailParser.java
new file mode 100644
index 0000000..b243cc4
--- /dev/null
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/PBSEmailParser.java
@@ -0,0 +1,102 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.mail.parser;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PBSEmailParser implements EmailParser {
+
+ private static final String STATUS = "status";
+ private static final String JOBID = "jobId";
+ private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<" + STATUS + ">[a-zA-Z\\ ]*)";
+
+ @Override
+ public void parseEmail(Message message) throws MessagingException {
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ String statusLine = matcher.group(STATUS);
+ String jobId = matcher.group(JOBID);
+ switch (statusLine) {
+ case "Begun execution":
+ System.out.println("Begun execution -> " + jobId);
+ break;
+ case "Execution terminated":
+ System.out.println("Execution terminated -> " + jobId);
+ break;
+ default:
+ System.out.println("Un-handle status change -> " + jobId);
+ break;
+ }
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Begun execution
+ */
+/*
+ -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Execution terminated
+ Exit_status=0
+ resources_used.cput=00:14:31
+ resources_used.mem=124712kb
+ resources_used.vmem=3504116kb
+ resources_used.walltime=00:04:10
+ Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
+ Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
+ */
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/fb6e3b6e/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/SlurmEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/SlurmEmailParser.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/SlurmEmailParser.java
new file mode 100644
index 0000000..51bd407
--- /dev/null
+++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/mail/parser/SlurmEmailParser.java
@@ -0,0 +1,72 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.impl.mail.parser;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SLURMEmailParser implements EmailParser {
+
+ private static final String JOBID = "jobId";
+ private static final String STATUS = "status";
+ private static final String REGEX = "[A-Z]*\\s[a-zA-Z]*_[a-z]*=(?<" + JOBID
+ + ">\\d*)\\s[a-zA-Z]*=[a-zA-Z0-9-]*\\s(?<" + STATUS + ">[]a-zA-Z]*),.*";
+
+ @Override
+ public void parseEmail(Message message) throws MessagingException {
+ String subject = message.getSubject();
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(subject);
+ if (matcher.find()) {
+ String jobId = matcher.group(JOBID);
+ String status = matcher.group(STATUS);
+ System.out.println("SLURM " + status + " message received -> " + jobId);
+ } else {
+ System.out.println("No matches found ");
+ }
+ }
+
+
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+}