You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sh...@apache.org on 2015/04/08 18:19:11 UTC
[14/15] airavata git commit: Add new gfac email monitor module to
gfac monitor module and Intorduced gfac-hpc-monitor module to keep all
hoc-monitor code
Add new gfac email monitor module to gfac monitor module and Intorduced gfac-hpc-monitor module to keep all hoc-monitor code
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/6ffc3ee7
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/6ffc3ee7
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/6ffc3ee7
Branch: refs/heads/master
Commit: 6ffc3ee70114f704e6ec7adcdd05c471d98517f0
Parents: 95fae20
Author: shamrath <sh...@gmail.com>
Authored: Wed Apr 8 11:24:32 2015 -0400
Committer: shamrath <sh...@gmail.com>
Committed: Wed Apr 8 11:24:32 2015 -0400
----------------------------------------------------------------------
.../core/monitor/mail/EmailBasedMonitor.java | 224 ---------
.../gfac/core/monitor/mail/JobStatusResult.java | 44 --
.../core/monitor/mail/parser/EmailParser.java | 35 --
.../mail/parser/LonestarEmailParser.java | 74 ---
.../monitor/mail/parser/PBSEmailParser.java | 139 ------
.../monitor/mail/parser/SLURMEmailParser.java | 101 ----
.../gfac-monitor/gfac-email-monitor/pom.xml | 37 ++
.../gfac/monitor/email/EmailBasedMonitor.java | 229 +++++++++
.../gfac/monitor/email/JobStatusResult.java | 44 ++
.../gfac/monitor/email/parser/EmailParser.java | 35 ++
.../email/parser/LonestarEmailParser.java | 74 +++
.../monitor/email/parser/PBSEmailParser.java | 139 ++++++
.../monitor/email/parser/SLURMEmailParser.java | 101 ++++
.../gfac/gfac-monitor/gfac-hpc-monitor/pom.xml | 165 +++++++
.../airavata/gfac/monitor/HPCMonitorID.java | 107 +++++
.../airavata/gfac/monitor/HostMonitorData.java | 88 ++++
.../airavata/gfac/monitor/UserMonitorData.java | 76 +++
.../command/ExperimentCancelRequest.java | 38 ++
.../gfac/monitor/command/TaskCancelRequest.java | 52 ++
.../monitor/core/AiravataAbstractMonitor.java | 46 ++
.../gfac/monitor/core/MessageParser.java | 43 ++
.../airavata/gfac/monitor/core/Monitor.java | 30 ++
.../airavata/gfac/monitor/core/PullMonitor.java | 64 +++
.../airavata/gfac/monitor/core/PushMonitor.java | 60 +++
.../exception/AiravataMonitorException.java | 37 ++
.../handlers/GridPullMonitorHandler.java | 142 ++++++
.../handlers/GridPushMonitorHandler.java | 103 ++++
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 469 +++++++++++++++++++
.../impl/pull/qstat/ResourceConnection.java | 154 ++++++
.../monitor/impl/push/amqp/AMQPMonitor.java | 280 +++++++++++
.../monitor/impl/push/amqp/BasicConsumer.java | 87 ++++
.../impl/push/amqp/JSONMessageParser.java | 78 +++
.../impl/push/amqp/SimpleJobFinishConsumer.java | 86 ++++
.../impl/push/amqp/UnRegisterWorker.java | 67 +++
.../gfac/monitor/util/AMQPConnectionUtil.java | 80 ++++
.../airavata/gfac/monitor/util/CommonUtils.java | 298 ++++++++++++
.../airavata/gfac/monitor/util/X509Helper.java | 164 +++++++
.../src/main/resources/errors.properties | 197 ++++++++
.../src/main/resources/schema/AccessPolicy.json | 13 +
.../src/main/resources/schema/Activity.json | 31 ++
.../src/main/resources/schema/AdminDomain.json | 51 ++
.../schema/ApplicationEnvironment.json | 86 ++++
.../resources/schema/ApplicationHandle.json | 21 +
.../src/main/resources/schema/Benchmark.json | 21 +
.../resources/schema/ComputingActivity.json | 165 +++++++
.../resources/schema/ComputingEndpoint.json | 44 ++
.../main/resources/schema/ComputingManager.json | 117 +++++
.../main/resources/schema/ComputingService.json | 32 ++
.../main/resources/schema/ComputingShare.json | 182 +++++++
.../src/main/resources/schema/Contact.json | 32 ++
.../src/main/resources/schema/DataStore.json | 30 ++
.../src/main/resources/schema/Domain.json | 30 ++
.../src/main/resources/schema/Endpoint.json | 147 ++++++
.../src/main/resources/schema/Entity.json | 35 ++
.../resources/schema/ExecutionEnvironment.json | 115 +++++
.../src/main/resources/schema/Glue2.json | 246 ++++++++++
.../src/main/resources/schema/Location.json | 47 ++
.../src/main/resources/schema/Manager.json | 28 ++
.../main/resources/schema/MappingPolicy.json | 13 +
.../src/main/resources/schema/Policy.json | 27 ++
.../src/main/resources/schema/Resource.json | 27 ++
.../src/main/resources/schema/Service.json | 75 +++
.../src/main/resources/schema/Share.json | 45 ++
.../resources/schema/StorageAccessProtocol.json | 32 ++
.../main/resources/schema/StorageEndpoint.json | 8 +
.../main/resources/schema/StorageManager.json | 8 +
.../main/resources/schema/StorageService.json | 22 +
.../schema/StorageServiceCapacity.json | 33 ++
.../src/main/resources/schema/StorageShare.json | 65 +++
.../resources/schema/StorageShareCapacity.json | 33 ++
.../resources/schema/ToComputingService.json | 32 ++
.../main/resources/schema/ToStorageService.json | 25 +
.../src/main/resources/schema/UserDomain.json | 58 +++
.../src/main/resources/service.properties | 58 +++
.../apache/airavata/job/AMQPMonitorTest.java | 207 ++++++++
.../job/QstatMonitorTestWithMyProxyAuth.java | 172 +++++++
.../src/test/resources/PBSTemplate.xslt | 73 +++
.../src/test/resources/echo.bat | 22 +
.../src/test/resources/logging.properties | 42 ++
modules/gfac/gfac-monitor/pom.xml | 158 +------
.../airavata/gfac/monitor/HPCMonitorID.java | 107 -----
.../airavata/gfac/monitor/HostMonitorData.java | 88 ----
.../airavata/gfac/monitor/UserMonitorData.java | 76 ---
.../command/ExperimentCancelRequest.java | 38 --
.../gfac/monitor/command/TaskCancelRequest.java | 52 --
.../monitor/core/AiravataAbstractMonitor.java | 46 --
.../gfac/monitor/core/MessageParser.java | 43 --
.../airavata/gfac/monitor/core/Monitor.java | 30 --
.../airavata/gfac/monitor/core/PullMonitor.java | 64 ---
.../airavata/gfac/monitor/core/PushMonitor.java | 60 ---
.../exception/AiravataMonitorException.java | 37 --
.../handlers/GridPullMonitorHandler.java | 142 ------
.../handlers/GridPushMonitorHandler.java | 103 ----
.../monitor/impl/pull/qstat/HPCPullMonitor.java | 469 -------------------
.../impl/pull/qstat/ResourceConnection.java | 154 ------
.../monitor/impl/push/amqp/AMQPMonitor.java | 280 -----------
.../monitor/impl/push/amqp/BasicConsumer.java | 87 ----
.../impl/push/amqp/JSONMessageParser.java | 78 ---
.../impl/push/amqp/SimpleJobFinishConsumer.java | 86 ----
.../impl/push/amqp/UnRegisterWorker.java | 67 ---
.../gfac/monitor/util/AMQPConnectionUtil.java | 80 ----
.../airavata/gfac/monitor/util/CommonUtils.java | 298 ------------
.../airavata/gfac/monitor/util/X509Helper.java | 164 -------
.../src/main/resources/errors.properties | 197 --------
.../src/main/resources/schema/AccessPolicy.json | 13 -
.../src/main/resources/schema/Activity.json | 31 --
.../src/main/resources/schema/AdminDomain.json | 51 --
.../schema/ApplicationEnvironment.json | 86 ----
.../resources/schema/ApplicationHandle.json | 21 -
.../src/main/resources/schema/Benchmark.json | 21 -
.../resources/schema/ComputingActivity.json | 165 -------
.../resources/schema/ComputingEndpoint.json | 44 --
.../main/resources/schema/ComputingManager.json | 117 -----
.../main/resources/schema/ComputingService.json | 32 --
.../main/resources/schema/ComputingShare.json | 182 -------
.../src/main/resources/schema/Contact.json | 32 --
.../src/main/resources/schema/DataStore.json | 30 --
.../src/main/resources/schema/Domain.json | 30 --
.../src/main/resources/schema/Endpoint.json | 147 ------
.../src/main/resources/schema/Entity.json | 35 --
.../resources/schema/ExecutionEnvironment.json | 115 -----
.../src/main/resources/schema/Glue2.json | 246 ----------
.../src/main/resources/schema/Location.json | 47 --
.../src/main/resources/schema/Manager.json | 28 --
.../main/resources/schema/MappingPolicy.json | 13 -
.../src/main/resources/schema/Policy.json | 27 --
.../src/main/resources/schema/Resource.json | 27 --
.../src/main/resources/schema/Service.json | 75 ---
.../src/main/resources/schema/Share.json | 45 --
.../resources/schema/StorageAccessProtocol.json | 32 --
.../main/resources/schema/StorageEndpoint.json | 8 -
.../main/resources/schema/StorageManager.json | 8 -
.../main/resources/schema/StorageService.json | 22 -
.../schema/StorageServiceCapacity.json | 33 --
.../src/main/resources/schema/StorageShare.json | 65 ---
.../resources/schema/StorageShareCapacity.json | 33 --
.../resources/schema/ToComputingService.json | 32 --
.../main/resources/schema/ToStorageService.json | 25 -
.../src/main/resources/schema/UserDomain.json | 58 ---
.../src/main/resources/service.properties | 58 ---
.../apache/airavata/job/AMQPMonitorTest.java | 207 --------
.../job/QstatMonitorTestWithMyProxyAuth.java | 172 -------
.../src/test/resources/PBSTemplate.xslt | 73 ---
.../gfac-monitor/src/test/resources/echo.bat | 22 -
.../src/test/resources/logging.properties | 42 --
modules/gfac/gfac-ssh/pom.xml | 7 +
.../gfac/ssh/provider/impl/SSHProvider.java | 2 +-
147 files changed, 6235 insertions(+), 6165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
deleted file mode 100644
index f15e582..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/EmailBasedMonitor.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.common.exception.ApplicationSettingsException;
-import org.apache.airavata.common.logger.AiravataLogger;
-import org.apache.airavata.common.logger.AiravataLoggerFactory;
-import org.apache.airavata.common.utils.MonitorPublisher;
-import org.apache.airavata.common.utils.ServerSettings;
-import org.apache.airavata.gfac.core.context.JobExecutionContext;
-import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
-import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
-import org.apache.airavata.gfac.core.monitor.mail.parser.EmailParser;
-import org.apache.airavata.gfac.core.monitor.mail.parser.PBSEmailParser;
-import org.apache.airavata.gfac.core.monitor.mail.parser.SLURMEmailParser;
-import org.apache.airavata.model.messaging.event.JobIdentifier;
-import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.apache.airavata.model.workspace.experiment.JobStatus;
-
-import javax.mail.Address;
-import javax.mail.Flags;
-import javax.mail.Folder;
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import javax.mail.Session;
-import javax.mail.Store;
-import javax.mail.search.FlagTerm;
-import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class EmailBasedMonitor implements Runnable{
-
- private static final AiravataLogger log = AiravataLoggerFactory.getLogger(EmailBasedMonitor.class);
-
- private static final String PBS_CONSULT_SDSC_EDU = "pbsconsult@sdsc.edu";
- private static final String SLURM_BATCH_STAMPEDE = "slurm@batch1.stampede.tacc.utexas.edu";
- private static EmailBasedMonitor emailBasedMonitor;
- private final MonitorPublisher monitorPublisher;
-
- private Session session ;
- private Store store;
- private Folder emailFolder;
- private String host, emailAddress, password, folderName, mailStoreProtocol;
- private Properties properties;
-
- private Map<String, JobExecutionContext> jobMonitorMap = new ConcurrentHashMap<String, JobExecutionContext>();
-
- private EmailBasedMonitor(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
- this.monitorPublisher = monitorPublisher;
- init();
- }
-
- private void init() throws ApplicationSettingsException {
- host = ServerSettings.getEmailBasedMonitorHost();
- emailAddress = ServerSettings.getEmailBasedMonitorAddress();
- password = ServerSettings.getEmailBasedMonitorPassword();
- folderName = ServerSettings.getEmailBasedMonitorFolderName();
- mailStoreProtocol = ServerSettings.getEmailBasedMonitorStoreProtocol();
-
- properties = new Properties();
- properties.put("mail.store.protocol", mailStoreProtocol);
-
- }
-
- public static EmailBasedMonitor getInstant(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
- if (emailBasedMonitor == null) {
- synchronized (EmailBasedMonitor.class) {
- if (emailBasedMonitor == null) {
- emailBasedMonitor = new EmailBasedMonitor(monitorPublisher);
- Thread thread = new Thread(emailBasedMonitor);
- thread.start();
- }
- }
- }
-
- return emailBasedMonitor;
- }
-
- public void addToJobMonitorMap(JobExecutionContext jobExecutionContext) {
- addToJobMonitorMap(jobExecutionContext.getJobDetails().getJobID(), jobExecutionContext);
- }
-
- public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) {
- log.info("Added Job Id : " + jobId + " to email based monitor map");
- jobMonitorMap.put(jobId, jobExecutionContext);
- }
-
- private JobStatusResult parse(Message message) throws MessagingException, AiravataException {
- Address fromAddress = message.getFrom()[0];
- EmailParser emailParser;
- String addressStr = fromAddress.toString();
- switch (addressStr) {
- case PBS_CONSULT_SDSC_EDU:
- emailParser = new PBSEmailParser();
- break;
- case SLURM_BATCH_STAMPEDE:
- emailParser = new SLURMEmailParser();
- break;
- default:
- throw new AiravataException("Un-handle address type for email monitoring --> " + addressStr);
- }
- return emailParser.parseEmail(message);
- }
-
- @Override
- public void run() {
- try {
- session = Session.getDefaultInstance(properties);
- store = session.getStore(mailStoreProtocol);
- store.connect(host, emailAddress, password);
- while (!ServerSettings.isStopAllThreads()) {
- if (!store.isConnected()) {
- store.connect();
- }
- Thread.sleep(2000);
- emailFolder = store.getFolder(folderName);
- emailFolder.open(Folder.READ_WRITE);
- Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
- for (Message message : searchMessages) {
- try {
- JobStatusResult jobStatusResult = parse(message);
- process(jobStatusResult);
- } catch (AiravataException e) {
- log.error("Error parsing email message =====================================>", e);
- try {
- writeEnvelopeOnError(message);
- } catch (MessagingException e1) {
- log.error("Error printing envelop of the email");
- }
-
- }
- }
- emailFolder.setFlags(searchMessages, new Flags(Flags.Flag.SEEN), true);
- emailFolder.close(false);
- }
-
- } catch (MessagingException e) {
- log.error("Couldn't connect to the store ", e);
- } catch (InterruptedException e) {
- log.error("Interrupt exception while sleep ", e);
- } finally {
- try {
- store.close();
- } catch (MessagingException e) {
- log.error("Store close operation failed, couldn't close store", e);
- }
- }
- }
-
- private void process(JobStatusResult jobStatusResult) throws AiravataException {
- JobExecutionContext jEC = jobMonitorMap.get(jobStatusResult.getJobId());
- if (jEC == null) {
- throw new AiravataException("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
- }
- JobState resultState = jobStatusResult.getState();
- jEC.getJobDetails().setJobStatus(new JobStatus(resultState));
- if (resultState == JobState.COMPLETE) {
- GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(jEC, monitorPublisher));
- }else if (resultState == JobState.QUEUED) {
- // TODO - publish queued rabbitmq message
- }else if (resultState == JobState.FAILED) {
- // TODO - handle failed scenario
- jobMonitorMap.remove(jobStatusResult.getJobId());
- log.info("Job failed email received , removed job from job monitoring");
-// monitorPublisher.publish(jEC.getJobDetails().getJobStatus());
- }
- publishJobStatusChange(jEC);
- }
-
- private void publishJobStatusChange(JobExecutionContext jobExecutionContext) {
- JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
- JobIdentifier jobIdentity = new JobIdentifier(jobExecutionContext.getJobDetails().getJobID(),
- jobExecutionContext.getTaskData().getTaskID(),
- jobExecutionContext.getWorkflowNodeDetails().getNodeInstanceId(),
- jobExecutionContext.getExperimentID(),
- jobExecutionContext.getGatewayID());
- jobStatus.setJobIdentity(jobIdentity);
- jobStatus.setState(jobExecutionContext.getJobDetails().getJobStatus().getJobState());
- // we have this JobStatus class to handle amqp monitoring
- log.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
- "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
- jobStatus.getJobIdentity().getTaskId());
-
- monitorPublisher.publish(jobStatus);
- }
-
- private void writeEnvelopeOnError(Message m) throws MessagingException {
- Address[] a;
- // FROM
- if ((a = m.getFrom()) != null) {
- for (int j = 0; j < a.length; j++)
- log.error("FROM: " + a[j].toString());
- }
- // TO
- if ((a = m.getRecipients(Message.RecipientType.TO)) != null) {
- for (int j = 0; j < a.length; j++)
- log.error("TO: " + a[j].toString());
- }
- // SUBJECT
- if (m.getSubject() != null)
- log.error("SUBJECT: " + m.getSubject());
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
deleted file mode 100644
index 0c549a6..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/JobStatusResult.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail;
-
-import org.apache.airavata.model.workspace.experiment.JobState;
-
-public class JobStatusResult {
- private JobState state;
- private String jobId;
-
- public JobState getState() {
- return state;
- }
-
- public void setState(JobState state) {
- this.state = state;
- }
-
- public String getJobId() {
- return jobId;
- }
-
- public void setJobId(String jobId) {
- this.jobId = jobId;
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
deleted file mode 100644
index ad8e4a3..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/EmailParser.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-
-public interface EmailParser {
- static final String STATUS = "status";
- static final String JOBID = "jobId";
- static final String EXIT_STATUS = "exitStatus";
-
- JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
deleted file mode 100644
index 59651da..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/LonestarEmailParser.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class LonestarEmailParser implements EmailParser {
- private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class);
- //root@c312-206.ls4.tacc.utexas.edu
- private static final String SIGNAL = "signal";
- private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS
- + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
- JobStatusResult jobStatusResult = new JobStatusResult();
- try {
- String content = ((String) message.getContent());
- Pattern pattern = Pattern.compile(LONESTAR_REGEX);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- String status = matcher.group(STATUS);
- jobStatusResult.setState(getJobState(status, content));
- return jobStatusResult;
- } else {
- log.error("No matched found for content => \n" + content);
- }
- } catch (IOException e) {
- throw new AiravataException("Error while reading content of the email message");
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String status, String content) {
- switch (status) {
- case "Aborted":
- return JobState.FAILED;
- case "Scucess":
- return JobState.COMPLETE;
- default:
- return JobState.UNKNOWN;
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
deleted file mode 100644
index cd6142c..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/PBSEmailParser.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class PBSEmailParser implements EmailParser {
-
- private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class);
-
-
- private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<"
- + STATUS + ">[a-zA-Z\\ ]*)";
- private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
- JobStatusResult jobStatusResult = new JobStatusResult();
- try {
- String content = ((String) message.getContent());
- Pattern pattern = Pattern.compile(REGEX);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- String statusLine = matcher.group(STATUS);
- jobStatusResult.setState(getJobState(statusLine, content));
- return jobStatusResult;
- } else {
- log.error("No matched found for content => \n" + content);
- }
-
- } catch (IOException e) {
- throw new AiravataException("Error while reading content of the email message");
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String statusLine, String content) {
- switch (statusLine) {
- case "Begun execution":
- return JobState.QUEUED;
- case "Execution terminated":
- int exitStatus = getExitStatus(content);
- switch (exitStatus) {
- case 0:
- return JobState.COMPLETE;
- case 1:
- return JobState.FAILED;
- default:
- return JobState.UNKNOWN;
- }
- default:
- return JobState.UNKNOWN;
- }
- }
-
- private int getExitStatus(String content) {
- Pattern pattern = Pattern.compile(REGEX_EXIT_STATUS);
- Matcher matcher = pattern.matcher(content);
- if (matcher.find()) {
- String group = matcher.group(EXIT_STATUS);
- if (group != null && !group.trim().isEmpty()) {
- return Integer.valueOf(group.trim());
- }
- }
- return -1;
- }
-
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Begun execution
- */
-/*
- -----------------------
- This is the message envelope
- ---------------------------
- FROM: pbsconsult@sdsc.edu
- TO: shameera@scigap.org
- SUBJECT: PBS JOB 2556782.trestles-fe1.local
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN
- This is plain text
- ---------------------------
- PBS Job Id: 2556782.trestles-fe1.local
- Job Name: A1182004055
- Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
- Execution terminated
- Exit_status=0
- resources_used.cput=00:14:31
- resources_used.mem=124712kb
- resources_used.vmem=3504116kb
- resources_used.walltime=00:04:10
- Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
- Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
- */
-
-
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
deleted file mode 100644
index ceb0013..0000000
--- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/monitor/mail/parser/SLURMEmailParser.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
-*/
-package org.apache.airavata.gfac.core.monitor.mail.parser;
-
-import org.apache.airavata.common.exception.AiravataException;
-import org.apache.airavata.gfac.core.monitor.mail.JobStatusResult;
-import org.apache.airavata.model.workspace.experiment.JobState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.mail.Message;
-import javax.mail.MessagingException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class SLURMEmailParser implements EmailParser {
-
- private static final Logger log = LoggerFactory.getLogger(SLURMEmailParser.class);
-
- private static final String JOBID = "jobId";
- private static final String STATUS = "status";
- private static final String REGEX = "[A-Z]*\\s[a-zA-Z]*_[a-z]*=(?<" + JOBID
- + ">\\d*)\\s[a-zA-Z]*=[a-zA-Z0-9-]*\\s(?<" + STATUS + ">[]a-zA-Z]*),.*";
-
- @Override
- public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
- JobStatusResult jobStatusResult = new JobStatusResult();
- String subject = message.getSubject();
- Pattern pattern = Pattern.compile(REGEX);
- Matcher matcher = pattern.matcher(subject);
- if (matcher.find()) {
- jobStatusResult.setJobId(matcher.group(JOBID));
- jobStatusResult.setState(getJobState(matcher.group(STATUS)));
- // TODO remove following test lines
- String jobId = matcher.group(JOBID);
- String status = matcher.group(STATUS);
- log.info("SLURM " + status + " message received -> " + jobId);
- return jobStatusResult;
- } else {
- log.error("No matched found for subject -> " + subject);
- }
- return jobStatusResult;
- }
-
- private JobState getJobState(String state) {
- switch (state.trim()) {
- case "Began":
- return JobState.QUEUED;
- case "Ended":
- return JobState.COMPLETE;
- case "Failed":
- return JobState.FAILED;
- default:
- log.error("Job State " + state + " isn't handle by SLURM parser");
- return JobState.UNKNOWN;
-
- }
- }
-
-
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
-
-/* -----------------------
- This is the message envelope
- ---------------------------
- FROM: slurm@batch1.stampede.tacc.utexas.edu
- TO: shameera@scigap.org
- SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
- ----------------------------
- CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
- This is plain text
- ---------------------------*/
-}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml b/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
new file mode 100644
index 0000000..0bff287
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/pom.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>airavata-gfac-monitor</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.15-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>airavata-gfac-email-monitor</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>javax.mail</groupId>
+ <artifactId>mail</artifactId>
+ <version>1.4.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-common-utils</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-data-models</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
new file mode 100644
index 0000000..6e12d83
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/EmailBasedMonitor.java
@@ -0,0 +1,229 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.common.exception.ApplicationSettingsException;
+import org.apache.airavata.common.logger.AiravataLogger;
+import org.apache.airavata.common.logger.AiravataLoggerFactory;
+import org.apache.airavata.common.utils.MonitorPublisher;
+import org.apache.airavata.common.utils.ServerSettings;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.utils.GFacThreadPoolExecutor;
+import org.apache.airavata.gfac.core.utils.OutHandlerWorker;
+import org.apache.airavata.gfac.monitor.email.parser.EmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.LonestarEmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser;
+import org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser;
+import org.apache.airavata.model.messaging.event.JobIdentifier;
+import org.apache.airavata.model.messaging.event.JobStatusChangeRequestEvent;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.apache.airavata.model.workspace.experiment.JobStatus;
+
+import javax.mail.Address;
+import javax.mail.Flags;
+import javax.mail.Folder;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.Session;
+import javax.mail.Store;
+import javax.mail.search.FlagTerm;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class EmailBasedMonitor implements Runnable{
+
+ private static final AiravataLogger log = AiravataLoggerFactory.getLogger(EmailBasedMonitor.class);
+
+ private static final String PBS_CONSULT_SDSC_EDU = "pbsconsult@sdsc.edu";
+ private static final String SLURM_BATCH_STAMPEDE = "slurm@batch1.stampede.tacc.utexas.edu";
+ private static final String LONESTAR_ADDRESS = "root@c312-206.ls4.tacc.utexas.edu";
+ private static EmailBasedMonitor emailBasedMonitor;
+ private final MonitorPublisher monitorPublisher;
+
+ private Session session ;
+ private Store store;
+ private Folder emailFolder;
+ private String host, emailAddress, password, folderName, mailStoreProtocol;
+ private Properties properties;
+
+ private Map<String, JobExecutionContext> jobMonitorMap = new ConcurrentHashMap<String, JobExecutionContext>();
+
+ private EmailBasedMonitor(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
+ this.monitorPublisher = monitorPublisher;
+ init();
+ }
+
+ private void init() throws ApplicationSettingsException {
+ host = ServerSettings.getEmailBasedMonitorHost();
+ emailAddress = ServerSettings.getEmailBasedMonitorAddress();
+ password = ServerSettings.getEmailBasedMonitorPassword();
+ folderName = ServerSettings.getEmailBasedMonitorFolderName();
+ mailStoreProtocol = ServerSettings.getEmailBasedMonitorStoreProtocol();
+
+ properties = new Properties();
+ properties.put("mail.store.protocol", mailStoreProtocol);
+
+ }
+
+ public static EmailBasedMonitor getInstant(MonitorPublisher monitorPublisher) throws ApplicationSettingsException {
+ if (emailBasedMonitor == null) {
+ synchronized (EmailBasedMonitor.class) {
+ if (emailBasedMonitor == null) {
+ emailBasedMonitor = new EmailBasedMonitor(monitorPublisher);
+ Thread thread = new Thread(emailBasedMonitor);
+ thread.start();
+ }
+ }
+ }
+
+ return emailBasedMonitor;
+ }
+
+ public void addToJobMonitorMap(JobExecutionContext jobExecutionContext) {
+ addToJobMonitorMap(jobExecutionContext.getJobDetails().getJobID(), jobExecutionContext);
+ }
+
+ public void addToJobMonitorMap(String jobId, JobExecutionContext jobExecutionContext) {
+ log.info("Added Job Id : " + jobId + " to email based monitor map");
+ jobMonitorMap.put(jobId, jobExecutionContext);
+ }
+
+ private JobStatusResult parse(Message message) throws MessagingException, AiravataException {
+ Address fromAddress = message.getFrom()[0];
+ EmailParser emailParser;
+ String addressStr = fromAddress.toString();
+ switch (addressStr) {
+ case PBS_CONSULT_SDSC_EDU:
+ emailParser = new PBSEmailParser();
+ break;
+ case SLURM_BATCH_STAMPEDE:
+ emailParser = new SLURMEmailParser();
+ break;
+ case LONESTAR_ADDRESS:
+ emailParser = new LonestarEmailParser();
+ break;
+ default:
+ throw new AiravataException("Un-handle address type for email monitoring --> " + addressStr);
+ }
+ return emailParser.parseEmail(message);
+ }
+
+ @Override
+ public void run() {
+ try {
+ session = Session.getDefaultInstance(properties);
+ store = session.getStore(mailStoreProtocol);
+ store.connect(host, emailAddress, password);
+ while (!ServerSettings.isStopAllThreads()) {
+ if (!store.isConnected()) {
+ store.connect();
+ }
+ Thread.sleep(2000);
+ emailFolder = store.getFolder(folderName);
+ emailFolder.open(Folder.READ_WRITE);
+ Message[] searchMessages = emailFolder.search(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
+ for (Message message : searchMessages) {
+ try {
+ JobStatusResult jobStatusResult = parse(message);
+ process(jobStatusResult);
+ } catch (AiravataException e) {
+ log.error("Error parsing email message =====================================>", e);
+ try {
+ writeEnvelopeOnError(message);
+ } catch (MessagingException e1) {
+ log.error("Error printing envelop of the email");
+ }
+
+ }
+ }
+ emailFolder.setFlags(searchMessages, new Flags(Flags.Flag.SEEN), true);
+ emailFolder.close(false);
+ }
+
+ } catch (MessagingException e) {
+ log.error("Couldn't connect to the store ", e);
+ } catch (InterruptedException e) {
+ log.error("Interrupt exception while sleep ", e);
+ } finally {
+ try {
+ store.close();
+ } catch (MessagingException e) {
+ log.error("Store close operation failed, couldn't close store", e);
+ }
+ }
+ }
+
+ private void process(JobStatusResult jobStatusResult) throws AiravataException {
+ JobExecutionContext jEC = jobMonitorMap.get(jobStatusResult.getJobId());
+ if (jEC == null) {
+ throw new AiravataException("JobExecutionContext is not found for job Id " + jobStatusResult.getJobId());
+ }
+ JobState resultState = jobStatusResult.getState();
+ jEC.getJobDetails().setJobStatus(new JobStatus(resultState));
+ if (resultState == JobState.COMPLETE) {
+ GFacThreadPoolExecutor.getFixedThreadPool().submit(new OutHandlerWorker(jEC, monitorPublisher));
+ }else if (resultState == JobState.QUEUED) {
+ // TODO - publish queued rabbitmq message
+ }else if (resultState == JobState.FAILED) {
+ // TODO - handle failed scenario
+ jobMonitorMap.remove(jobStatusResult.getJobId());
+ log.info("Job failed email received , removed job from job monitoring");
+// monitorPublisher.publish(jEC.getJobDetails().getJobStatus());
+ }
+ publishJobStatusChange(jEC);
+ }
+
+ private void publishJobStatusChange(JobExecutionContext jobExecutionContext) {
+ JobStatusChangeRequestEvent jobStatus = new JobStatusChangeRequestEvent();
+ JobIdentifier jobIdentity = new JobIdentifier(jobExecutionContext.getJobDetails().getJobID(),
+ jobExecutionContext.getTaskData().getTaskID(),
+ jobExecutionContext.getWorkflowNodeDetails().getNodeInstanceId(),
+ jobExecutionContext.getExperimentID(),
+ jobExecutionContext.getGatewayID());
+ jobStatus.setJobIdentity(jobIdentity);
+ jobStatus.setState(jobExecutionContext.getJobDetails().getJobStatus().getJobState());
+ // we have this JobStatus class to handle amqp monitoring
+ log.debugId(jobStatus.getJobIdentity().getJobId(), "Published job status change request, " +
+ "experiment {} , task {}", jobStatus.getJobIdentity().getExperimentId(),
+ jobStatus.getJobIdentity().getTaskId());
+
+ monitorPublisher.publish(jobStatus);
+ }
+
+ private void writeEnvelopeOnError(Message m) throws MessagingException {
+ Address[] a;
+ // FROM
+ if ((a = m.getFrom()) != null) {
+ for (int j = 0; j < a.length; j++)
+ log.error("FROM: " + a[j].toString());
+ }
+ // TO
+ if ((a = m.getRecipients(Message.RecipientType.TO)) != null) {
+ for (int j = 0; j < a.length; j++)
+ log.error("TO: " + a[j].toString());
+ }
+ // SUBJECT
+ if (m.getSubject() != null)
+ log.error("SUBJECT: " + m.getSubject());
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
new file mode 100644
index 0000000..9145993
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/JobStatusResult.java
@@ -0,0 +1,44 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email;
+
+import org.apache.airavata.model.workspace.experiment.JobState;
+
+public class JobStatusResult {
+ private JobState state;
+ private String jobId;
+
+ public JobState getState() {
+ return state;
+ }
+
+ public void setState(JobState state) {
+ this.state = state;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
new file mode 100644
index 0000000..84899f0
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/EmailParser.java
@@ -0,0 +1,35 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+
+public interface EmailParser {
+ static final String STATUS = "status";
+ static final String JOBID = "jobId";
+ static final String EXIT_STATUS = "exitStatus";
+
+ JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException;
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
new file mode 100644
index 0000000..f02a032
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/LonestarEmailParser.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LonestarEmailParser implements EmailParser {
+ private static final Logger log = LoggerFactory.getLogger(LonestarEmailParser.class);
+ //root@c312-206.ls4.tacc.utexas.edu
+ private static final String SIGNAL = "signal";
+ private static final String LONESTAR_REGEX = "Job (?<" + JOBID + ">\\d+) \\(.*\\) (?<" + STATUS
+ + ">.*)\\s[a-zA-Z =]+(?<" + EXIT_STATUS + ">\\d+)\\sSignal[ ]*=[ ]*(?<" + SIGNAL + ">[a-zA-z]*)";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(LONESTAR_REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ String status = matcher.group(STATUS);
+ jobStatusResult.setState(getJobState(status, content));
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for content => \n" + content);
+ }
+ } catch (IOException e) {
+ throw new AiravataException("Error while reading content of the email message");
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String status, String content) {
+ switch (status) {
+ case "Aborted":
+ return JobState.FAILED;
+ case "Success":
+ return JobState.COMPLETE;
+ default:
+ return JobState.UNKNOWN;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
new file mode 100644
index 0000000..b748704
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/PBSEmailParser.java
@@ -0,0 +1,139 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PBSEmailParser implements EmailParser {
+
+ private static final Logger log = LoggerFactory.getLogger(PBSEmailParser.class);
+
+
+ private static final String REGEX = "[a-zA-Z: ]*(?<" + JOBID + ">[a-zA-Z0-9-\\.]*)\\s+.*\\s+.*\\s+(?<"
+ + STATUS + ">[a-zA-Z\\ ]*)";
+ private static final String REGEX_EXIT_STATUS = "Exit_status=(?<" + EXIT_STATUS + ">[\\d]+)";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException {
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ try {
+ String content = ((String) message.getContent());
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ String statusLine = matcher.group(STATUS);
+ jobStatusResult.setState(getJobState(statusLine, content));
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for content => \n" + content);
+ }
+
+ } catch (IOException e) {
+ throw new AiravataException("Error while reading content of the email message");
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String statusLine, String content) {
+ switch (statusLine) {
+ case "Begun execution":
+ return JobState.QUEUED;
+ case "Execution terminated":
+ int exitStatus = getExitStatus(content);
+ switch (exitStatus) {
+ case 0:
+ return JobState.COMPLETE;
+ case 1:
+ return JobState.FAILED;
+ default:
+ return JobState.UNKNOWN;
+ }
+ default:
+ return JobState.UNKNOWN;
+ }
+ }
+
+ private int getExitStatus(String content) {
+ Pattern pattern = Pattern.compile(REGEX_EXIT_STATUS);
+ Matcher matcher = pattern.matcher(content);
+ if (matcher.find()) {
+ String group = matcher.group(EXIT_STATUS);
+ if (group != null && !group.trim().isEmpty()) {
+ return Integer.valueOf(group.trim());
+ }
+ }
+ return -1;
+ }
+
+
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Begun execution
+ */
+/*
+ -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: pbsconsult@sdsc.edu
+ TO: shameera@scigap.org
+ SUBJECT: PBS JOB 2556782.trestles-fe1.local
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN
+ This is plain text
+ ---------------------------
+ PBS Job Id: 2556782.trestles-fe1.local
+ Job Name: A1182004055
+ Exec host: trestles-1-12/0+trestles-1-12/1+trestles-1-12/2+trestles-1-12/3
+ Execution terminated
+ Exit_status=0
+ resources_used.cput=00:14:31
+ resources_used.mem=124712kb
+ resources_used.vmem=3504116kb
+ resources_used.walltime=00:04:10
+ Error_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stderr
+ Output_Path: trestles-login2.sdsc.edu:/oasis/scratch/trestles/ogce/temp_project/gta-work-dirs/MonitorTest_9169517d-e2d9-4ff5-bed5-dee6eb3eebb2/Amber_Sander.stdout
+ */
+
+
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
new file mode 100644
index 0000000..643265e
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-email-monitor/src/main/java/org/apache/airavata/gfac/monitor/email/parser/SLURMEmailParser.java
@@ -0,0 +1,101 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor.email.parser;
+
+import org.apache.airavata.common.exception.AiravataException;
+import org.apache.airavata.gfac.monitor.email.JobStatusResult;
+import org.apache.airavata.model.workspace.experiment.JobState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class SLURMEmailParser implements EmailParser {
+
+ private static final Logger log = LoggerFactory.getLogger(SLURMEmailParser.class);
+
+ private static final String JOBID = "jobId";
+ private static final String STATUS = "status";
+ private static final String REGEX = "[A-Z]*\\s[a-zA-Z]*_[a-z]*=(?<" + JOBID
+ + ">\\d*)\\s[a-zA-Z]*=[a-zA-Z0-9-]*\\s(?<" + STATUS + ">[]a-zA-Z]*),.*";
+
+ @Override
+ public JobStatusResult parseEmail(Message message) throws MessagingException, AiravataException{
+ JobStatusResult jobStatusResult = new JobStatusResult();
+ String subject = message.getSubject();
+ Pattern pattern = Pattern.compile(REGEX);
+ Matcher matcher = pattern.matcher(subject);
+ if (matcher.find()) {
+ jobStatusResult.setJobId(matcher.group(JOBID));
+ jobStatusResult.setState(getJobState(matcher.group(STATUS)));
+ // TODO remove following test lines
+ String jobId = matcher.group(JOBID);
+ String status = matcher.group(STATUS);
+ log.info("SLURM " + status + " message received -> " + jobId);
+ return jobStatusResult;
+ } else {
+ log.error("No matched found for subject -> " + subject);
+ }
+ return jobStatusResult;
+ }
+
+ private JobState getJobState(String state) {
+ switch (state.trim()) {
+ case "Began":
+ return JobState.QUEUED;
+ case "Ended":
+ return JobState.COMPLETE;
+ case "Failed":
+ return JobState.FAILED;
+ default:
+ log.error("Job State " + state + " isn't handle by SLURM parser");
+ return JobState.UNKNOWN;
+
+ }
+ }
+
+
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Began, Queued time 00:00:01
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+
+/* -----------------------
+ This is the message envelope
+ ---------------------------
+ FROM: slurm@batch1.stampede.tacc.utexas.edu
+ TO: shameera@scigap.org
+ SUBJECT: SLURM Job_id=5055468 Name=A433255759 Ended, Run time 00:02:40
+ ----------------------------
+ CONTENT-TYPE: TEXT/PLAIN; charset=us-ascii
+ This is plain text
+ ---------------------------*/
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml b/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
new file mode 100644
index 0000000..a3e15a2
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/pom.xml
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>airavata-gfac-monitor</artifactId>
+ <groupId>org.apache.airavata</groupId>
+ <version>0.15-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>airavata-gfac-hpc-monitor</artifactId>
+ <name>Airavata GFac Grid Job Monitor</name>
+ <description>The Grid related monitoring implementation</description>
+ <url>http://airavata.apache.org/</url>
+
+ <dependencies>
+ <!-- Logging -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <!-- GFAC schemas -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-gsissh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-gfac-ssh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-workflow-execution-context</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-registry-cpi</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-jpa-registry</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Workflow Tracking -->
+ <!--<dependency>-->
+ <!--<groupId>org.apache.airavata</groupId>-->
+ <!--<artifactId>airavata-workflow-tracking</artifactId>-->
+ <!--<version>${project.version}</version>-->
+ <!--</dependency>-->
+ <!-- Credential Store -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-credential-store</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Test -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-server-configuration</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-client-configuration</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- Guava -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>12.0</version>
+ </dependency>
+ <!-- gsi-ssh api dependencies -->
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>gsissh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.airavata</groupId>
+ <artifactId>airavata-data-models</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.50</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.xmlbeans</groupId>
+ <artifactId>xmlbeans</artifactId>
+ <version>${xmlbeans.version}</version>
+ </dependency>
+ <!-- this is the dependency for amqp implementation -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.0.0</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skip>false</skip>
+ <forkMode>always</forkMode>
+ <failIfNoTests>false</failIfNoTests>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <version>0.4.0</version>
+ <configuration>
+ <sourceDirectory>${basedir}/src/main/resources/schema</sourceDirectory>
+ <targetPackage>org.apache.airavata</targetPackage>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
new file mode 100644
index 0000000..ae463a7
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HPCMonitorID.java
@@ -0,0 +1,107 @@
+package org.apache.airavata.gfac.monitor;/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+import org.apache.airavata.gfac.GFacException;
+import org.apache.airavata.gfac.SecurityContext;
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.gsissh.security.GSISecurityContext;
+import org.apache.airavata.gfac.ssh.security.SSHSecurityContext;
+import org.apache.airavata.gsi.ssh.api.ServerInfo;
+import org.apache.airavata.gsi.ssh.api.authentication.AuthenticationInfo;
+import org.apache.airavata.gsi.ssh.impl.authentication.MyProxyAuthenticationInfo;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Timestamp;
+import java.util.Date;
+
+public class HPCMonitorID extends MonitorID {
+ private final static Logger logger = LoggerFactory.getLogger(HPCMonitorID.class);
+
+
+ private AuthenticationInfo authenticationInfo = null;
+
+ public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID,
+ String experimentID, String userName,String jobName) {
+ super(computeResourceDescription, jobID, taskID, workflowNodeID, experimentID, userName,jobName);
+ setComputeResourceDescription(computeResourceDescription);
+ setJobStartedTime(new Timestamp((new Date()).getTime()));
+ setUserName(userName);
+ setJobID(jobID);
+ setTaskID(taskID);
+ setExperimentID(experimentID);
+ setWorkflowNodeID(workflowNodeID);
+ }
+
+ public HPCMonitorID(AuthenticationInfo authenticationInfo, JobExecutionContext jobExecutionContext) {
+ super(jobExecutionContext);
+ this.authenticationInfo = authenticationInfo;
+ if (this.authenticationInfo != null) {
+ try {
+ String hostAddress = jobExecutionContext.getHostName();
+ SecurityContext securityContext = jobExecutionContext.getSecurityContext(hostAddress);
+ ServerInfo serverInfo = null;
+ if (securityContext != null) {
+ if (securityContext instanceof GSISecurityContext){
+ serverInfo = (((GSISecurityContext) securityContext).getPbsCluster()).getServerInfo();
+ if (serverInfo.getUserName() != null) {
+ setUserName(serverInfo.getUserName());
+ }
+ }
+ if (securityContext instanceof SSHSecurityContext){
+ serverInfo = (((SSHSecurityContext) securityContext).getPbsCluster()).getServerInfo();
+ if (serverInfo.getUserName() != null) {
+ setUserName(serverInfo.getUserName());
+ }
+ }
+ }
+ } catch (GFacException e) {
+ logger.error("Error while getting security context", e);
+ }
+ }
+ }
+
+ public HPCMonitorID(ComputeResourceDescription computeResourceDescription, String jobID, String taskID, String workflowNodeID, String experimentID, String userName, AuthenticationInfo authenticationInfo) {
+ setComputeResourceDescription(computeResourceDescription);
+ setJobStartedTime(new Timestamp((new Date()).getTime()));
+ this.authenticationInfo = authenticationInfo;
+ // if we give myproxyauthenticationInfo, so we try to use myproxy user as the user
+ if (this.authenticationInfo != null) {
+ if (this.authenticationInfo instanceof MyProxyAuthenticationInfo) {
+ setUserName(((MyProxyAuthenticationInfo) this.authenticationInfo).getUserName());
+ }
+ }
+ setJobID(jobID);
+ setTaskID(taskID);
+ setExperimentID(experimentID);
+ setWorkflowNodeID(workflowNodeID);
+ }
+
+ public AuthenticationInfo getAuthenticationInfo() {
+ return authenticationInfo;
+ }
+
+ public void setAuthenticationInfo(AuthenticationInfo authenticationInfo) {
+ this.authenticationInfo = authenticationInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/airavata/blob/6ffc3ee7/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
----------------------------------------------------------------------
diff --git a/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
new file mode 100644
index 0000000..f29e3e6
--- /dev/null
+++ b/modules/gfac/gfac-monitor/gfac-hpc-monitor/src/main/java/org/apache/airavata/gfac/monitor/HostMonitorData.java
@@ -0,0 +1,88 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+package org.apache.airavata.gfac.monitor;
+
+import org.apache.airavata.gfac.core.context.JobExecutionContext;
+import org.apache.airavata.gfac.core.monitor.MonitorID;
+import org.apache.airavata.gfac.monitor.exception.AiravataMonitorException;
+import org.apache.airavata.model.appcatalog.computeresource.ComputeResourceDescription;
+import org.apache.airavata.model.appcatalog.computeresource.DataMovementProtocol;
+import org.apache.airavata.model.appcatalog.computeresource.JobSubmissionProtocol;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class HostMonitorData {
+// private HostDescription host;
+ private ComputeResourceDescription computeResourceDescription;
+ private JobSubmissionProtocol jobSubmissionProtocol;
+ private DataMovementProtocol dataMovementProtocol;
+
+ private List<MonitorID> monitorIDs;
+
+ public HostMonitorData(JobExecutionContext jobExecutionContext) {
+ this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
+ this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
+ this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
+ this.monitorIDs = new ArrayList<MonitorID>();
+ }
+
+ public HostMonitorData(JobExecutionContext jobExecutionContext, List<MonitorID> monitorIDs) {
+ this.computeResourceDescription = jobExecutionContext.getApplicationContext().getComputeResourceDescription();
+ this.jobSubmissionProtocol = jobExecutionContext.getPreferredJobSubmissionProtocol();
+ this.dataMovementProtocol = jobExecutionContext.getPreferredDataMovementProtocol();
+ this.monitorIDs = monitorIDs;
+ }
+
+ public ComputeResourceDescription getComputeResourceDescription() {
+ return computeResourceDescription;
+ }
+
+ public void setComputeResourceDescription(ComputeResourceDescription computeResourceDescription) {
+ this.computeResourceDescription = computeResourceDescription;
+ }
+
+ public List<MonitorID> getMonitorIDs() {
+ return monitorIDs;
+ }
+
+ public void setMonitorIDs(List<MonitorID> monitorIDs) {
+ this.monitorIDs = monitorIDs;
+ }
+
+ /**
+ * this method get called by CommonUtils and it will check the right place before adding
+ * so there will not be a mismatch between this.host and monitorID.host
+ * @param monitorID
+ * @throws org.apache.airavata.gfac.monitor.exception.AiravataMonitorException
+ */
+ public void addMonitorIDForHost(MonitorID monitorID)throws AiravataMonitorException {
+ monitorIDs.add(monitorID);
+ }
+
+ public JobSubmissionProtocol getJobSubmissionProtocol() {
+ return jobSubmissionProtocol;
+ }
+
+ public DataMovementProtocol getDataMovementProtocol() {
+ return dataMovementProtocol;
+ }
+}