You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/12/15 03:49:21 UTC
incubator-eagle git commit: [EAGLE-834] Add Daily Job Summary Report
Repository: incubator-eagle
Updated Branches:
refs/heads/master 67c915127 -> f430f521f
[EAGLE-834] Add Daily Job Summary Report
https://issues.apache.org/jira/browse/EAGLE-834
Author: Zhao, Qingwen <qi...@apache.org>
Author: Qingwen Zhao <qi...@gmail.com>
Closes #742 from qingwen220/EAGLE-834.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/f430f521
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/f430f521
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/f430f521
Branch: refs/heads/master
Commit: f430f521faea959491208c199b59244386cef438
Parents: 67c9151
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Thu Dec 15 11:49:13 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Thu Dec 15 11:49:13 2016 +0800
----------------------------------------------------------------------
.../engine/coordinator/AlertDefinition.java | 12 +-
.../publisher/impl/AlertEmailPublisher.java | 30 +-
.../src/test/resources/application-test.conf | 18 +-
.../app/service/ApplicationEmailService.java | 104 ++++++
.../eagle/common/mail/AbstractEmailService.java | 82 +++++
.../eagle/common/mail/AlertEmailConstants.java | 57 ++++
.../eagle/common/mail/AlertEmailContext.java | 68 ++++
.../eagle/common/mail/AlertEmailSender.java | 111 +++++++
.../eagle/common/mail/EagleMailClient.java | 229 ++++++++++++++
eagle-jpm/eagle-jpm-mr-history/pom.xml | 6 +
.../mr/history/MRHistoryJobDailyReporter.java | 317 +++++++++++++++++++
.../src/main/resources/JobReportTemplate.vm | 141 +++++++++
.../history/MRHistoryJobDailyReporterTest.java | 109 +++++++
.../src/test/resources/JobReportTemplate.vm | 141 +++++++++
.../src/test/resources/application-test.conf | 58 ++++
.../src/test/resources/log4j.properties | 34 ++
eagle-server-assembly/src/main/conf/eagle.conf | 25 +-
.../apache/eagle/server/ServerApplication.java | 11 +
.../src/main/resources/application.conf | 27 +-
19 files changed, 1537 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java
index 66a9bce..e94d6fa 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDefinition.java
@@ -18,6 +18,8 @@ package org.apache.eagle.alert.engine.coordinator;
import org.apache.commons.lang3.builder.HashCodeBuilder;
+import java.util.Objects;
+
public class AlertDefinition {
private TemplateType templateType = TemplateType.TEXT;
private String subject;
@@ -92,11 +94,11 @@ public class AlertDefinition {
return false;
}
AlertDefinition another = (AlertDefinition) that;
- if (another.templateType.equals(this.templateType)
- && another.body.equals(this.body)
- && another.category.equals(this.category)
- && another.severity.equals(this.severity)
- && another.subject.equals(this.subject)) {
+ if (Objects.equals(another.templateType, this.templateType)
+ && Objects.equals(another.body, this.body)
+ && Objects.equals(another.category, this.category)
+ && Objects.equals(another.severity, this.severity)
+ && Objects.equals(another.subject, this.subject)) {
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
index d81ec2a..d08d114 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
@@ -39,6 +39,7 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import static org.apache.eagle.alert.service.MetadataServiceClientImpl.*;
+import static org.apache.eagle.common.mail.AlertEmailConstants.*;
public class AlertEmailPublisher extends AbstractPublishPlugin {
@@ -47,14 +48,6 @@ public class AlertEmailPublisher extends AbstractPublishPlugin {
private static final int DEFAULT_THREAD_POOL_MAX_SIZE = 8;
private static final long DEFAULT_THREAD_POOL_SHRINK_TIME = 60000L; // 1 minute
- private static final String EAGLE_CORRELATION_SMTP_SERVER = "metadataService.mailSmtpServer";
- private static final String EAGLE_CORRELATION_SMTP_PORT = "metadataService.mailSmtpPort";
- private static final String EAGLE_CORRELATION_SMTP_CONN = "metadataService.mailSmtpConn";
- private static final String EAGLE_CORRELATION_SMTP_AUTH = "metadataService.mailSmtpAuth";
- private static final String EAGLE_CORRELATION_SMTP_USERNAME = "metadataService.mailSmtpUsername";
- private static final String EAGLE_CORRELATION_SMTP_PASSWORD = "metadataService.mailSmtpPassword";
- private static final String EAGLE_CORRELATION_SMTP_DEBUG = "metadataService.mailSmtpDebug";
-
private AlertEmailGenerator emailGenerator;
private Map<String, Object> emailConfig;
@@ -83,23 +76,28 @@ public class AlertEmailPublisher extends AbstractPublishPlugin {
private Properties parseMailClientConfig(Config config) {
Properties props = new Properties();
- String mailSmtpServer = config.getString(EAGLE_CORRELATION_SMTP_SERVER);
- String mailSmtpPort = config.getString(EAGLE_CORRELATION_SMTP_PORT);
- String mailSmtpAuth = config.getString(EAGLE_CORRELATION_SMTP_AUTH);
+ Config mailConfig = null;
+ if (config.hasPath(EAGLE_COORDINATOR_EMAIL_SERVICE)) {
+ mailConfig = config.getConfig(EAGLE_COORDINATOR_EMAIL_SERVICE);
+ } else if (config.hasPath(EAGLE_APPLICATION_EMAIL_SERVICE)) {
+ mailConfig = config.getConfig(EAGLE_APPLICATION_EMAIL_SERVICE);
+ }
+ String mailSmtpServer = mailConfig.getString(EAGLE_EMAIL_SMTP_SERVER);
+ String mailSmtpPort = mailConfig.getString(EAGLE_EMAIL_SMTP_PORT);
+ String mailSmtpAuth = mailConfig.getString(EAGLE_EMAIL_SMTP_AUTH);
props.put(AlertEmailConstants.CONF_MAIL_HOST, mailSmtpServer);
props.put(AlertEmailConstants.CONF_MAIL_PORT, mailSmtpPort);
props.put(AlertEmailConstants.CONF_MAIL_AUTH, mailSmtpAuth);
if (Boolean.parseBoolean(mailSmtpAuth)) {
- String mailSmtpUsername = config.getString(EAGLE_CORRELATION_SMTP_USERNAME);
- String mailSmtpPassword = config.getString(EAGLE_CORRELATION_SMTP_PASSWORD);
+ String mailSmtpUsername = mailConfig.getString(EAGLE_EMAIL_SMTP_USERNAME);
+ String mailSmtpPassword = mailConfig.getString(EAGLE_EMAIL_SMTP_PASSWORD);
props.put(AlertEmailConstants.CONF_AUTH_USER, mailSmtpUsername);
props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, mailSmtpPassword);
}
- String mailSmtpConn = config.hasPath(EAGLE_CORRELATION_SMTP_CONN) ? config.getString(EAGLE_CORRELATION_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT;
- String mailSmtpDebug = config.hasPath(EAGLE_CORRELATION_SMTP_DEBUG) ? config.getString(EAGLE_CORRELATION_SMTP_DEBUG) : "false";
+ String mailSmtpConn = mailConfig.hasPath(EAGLE_EMAIL_SMTP_CONN) ? mailConfig.getString(EAGLE_EMAIL_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT;
if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) {
props.put("mail.smtp.starttls.enable", "true");
}
@@ -108,6 +106,8 @@ public class AlertEmailPublisher extends AbstractPublishPlugin {
props.put("mail.smtp.socketFactory.class",
"javax.net.ssl.SSLSocketFactory");
}
+
+ String mailSmtpDebug = mailConfig.hasPath(EAGLE_EMAIL_SMTP_DEBUG) ? mailConfig.getString(EAGLE_EMAIL_SMTP_DEBUG) : "false";
props.put(AlertEmailConstants.CONF_MAIL_DEBUG, mailSmtpDebug);
return props;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf
index 3573507..99bbee6 100755
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/resources/application-test.conf
@@ -48,14 +48,7 @@
"metadataService": {
"context": "/api",
"host": "localhost",
- "port": 8080,
- mailSmtpServer = "localhost",
- mailSmtpPort = 5025,
- mailSmtpAuth = "false"
- //mailSmtpConn = "plaintext",
- //mailSmtpUsername = ""
- //mailSmtpPassword = ""
- //mailSmtpDebug = false
+ "port": 8080
},
"metric": {
"sink": {
@@ -73,4 +66,13 @@
}
},
"connection": "mongodb://localhost:27017"
+ application.mailService {
+ mailSmtpServer = "localhost",
+ mailSmtpPort = 5025,
+ mailSmtpAuth = "false"
+ //mailSmtpConn = "plaintext",
+ //mailSmtpUsername = ""
+ //mailSmtpPassword = ""
+ //mailSmtpDebug = false
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java
new file mode 100644
index 0000000..7498748
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationEmailService.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.app.service;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.common.mail.AbstractEmailService;
+import org.apache.eagle.common.mail.AlertEmailConstants;
+import org.apache.eagle.common.mail.AlertEmailContext;
+import org.apache.eagle.common.mail.AlertEmailSender;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Map;
+
+import static org.apache.eagle.common.mail.AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE;
+
+public class ApplicationEmailService extends AbstractEmailService {
+ private static final Logger LOG = LoggerFactory.getLogger(ApplicationEmailService.class);
+ private String appConfPath;
+ private Config config;
+
+ public ApplicationEmailService(Config config, String appConfPath) {
+ super(config.getConfig(EAGLE_APPLICATION_EMAIL_SERVICE));
+ this.appConfPath = appConfPath;
+ this.config = config;
+ }
+
+ public boolean onAlert(Map<String, Object> alertData) {
+ return super.onAlert(buildEmailContext(), alertData);
+ }
+
+ private String buildDefaultSender() {
+ String hostname = "";
+ try {
+ hostname = InetAddress.getLocalHost().getHostName();
+ if (!hostname.endsWith(".com")) {
+ //avoid invalid host exception
+ hostname += ".com";
+ }
+ } catch (UnknownHostException e) {
+ LOG.warn("UnknownHostException when get local hostname");
+ }
+ return System.getProperty("user.name") + "@" + hostname;
+ }
+
+ public AlertEmailContext buildEmailContext() {
+ return buildEmailContext(null);
+ }
+
+ public AlertEmailContext buildEmailContext(String mailSubject) {
+ AlertEmailContext mailProps = new AlertEmailContext();
+ Config appConfig = config.getConfig(appConfPath);
+ String tplFileName = appConfig.getString(AlertEmailConstants.TEMPLATE);
+ if (tplFileName == null || tplFileName.equals("")) {
+ tplFileName = "ALERT_INLINED_TEMPLATE.vm";
+ }
+ String subject;
+ if (mailSubject != null) {
+ subject = mailSubject;
+ } else {
+ subject = appConfig.getString(AlertEmailConstants.SUBJECT);
+ }
+ String sender;
+ if (!appConfig.hasPath(AlertEmailConstants.SENDER)) {
+ sender = buildDefaultSender();
+ } else {
+ sender = appConfig.getString(AlertEmailConstants.SENDER);
+ }
+ if (!appConfig.hasPath(AlertEmailConstants.RECIPIENTS)) {
+ throw new IllegalArgumentException("email recipients is null or unset");
+ }
+ if (appConfig.hasPath(AlertEmailConstants.CC_RECIPIENTS)) {
+ mailProps.setCc(appConfig.getString(AlertEmailConstants.CC_RECIPIENTS));
+ }
+ String recipients = appConfig.getString(AlertEmailConstants.RECIPIENTS);
+ mailProps.setSubject(subject);
+ mailProps.setRecipients(recipients);
+ mailProps.setSender(sender);
+ mailProps.setVelocityTplFile(tplFileName);
+ return mailProps;
+ }
+
+ @Override
+ protected Logger getLogger() {
+ return LOG;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java
new file mode 100644
index 0000000..57e5cba
--- /dev/null
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AbstractEmailService.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.common.mail;
+
+import com.typesafe.config.Config;
+import org.slf4j.Logger;
+
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.eagle.common.mail.AlertEmailConstants.*;
+
+public abstract class AbstractEmailService {
+
+ private Properties serverProps;
+
+ public AbstractEmailService(Config config) {
+ serverProps = parseMailClientConfig(config);
+ }
+
+ protected abstract Logger getLogger();
+
+ private Properties parseMailClientConfig(Config config) {
+ Properties props = new Properties();
+ String mailSmtpServer = config.getString(EAGLE_EMAIL_SMTP_SERVER);
+ String mailSmtpPort = config.getString(EAGLE_EMAIL_SMTP_PORT);
+ String mailSmtpAuth = config.getString(EAGLE_EMAIL_SMTP_AUTH);
+
+ props.put(AlertEmailConstants.CONF_MAIL_HOST, mailSmtpServer);
+ props.put(AlertEmailConstants.CONF_MAIL_PORT, mailSmtpPort);
+ props.put(AlertEmailConstants.CONF_MAIL_AUTH, mailSmtpAuth);
+
+ if (Boolean.parseBoolean(mailSmtpAuth)) {
+ String mailSmtpUsername = config.getString(EAGLE_EMAIL_SMTP_USERNAME);
+ String mailSmtpPassword = config.getString(EAGLE_EMAIL_SMTP_PASSWORD);
+ props.put(AlertEmailConstants.CONF_AUTH_USER, mailSmtpUsername);
+ props.put(AlertEmailConstants.CONF_AUTH_PASSWORD, mailSmtpPassword);
+ }
+
+ String mailSmtpConn = config.hasPath(EAGLE_EMAIL_SMTP_CONN) ? config.getString(EAGLE_EMAIL_SMTP_CONN) : AlertEmailConstants.CONN_PLAINTEXT;
+ String mailSmtpDebug = config.hasPath(EAGLE_EMAIL_SMTP_DEBUG) ? config.getString(EAGLE_EMAIL_SMTP_DEBUG) : "false";
+ if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_TLS)) {
+ props.put("mail.smtp.starttls.enable", "true");
+ }
+ if (mailSmtpConn.equalsIgnoreCase(AlertEmailConstants.CONN_SSL)) {
+ props.put("mail.smtp.socketFactory.port", "465");
+ props.put("mail.smtp.socketFactory.class",
+ "javax.net.ssl.SSLSocketFactory");
+ }
+ props.put(AlertEmailConstants.CONF_MAIL_DEBUG, mailSmtpDebug);
+ return props;
+ }
+
+ public boolean onAlert(AlertEmailContext mailContext, Map<String, Object> alertData) {
+ /** synchronized email sending. */
+ if (alertData == null || alertData.isEmpty()) {
+ getLogger().warn("alertData for {} is empty");
+ return false;
+ }
+ AlertEmailSender mailSender = new AlertEmailSender(mailContext, serverProps);
+ mailSender.addAlertContext(alertData);
+ getLogger().info("Sending email in synchronous mode to: {} cc: {}", mailContext.getRecipients(), mailContext.getCc());
+ return mailSender.send();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java
new file mode 100644
index 0000000..140d306
--- /dev/null
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailConstants.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.common.mail;
+
+public class AlertEmailConstants {
+
+ public static final String CONN_PLAINTEXT = "plaintext";
+ public static final String CONN_TLS = "tls";
+ public static final String CONN_SSL = "ssl";
+
+ public static final String CONF_MAIL_HOST = "mail.smtp.host";
+ public static final String CONF_MAIL_PORT = "mail.smtp.port";
+ public static final String CONF_MAIL_AUTH = "mail.smtp.auth";
+ public static final String CONF_AUTH_USER = "mail.username";
+ public static final String CONF_AUTH_PASSWORD = "mail.password";
+ public static final String CONF_MAIL_CONN = "mail.connection";
+ public static final String CONF_MAIL_DEBUG = "mail.debug";
+
+ public static final String SUBJECT = "subject";
+ public static final String SENDER = "sender";
+ public static final String RECIPIENTS = "recipients";
+ public static final String TEMPLATE = "template";
+ public static final String CC_RECIPIENTS = "cc";
+
+ public static final String ALERT_EMAIL_TIME_PROPERTY = "timestamp";
+ public static final String ALERT_EMAIL_COUNT_PROPERTY = "count";
+ public static final String ALERT_EMAIL_ALERTLIST_PROPERTY = "alertList";
+ public static final String ALERT_EMAIL_ORIGIN_PROPERTY = "alertEmailOrigin";
+
+ public static final String EAGLE_APPLICATION_EMAIL_SERVICE = "application.mailService";
+ public static final String EAGLE_COORDINATOR_EMAIL_SERVICE = "coordinator.mailService";
+
+ public static final String EAGLE_EMAIL_SMTP_SERVER = "mailSmtpServer";
+ public static final String EAGLE_EMAIL_SMTP_PORT = "mailSmtpPort";
+ public static final String EAGLE_EMAIL_SMTP_CONN = "mailSmtpConn";
+ public static final String EAGLE_EMAIL_SMTP_AUTH = "mailSmtpAuth";
+ public static final String EAGLE_EMAIL_SMTP_USERNAME = "mailSmtpUsername";
+ public static final String EAGLE_EMAIL_SMTP_PASSWORD = "mailSmtpPassword";
+ public static final String EAGLE_EMAIL_SMTP_DEBUG = "mailSmtpDebug";
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java
new file mode 100644
index 0000000..c01d51b
--- /dev/null
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailContext.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.common.mail;
+
+public class AlertEmailContext {
+ private String sender;
+ private String subject;
+ private String recipients;
+ private String velocityTplFile;
+ private String cc;
+
+ public String getSender() {
+ return sender;
+ }
+
+ public void setSender(String sender) {
+ this.sender = sender;
+ }
+
+ public String getSubject() {
+ return subject;
+ }
+
+ public void setSubject(String subject) {
+ this.subject = subject;
+ }
+
+ public String getRecipients() {
+ return recipients;
+ }
+
+ public void setRecipients(String recipients) {
+ this.recipients = recipients;
+ }
+
+ public String getVelocityTplFile() {
+ return velocityTplFile;
+ }
+
+ public void setVelocityTplFile(String velocityTplFile) {
+ this.velocityTplFile = velocityTplFile;
+ }
+
+ public String getCc() {
+ return cc;
+ }
+
+ public void setCc(String cc) {
+ this.cc = cc;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java
new file mode 100644
index 0000000..ffc249d
--- /dev/null
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/AlertEmailSender.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.common.mail;
+
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.velocity.VelocityContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class AlertEmailSender {
+ private static final Logger LOG = LoggerFactory.getLogger(AlertEmailSender.class);
+
+ private List<Map<String, Object>> alertContexts = new ArrayList<>();
+ private String origin;
+
+ private AlertEmailContext mailProps;
+ private Properties serverProps;
+
+ private static final int MAX_RETRY_COUNT = 3;
+
+ /**
+ * Derived class may have some additional context properties to add.
+ * @param context velocity context
+ */
+ public void addAlertContext(Map<String, Object> context) {
+ alertContexts.add(context);
+ }
+
+ public AlertEmailSender(AlertEmailContext mailProps, Properties serverProps) {
+ this.mailProps = mailProps;
+ this.serverProps = serverProps;
+ String tmp = ManagementFactory.getRuntimeMXBean().getName();
+ this.origin = tmp.split("@")[1] + "(pid:" + tmp.split("@")[0] + ")";
+ LOG.info("Initialized email sender: origin is :{}, recipient of the email: {}, velocity TPL file: {}",
+ origin, mailProps.getRecipients(), mailProps.getVelocityTplFile());
+ }
+
+ public boolean send() {
+ int count = 0;
+ boolean success = false;
+ while (count++ < MAX_RETRY_COUNT && !success) {
+ LOG.info("Sending email, tried: " + count + ", max: " + MAX_RETRY_COUNT);
+ try {
+ final EagleMailClient client;
+ if (serverProps != null) {
+ client = new EagleMailClient(serverProps);
+ } else {
+ client = new EagleMailClient();
+ }
+
+ final VelocityContext context = new VelocityContext();
+ generateCommonContext(context);
+ LOG.info("After calling generateCommonContext...");
+
+ if (mailProps.getRecipients() == null || mailProps.getRecipients().equals("")) {
+ LOG.error("Recipients is null, skip sending emails ");
+ return success;
+ }
+
+ success = client.send(mailProps.getSender(),
+ mailProps.getRecipients(),
+ mailProps.getCc(),
+ mailProps.getSubject(),
+ mailProps.getVelocityTplFile(),
+ context,
+ null);
+ LOG.info("Success of sending email: " + success);
+ if (!success && count < MAX_RETRY_COUNT) {
+ LOG.info("Sleep for a while before retrying");
+ Thread.sleep(10 * 1000);
+ }
+ } catch (Exception e) {
+ LOG.warn("Sending mail exception", e);
+ }
+ }
+ if (success) {
+ LOG.info("Successfully send email with subject {}", mailProps.getSubject());
+ } else {
+ LOG.warn("Fail sending email after tries {} times, subject: %s", MAX_RETRY_COUNT, mailProps.getSubject());
+ }
+ return success;
+ }
+
+ private void generateCommonContext(VelocityContext context) {
+ context.put(AlertEmailConstants.ALERT_EMAIL_TIME_PROPERTY, DateTimeUtil.millisecondsToHumanDateWithSeconds(System.currentTimeMillis()));
+ context.put(AlertEmailConstants.ALERT_EMAIL_COUNT_PROPERTY, alertContexts.size());
+ context.put(AlertEmailConstants.ALERT_EMAIL_ALERTLIST_PROPERTY, alertContexts);
+ context.put(AlertEmailConstants.ALERT_EMAIL_ORIGIN_PROPERTY, origin);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java
new file mode 100644
index 0000000..81b1114
--- /dev/null
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/mail/EagleMailClient.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.common.mail;
+
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.velocity.exception.ResourceNotFoundException;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.activation.DataHandler;
+import javax.activation.DataSource;
+import javax.activation.FileDataSource;
+import javax.mail.*;
+import javax.mail.internet.*;
+import java.io.File;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+public class EagleMailClient {
+ private static final Logger LOG = LoggerFactory.getLogger(EagleMailClient.class);
+ private static final String BASE_PATH = "templates/";
+
+ private VelocityEngine velocityEngine;
+ private Session session;
+
+ public EagleMailClient() {
+ this(new Properties());
+ }
+
+ public EagleMailClient(final Properties config) {
+ try {
+ velocityEngine = new VelocityEngine();
+ velocityEngine.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");
+ velocityEngine.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
+ velocityEngine.init();
+
+ config.put("mail.transport.protocol", "smtp");
+ if (Boolean.parseBoolean(config.getProperty(AlertEmailConstants.CONF_MAIL_AUTH))) {
+ session = Session.getInstance(config, new Authenticator() {
+ protected PasswordAuthentication getPasswordAuthentication() {
+ return new PasswordAuthentication(
+ config.getProperty(AlertEmailConstants.CONF_AUTH_USER),
+ config.getProperty(AlertEmailConstants.CONF_AUTH_PASSWORD)
+ );
+ }
+ });
+ } else {
+ session = Session.getInstance(config, new Authenticator() {
+ });
+ }
+
+ final String debugMode = config.getProperty(AlertEmailConstants.CONF_MAIL_DEBUG, "false");
+ final boolean debug = Boolean.parseBoolean(debugMode);
+ LOG.info("Set email debug mode: " + debugMode);
+ session.setDebug(debug);
+ } catch (Exception e) {
+ LOG.error("Failed to connect to smtp server", e);
+ }
+ }
+
+ private boolean sendInternal(String from, String to, String cc, String title, String content) {
+ Message msg = new MimeMessage(session);
+ try {
+ msg.setFrom(new InternetAddress(from));
+ msg.setSubject(title);
+ if (to != null) {
+ msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to));
+ }
+ if (cc != null) {
+ msg.setRecipients(Message.RecipientType.CC, InternetAddress.parse(cc));
+ }
+ //msg.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS));
+ msg.setContent(content, "text/html;charset=utf-8");
+ LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title));
+ Transport.send(msg);
+ return true;
+ } catch (AddressException e) {
+ LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e);
+ return false;
+ } catch (MessagingException e) {
+ LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private boolean sendInternal(String from, String to, String cc, String title, String content, List<MimeBodyPart> attachments) {
+ MimeMessage mail = new MimeMessage(session);
+ try {
+ mail.setFrom(new InternetAddress(from));
+ mail.setSubject(title);
+ if (to != null) {
+ mail.setRecipients(Message.RecipientType.TO, InternetAddress.parse(to));
+ }
+ if (cc != null) {
+ mail.setRecipients(Message.RecipientType.CC, InternetAddress.parse(cc));
+ }
+
+ //mail.setRecipients(Message.RecipientType.BCC, InternetAddress.parse(DEFAULT_BCC_ADDRESS));
+
+ MimeBodyPart mimeBodyPart = new MimeBodyPart();
+ mimeBodyPart.setContent(content, "text/html;charset=utf-8");
+
+ Multipart multipart = new MimeMultipart();
+ multipart.addBodyPart(mimeBodyPart);
+
+ for (MimeBodyPart attachment : attachments) {
+ multipart.addBodyPart(attachment);
+ }
+
+ mail.setContent(multipart);
+ // mail.setContent(content, "text/html;charset=utf-8");
+ LOG.info(String.format("Going to send mail: from[%s], to[%s], cc[%s], title[%s]", from, to, cc, title));
+ Transport.send(mail);
+ return true;
+ } catch (AddressException e) {
+ LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e);
+ return false;
+ } catch (MessagingException e) {
+ LOG.info("Failed to send mail, got an AddressException: " + e.getMessage(), e);
+ return false;
+ }
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String content) {
+ return this.sendInternal(from, to, cc, title, content);
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String templatePath, VelocityContext context) {
+ Template t = null;
+ try {
+ t = velocityEngine.getTemplate(BASE_PATH + templatePath);
+ } catch (ResourceNotFoundException ex) {
+ // ignored
+ }
+ if (t == null) {
+ try {
+ t = velocityEngine.getTemplate(templatePath);
+ } catch (ResourceNotFoundException e) {
+ t = velocityEngine.getTemplate("/" + templatePath);
+ }
+ }
+ final StringWriter writer = new StringWriter();
+ t.merge(context, writer);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(writer.toString());
+ }
+
+ return this.send(from, to, cc, title, writer.toString());
+ }
+
+ public boolean send(String from, String to, String cc, String title,
+ String templatePath, VelocityContext context, Map<String, File> attachments) {
+ if (attachments == null || attachments.isEmpty()) {
+ return send(from, to, cc, title, templatePath, context);
+ }
+ Template t = null;
+
+ List<MimeBodyPart> mimeBodyParts = new ArrayList<MimeBodyPart>();
+
+ for (Map.Entry<String, File> entry : attachments.entrySet()) {
+ final String attachment = entry.getKey();
+ final File attachmentFile = entry.getValue();
+ final MimeBodyPart mimeBodyPart = new MimeBodyPart();
+ if (attachmentFile != null && attachmentFile.exists()) {
+ DataSource source = new FileDataSource(attachmentFile);
+ try {
+ mimeBodyPart.setDataHandler(new DataHandler(source));
+ mimeBodyPart.setFileName(attachment);
+ mimeBodyPart.setDisposition(MimeBodyPart.ATTACHMENT);
+ mimeBodyPart.setContentID(attachment);
+ mimeBodyParts.add(mimeBodyPart);
+ } catch (MessagingException e) {
+ LOG.error("Generate mail failed, got exception while attaching files: " + e.getMessage(), e);
+ }
+ } else {
+ LOG.error("Attachment: " + attachment + " is null or not exists");
+ }
+ }
+
+ try {
+ t = velocityEngine.getTemplate(BASE_PATH + templatePath);
+ } catch (ResourceNotFoundException ex) {
+ // ignored
+ }
+
+ if (t == null) {
+ try {
+ t = velocityEngine.getTemplate(templatePath);
+ } catch (ResourceNotFoundException e) {
+ try {
+ t = velocityEngine.getTemplate("/" + templatePath);
+ } catch (Exception ex) {
+ LOG.error("Template not found:" + "/" + templatePath, ex);
+ }
+ }
+ }
+
+ final StringWriter writer = new StringWriter();
+ t.merge(context, writer);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(writer.toString());
+ }
+
+ return this.sendInternal(from, to, cc, title, writer.toString(), mimeBodyParts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/pom.xml b/eagle-jpm/eagle-jpm-mr-history/pom.xml
index dde7056..cbe8de1 100644
--- a/eagle-jpm/eagle-jpm-mr-history/pom.xml
+++ b/eagle-jpm/eagle-jpm-mr-history/pom.xml
@@ -74,6 +74,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>dumbster</groupId>
+ <artifactId>dumbster</artifactId>
+ <version>1.6</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<resources>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
new file mode 100644
index 0000000..42f2b29
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.mr.history;
+
+import com.google.common.util.concurrent.AbstractScheduledService;
+import com.google.inject.Inject;
+import com.typesafe.config.Config;
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.eagle.app.service.ApplicationEmailService;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.common.mail.AlertEmailConstants;
+import org.apache.eagle.common.mail.AlertEmailContext;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.service.client.EagleServiceClientException;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_HOST;
+import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_PORT;
+
+public class MRHistoryJobDailyReporter extends AbstractScheduledService {
+ private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporter.class);
+ private static final String TIMEZONE_PATH = "service.timezone";
+ private static final String DAILY_SENT_HOUROFDAY = "application.dailyJobReport.reportHourTime";
+ private static final String DAILY_SENT_PERIOD = "application.dailyJobReport.reportPeriodInHour";
+ private static final String NUM_TOP_USERS = "application.dailyJobReport.numTopUsers";
+ private static final String JOB_OVERTIME_LIMIT_HOUR = "application.dailyJobReport.jobOvertimeLimitInHour";
+
+ public static final String SERVICE_PATH = "application.dailyJobReport";
+ public static final String APP_TYPE = "MR_HISTORY_JOB_APP";
+
+ // alert context keys
+ protected static final String NUM_TOP_USERS_KEY = "numTopUsers";
+ protected static final String JOB_OVERTIME_LIMIT_KEY = "jobOvertimeLimit";
+ protected static final String ALERT_TITLE_KEY = "alertTitle";
+ protected static final String REPORT_RANGE_KEY = "reportRange";
+ protected static final String SUMMARY_INFO_KEY = "summaryInfo";
+ protected static final String FAILED_JOB_USERS_KEY = "failedJobUsers";
+ protected static final String SUCCEEDED_JOB_USERS_KEY = "succeededJobUsers";
+ protected static final String FINISHED_JOB_USERS_KEY = "finishedJobUsers";
+ protected static final String EAGLE_JOB_LINK_KEY = "eagleJobLink";
+
+ private Config config;
+ private IEagleServiceClient client;
+ private ApplicationEntityService applicationResource;
+ private ApplicationEmailService emailService;
+ private boolean isDailySent = false;
+ private long lastSentTime;
+
+ private int dailySentHour;
+ private int dailySentPeriod;
+ private int numTopUsers = 10;
+ private int jobOvertimeLimit = 6;
+
+ // scheduler
+ private int initialDelayMin = 10;
+ private int periodInMin = 60;
+ private TimeZone timeZone;
+
+ @Inject
+ public MRHistoryJobDailyReporter(Config config, ApplicationEntityService applicationEntityService) {
+ this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH));
+
+ if (config.hasPath(SERVICE_PATH) && config.hasPath(AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE)) {
+ this.emailService = new ApplicationEmailService(config, SERVICE_PATH);
+ }
+ if (config.hasPath(DAILY_SENT_HOUROFDAY)) {
+ this.dailySentHour = config.getInt(DAILY_SENT_HOUROFDAY);
+ }
+ if (config.hasPath(DAILY_SENT_PERIOD)) {
+ this.dailySentPeriod = config.getInt(DAILY_SENT_PERIOD);
+ }
+ if (config.hasPath(NUM_TOP_USERS)) {
+ this.numTopUsers = config.getInt(NUM_TOP_USERS);
+ }
+ if (config.hasPath(JOB_OVERTIME_LIMIT_HOUR)) {
+ this.jobOvertimeLimit = config.getInt(JOB_OVERTIME_LIMIT_HOUR);
+ }
+ this.config = config;
+ this.applicationResource = applicationEntityService;
+ }
+
+ private boolean isSentHour(int currentHour) {
+ return Math.abs(currentHour - dailySentHour) % dailySentPeriod == 0;
+ }
+
+ private Collection<String> loadSites(String appType) {
+ Set<String> sites = new HashSet<>();
+ Collection<ApplicationEntity> apps = applicationResource.findAll();
+ for (ApplicationEntity app : apps) {
+ if (app.getDescriptor().getType().equalsIgnoreCase(appType) && app.getStatus().equals(ApplicationEntity.Status.RUNNING)) {
+ sites.add(app.getSite().getSiteId());
+ }
+ }
+ LOG.info("Detected {} sites where MR_HISTORY_JOB_APP is Running: {}", sites.size(), sites);
+ return sites;
+ }
+
+ @Override
+ protected void runOneIteration() throws Exception {
+ GregorianCalendar calendar = new GregorianCalendar(timeZone);
+ int currentHour = calendar.get(Calendar.HOUR_OF_DAY);
+ long currentTimestamp = calendar.getTimeInMillis();
+ if (!isSentHour(currentHour)) {
+ isDailySent = false;
+ } else if (!isDailySent) {
+ isDailySent = true;
+ LOG.info("last job report time is {} %s", DateTimeUtil.millisecondsToHumanDateWithSeconds(lastSentTime), timeZone.getID());
+ try {
+ Collection<String> sites = loadSites(APP_TYPE);
+ if (sites == null || sites.isEmpty()) {
+ LOG.warn("application MR_HISTORY_JOB_APP does not run on any sites!");
+ return;
+ }
+ for (String site : sites) {
+ int reportHour = currentHour / dailySentPeriod * dailySentPeriod;
+ calendar.set(Calendar.HOUR_OF_DAY, reportHour);
+ String subject = String.format("%s %s", site.toUpperCase(), config.getString(SERVICE_PATH + "." + AlertEmailConstants.SUBJECT));
+ Map<String, Object> alertData = buildAlertData(site, calendar.getTimeInMillis());
+ sendByEmailWithSubject(alertData, subject);
+ }
+ } catch (Exception ex) {
+ LOG.error("Fail to get job summery info due to {}", ex.getMessage(), ex);
+ }
+ lastSentTime = currentTimestamp;
+ }
+ }
+
+ protected void sendByEmail(Map<String, Object> alertData) {
+ emailService.onAlert(alertData);
+ }
+
+ protected void sendByEmailWithSubject(Map<String, Object> alertData, String subject) {
+ AlertEmailContext alertContext = emailService.buildEmailContext(subject);
+ emailService.onAlert(alertContext, alertData);
+ }
+
+ private Map<String, Object> buildAlertData(String site, long endTime) {
+ StopWatch watch = new StopWatch();
+ Map<String, Object> data = new HashMap<>();
+ this.client = new EagleServiceClientImpl(config);
+ long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod;
+ LOG.info("Going to report job summery info for site {} from {} to {}", site,
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime));
+ try {
+ watch.start();
+ data.putAll(buildJobSummery(site, startTime, endTime));
+ data.putAll(buildFailedJobInfo(site, startTime, endTime));
+ data.putAll(buildSucceededJobInfo(site, startTime, endTime));
+ data.putAll(buildFinishedJobInfo(site, startTime, endTime));
+ data.put(NUM_TOP_USERS_KEY, numTopUsers);
+ data.put(JOB_OVERTIME_LIMIT_KEY, jobOvertimeLimit);
+ data.put(ALERT_TITLE_KEY, String.format("%s Daily Job Report", site.toUpperCase()));
+ data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s",
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime),
+ DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+ watch.stop();
+ LOG.info("Fetching DailyJobReport tasks {} seconds", watch.getTime() / DateTimeUtil.ONESECOND);
+ } finally {
+ try {
+ client.close();
+ } catch (IOException e) {
+ LOG.info("fail to close eagle service client");
+ }
+ }
+ return data;
+ }
+
+ private Map<String, Long> parseQueryResult(List<Map<List<String>, List<Double>>> result, int limit) {
+ Map<String, Long> stateCount = new LinkedHashMap<>();
+ for (Map<List<String>, List<Double>> map : result) {
+ if (stateCount.size() >= limit) {
+ break;
+ }
+ String key = String.valueOf(map.get("key").get(0));
+ Long value = map.get("value").get(0).longValue();
+ stateCount.put(key, value);
+ }
+ return stateCount;
+ }
+
+ private Map<String, Long> queryGroupByMetrics(String condition, long startTime, long endTime, int limit) {
+ try {
+ GenericServiceAPIResponseEntity response = client.search()
+ .pageSize(Integer.MAX_VALUE)
+ .query(condition)
+ .startTime(startTime)
+ .endTime(endTime).send();
+ if (!response.isSuccess()) {
+ return null;
+ }
+ List<Map<List<String>, List<Double>>> result = response.getObj();
+ return parseQueryResult(result, limit);
+ } catch (EagleServiceClientException e) {
+ LOG.error(e.getMessage(), e);
+ return new HashMap<>();
+ }
+ }
+
+ private Map<String, Object> buildJobSummery(String site, long startTime, long endTime) {
+ Map<String, Object> data = new HashMap<>();
+ List<JobSummeryInfo> statusCount = new ArrayList<>();
+ String query = String.format("%s[@site=\"%s\" and @endTime<=%...@currentState>{count}",
+ Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+ Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE);
+ if (jobSummery == null || jobSummery.isEmpty()) {
+ return data;
+ }
+ Optional<Long> totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b);
+ for (Map.Entry<String, Long> entry : jobSummery.entrySet()) {
+ JobSummeryInfo summeryInfo = new JobSummeryInfo();
+ summeryInfo.status = entry.getKey();
+ summeryInfo.numOfJobs = entry.getValue();
+ summeryInfo.ratio = String.format("%.2f", entry.getValue() * 1d / totalJobs.get());
+ statusCount.add(summeryInfo);
+ }
+ data.put(SUMMARY_INFO_KEY, statusCount);
+ return data;
+ }
+
+ private Map<String, Object> buildFailedJobInfo(String site, long startTime, long endTime) {
+ Map<String, Object> data = new HashMap<>();
+ String query = String.format("%s[@site=\"%s\" and @currentState=\"FAILED\" and @endTime<=%...@user>{count}.{count desc}",
+ Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+
+ Map<String, Long> failedJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
+ if (failedJobUsers == null || failedJobUsers.isEmpty()) {
+ LOG.warn("Result set is empty for query={}", query);
+ return data;
+ }
+ data.put(FAILED_JOB_USERS_KEY, failedJobUsers);
+ data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s",
+ config.getString(SERVICE_HOST),
+ config.getInt(SERVICE_PORT),
+ site,
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime)));
+ return data;
+ }
+
+ private Map<String, Object> buildSucceededJobInfo(String site, long startTime, long endTime) {
+ Map<String, Object> data = new HashMap<>();
+ long overtimeLimit = jobOvertimeLimit * DateTimeUtil.ONEHOUR;
+ String query = String.format("%s[@site=\"%s\" and @currentState=\"SUCCEEDED\" and @durationTime>%s and @endTime<=%...@user>{count}.{count desc}",
+ Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, overtimeLimit, endTime);
+ Map<String, Long> succeededJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
+ if (succeededJobUsers == null || succeededJobUsers.isEmpty()) {
+ LOG.warn("Result set is empty for query={}", query);
+ return data;
+ }
+ data.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
+ return data;
+ }
+
+ private Map<String, Object> buildFinishedJobInfo(String site, long startTime, long endTime) {
+ Map<String, Object> data = new HashMap<>();
+ String query = String.format("%s[@site=\"%s\" and @endTime<=%...@user>{count}.{count desc}",
+ Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+ Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
+ if (jobUsers == null || jobUsers.isEmpty()) {
+ LOG.warn("Result set is empty for query={}", query);
+ return data;
+ }
+ data.put(FINISHED_JOB_USERS_KEY, jobUsers);
+ return data;
+ }
+
+ @Override
+ protected Scheduler scheduler() {
+ return Scheduler.newFixedRateSchedule(initialDelayMin, periodInMin, TimeUnit.MINUTES);
+ }
+
+ public static class JobSummeryInfo {
+ public String status;
+ public long numOfJobs;
+ public String ratio;
+
+ public String getStatus() {
+ return status;
+ }
+
+ public long getNumOfJobs() {
+ return numOfJobs;
+ }
+
+ public String getRatio() {
+ return ratio;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
new file mode 100644
index 0000000..fef83da
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
@@ -0,0 +1,141 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ #set ( $alert = $alertList[0] )
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+ <meta name="viewport" content="width=device-width"/>
+ <title>$alert["alertTitle"]</title>
+</head>
+<body>
+<table class="body">
+ <tr>
+ <td align="center" valign="top">
+ <!-- Eagle Body -->
+ <table width="580">
+ <tr>
+ <!-- Title -->
+ <td align="center">
+ <h2>$alert["alertTitle"]</h2>
+ </td>
+ </tr>
+
+ <!-- Basic Information -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Summery ($alert["reportRange"])</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Metrics</th>
+ <th>Number of Jobs</th>
+ <th>Ratio</th>
+ </tr>
+ #foreach($item in $alert["summaryInfo"])
+ <tr>
+ <td>$item.status</td>
+ <td>$item.numOfJobs</td>
+ <td>$item.ratio</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+
+ <!-- Top Users for Failed Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Failed Jobs)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Failed Jobs</th>
+ </tr>
+ #foreach($userItem in $alert["failedJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p>View more job information on <a href="$alert["eagleJobLink"]">Eagle</a></p>
+ </td>
+ </tr>
+
+ <!-- Top Users for Failed Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Succeeded Jobs Running over $alert["jobOvertimeLimit"] Hours)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Jobs Runing over $alert["jobOvertimeLimit"] hrs</th>
+ </tr>
+ #foreach($userItem in $alert["succeededJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+
+ <!-- Top Users for All Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Finished Jobs)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Finished Jobs</th>
+ </tr>
+ #foreach($userItem in $alert["finishedJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+</table>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
new file mode 100644
index 0000000..13c3350
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.mr.history;
+
+import com.dumbster.smtp.SimpleSmtpServer;
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.common.DateTimeUtil;
+import org.junit.After;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+import static org.apache.eagle.jpm.mr.history.MRHistoryJobDailyReporter.*;
+
+public class MRHistoryJobDailyReporterTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporterTest.class);
+ private static final int SMTP_PORT = 5025;
+ private Config config;
+ private SimpleSmtpServer server;
+
+ @Before
+ public void setUp(){
+ config = ConfigFactory.load("application-test.conf");
+ server = SimpleSmtpServer.start(SMTP_PORT);
+ }
+
+ @After
+ public void clear(){
+ if(server!=null) {
+ server.stop();
+ }
+ }
+ @Test
+ public void test() throws Exception {
+ MRHistoryJobDailyReporter reporter = new MRHistoryJobDailyReporter(config, null);
+ reporter.sendByEmail(mockAlertData());
+ Iterator it = server.getReceivedEmail();
+ Assert.assertTrue(server.getReceivedEmailSize() == 1);
+ while (it.hasNext()) {
+ LOG.info(it.next().toString());
+ }
+ }
+
+ private Map<String, Object> mockAlertData() {
+ Map<String, Object> alertData = new HashMap<>();
+ List<MRHistoryJobDailyReporter.JobSummeryInfo> summeryInfos = new ArrayList<>();
+ MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo1 = new MRHistoryJobDailyReporter.JobSummeryInfo();
+ summeryInfo1.status = "failed";
+ summeryInfo1.numOfJobs = 10;
+ summeryInfo1.ratio = "0.1";
+ MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo2 = new MRHistoryJobDailyReporter.JobSummeryInfo();
+ summeryInfo2.status = "succeeded";
+ summeryInfo2.numOfJobs = 90;
+ summeryInfo2.ratio = "0.9";
+ summeryInfos.add(summeryInfo1);
+ summeryInfos.add(summeryInfo2);
+ alertData.put(SUMMARY_INFO_KEY, summeryInfos);
+
+ Map<String,Double> failedJobUsers = new TreeMap<>();
+ failedJobUsers.put("alice", 100d);
+ failedJobUsers.put("bob", 97d);
+ alertData.put(FAILED_JOB_USERS_KEY, failedJobUsers);
+
+
+ Map<String,Double> succeededJobUsers = new TreeMap<>();
+ succeededJobUsers.put("alice1", 100d);
+ succeededJobUsers.put("bob1", 97d);
+ alertData.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
+
+
+ Map<String,Double> finishedJobUsers = new TreeMap<>();
+ finishedJobUsers.put("alice2", 100d);
+ finishedJobUsers.put("bob2", 97d);
+ alertData.put(FINISHED_JOB_USERS_KEY, finishedJobUsers);
+
+ alertData.put(ALERT_TITLE_KEY, "Daily Job Report");
+ alertData.put(NUM_TOP_USERS_KEY, 2);
+ alertData.put(JOB_OVERTIME_LIMIT_KEY, 6);
+ long currentTimestamp = System.currentTimeMillis();
+ alertData.put(REPORT_RANGE_KEY, String.format(" %s ~ %s %s",
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(currentTimestamp - 6 * DateTimeUtil.ONEHOUR),
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(currentTimestamp), DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+ alertData.put("joblink", "http://localhost:9090");
+ return alertData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm
new file mode 100644
index 0000000..fef83da
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/JobReportTemplate.vm
@@ -0,0 +1,141 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ #set ( $alert = $alertList[0] )
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+ <meta name="viewport" content="width=device-width"/>
+ <title>$alert["alertTitle"]</title>
+</head>
+<body>
+<table class="body">
+ <tr>
+ <td align="center" valign="top">
+ <!-- Eagle Body -->
+ <table width="580">
+ <tr>
+ <!-- Title -->
+ <td align="center">
+ <h2>$alert["alertTitle"]</h2>
+ </td>
+ </tr>
+
+ <!-- Basic Information -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Summery ($alert["reportRange"])</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Metrics</th>
+ <th>Number of Jobs</th>
+ <th>Ratio</th>
+ </tr>
+ #foreach($item in $alert["summaryInfo"])
+ <tr>
+ <td>$item.status</td>
+ <td>$item.numOfJobs</td>
+ <td>$item.ratio</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+
+ <!-- Top Users for Failed Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Failed Jobs)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Failed Jobs</th>
+ </tr>
+ #foreach($userItem in $alert["failedJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p>View more job information on <a href="$alert["eagleJobLink"]">Eagle</a></p>
+ </td>
+ </tr>
+
+ <!-- Top Users for Failed Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Succeeded Jobs Running over $alert["jobOvertimeLimit"] Hours)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Jobs Runing over $alert["jobOvertimeLimit"] hrs</th>
+ </tr>
+ #foreach($userItem in $alert["succeededJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+
+ <!-- Top Users for All Jobs -->
+ <tr>
+ <td style="padding: 20px 0 10px 0;">
+ <p><b>Top $alert["numTopUsers"] Users (Order by Number of Finished Jobs)</b></p>
+ </td>
+ </tr>
+ <tr>
+ <td>
+ <table class="tableBordered" width="580">
+ <tr>
+ <th>Name</th>
+ <th>Number of Finished Jobs</th>
+ </tr>
+ #foreach($userItem in $alert["finishedJobUsers"].entrySet())
+ <tr>
+ <td>$userItem.key</td>
+ <td>$userItem.value</td>
+ </tr>
+ #end
+ </table>
+ </td>
+ </tr>
+ </table>
+ </td>
+ </tr>
+</table>
+</body>
+</html>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf
new file mode 100644
index 0000000..057e189
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application-test.conf
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+service {
+ env = "testing"
+ host = "localhost"
+ port = 9090
+ username = "admin"
+ password = "secret"
+ readTimeOutSeconds = 60
+ context = "/rest"
+ timezone = "UTC"
+}
+
+
+# ---------------------------------------------
+# Eagle Application Configuration
+# ---------------------------------------------
+application {
+ sink {
+ type = org.apache.eagle.app.sink.KafkaStreamSink
+ }
+ storm {
+ nimbusHost = "server.eagle.apache.org"
+ nimbusThriftPort = 6627
+ }
+ mailService {
+ mailSmtpServer = "localhost",
+ mailSmtpPort = 5025,
+ mailSmtpAuth = "false"
+ //mailSmtpConn = "plaintext",
+ //mailSmtpUsername = ""
+ //mailSmtpPassword = ""
+ //mailSmtpDebug = false
+ }
+ dailyJobReport {
+ reportHourTime: 0
+ reportPeriodInHour: 12
+ numTopUsers : 10
+ jobOvertimeLimitInHour: 6
+ subject: "Job Report For 12 hours"
+ recipients: "nobody@abc.com"
+ template: "JobReportTemplate.vm"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties
new file mode 100644
index 0000000..71a5dac
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/log4j.properties
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, DRFA, stdout
+eagle.log.dir=./logs
+eagle.log.file=eagle.log
+
+# standard output
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
+
+# Daily Rolling File Appender
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${eagle.log.dir}/${eagle.log.file}
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p [%t] %c{2}[%L]: %m%n
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server-assembly/src/main/conf/eagle.conf
----------------------------------------------------------------------
diff --git a/eagle-server-assembly/src/main/conf/eagle.conf b/eagle-server-assembly/src/main/conf/eagle.conf
index 40a8c1e..af7e14a 100644
--- a/eagle-server-assembly/src/main/conf/eagle.conf
+++ b/eagle-server-assembly/src/main/conf/eagle.conf
@@ -90,6 +90,24 @@ application {
nimbusHost = "server.eagle.apache.org"
nimbusThriftPort = 6627
}
+ mailService {
+ mailSmtpServer = "",
+ mailSmtpPort = 25,
+ mailSmtpAuth = "false"
+ //mailSmtpConn = "plaintext",
+ //mailSmtpUsername = ""
+ //mailSmtpPassword = ""
+ //mailSmtpDebug = false
+ }
+ dailyJobReport {
+ reportHourTime: 1
+ reportPeriodInHour: 12
+ numTopUsers : 10
+ jobOvertimeLimitInHour: 6
+ subject: "Job Report For 12 hours"
+ recipients: "nobody@abc.com"
+ template: "JobReportTemplate.vm"
+ }
}
# ---------------------------------------------
@@ -116,13 +134,6 @@ coordinator {
host = "localhost",
port = 9090,
context = "/rest"
- mailSmtpServer = "",
- mailSmtpPort = 25,
- mailSmtpAuth = "false"
- //mailSmtpConn = "plaintext",
- //mailSmtpUsername = ""
- //mailSmtpPassword = ""
- //mailSmtpDebug = false
}
metadataDynamicCheck {
initDelayMillis = 1000
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java
index 495ec95..1d78ed1 100644
--- a/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java
+++ b/eagle-server/src/main/java/org/apache/eagle/server/ServerApplication.java
@@ -19,6 +19,7 @@ package org.apache.eagle.server;
import com.google.inject.Inject;
import com.hubspot.dropwizard.guice.GuiceBundle;
import com.sun.jersey.api.core.PackagesResourceConfig;
+import com.typesafe.config.Config;
import io.dropwizard.Application;
import io.dropwizard.assets.AssetsBundle;
import io.dropwizard.lifecycle.Managed;
@@ -30,6 +31,7 @@ import org.apache.eagle.alert.coordinator.CoordinatorListener;
import org.apache.eagle.alert.resource.SimpleCORSFiler;
import org.apache.eagle.app.service.ApplicationHealthCheckService;
import org.apache.eagle.common.Version;
+import org.apache.eagle.jpm.mr.history.MRHistoryJobDailyReporter;
import org.apache.eagle.log.base.taggedlog.EntityJsonModule;
import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
import org.apache.eagle.metadata.service.ApplicationStatusUpdateService;
@@ -48,6 +50,10 @@ class ServerApplication extends Application<ServerConfig> {
private ApplicationStatusUpdateService applicationStatusUpdateService;
@Inject
private ApplicationHealthCheckService applicationHealthCheckService;
+ @Inject
+ private MRHistoryJobDailyReporter mrHistoryJobDailyReporter;
+ @Inject
+ private Config config;
@Override
public void initialize(Bootstrap<ServerConfig> bootstrap) {
@@ -109,5 +115,10 @@ class ServerApplication extends Application<ServerConfig> {
applicationHealthCheckService.init(environment);
Managed appHealthCheckTask = new ApplicationTask(applicationHealthCheckService);
environment.lifecycle().manage(appHealthCheckTask);
+
+ if (config.hasPath(MRHistoryJobDailyReporter.SERVICE_PATH)) {
+ Managed jobReportTask = new ApplicationTask(mrHistoryJobDailyReporter);
+ environment.lifecycle().manage(jobReportTask);
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/f430f521/eagle-server/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf
index 84b467c..d657f54 100644
--- a/eagle-server/src/main/resources/application.conf
+++ b/eagle-server/src/main/resources/application.conf
@@ -110,6 +110,24 @@ application {
mail.smtp.template = "HealthCheckTemplate.vm"
}
}
+ mailService {
+ mailSmtpServer = "",
+ mailSmtpPort = 25,
+ mailSmtpAuth = "false"
+ //mailSmtpConn = "plaintext",
+ //mailSmtpUsername = ""
+ //mailSmtpPassword = ""
+ //mailSmtpDebug = false
+ }
+ dailyJobReport {
+ reportHourTime: 1
+ reportPeriodInHour: 12
+ numTopUsers : 10
+ jobOvertimeLimitInHour: 6
+ subject: "Job Report For 12 hours"
+ recipients: "nobody@abc.com"
+ template: "JobReportTemplate.vm"
+ }
}
# ---------------------------------------------
@@ -135,14 +153,7 @@ coordinator {
metadataService {
host = "localhost",
port = 9090,
- context = "/rest",
- mailSmtpServer = "",
- mailSmtpPort = 25,
- mailSmtpAuth = "false"
- //mailSmtpConn = "plaintext",
- //mailSmtpUsername = ""
- //mailSmtpPassword = ""
- //mailSmtpDebug = false
+ context = "/rest"
}
metadataDynamicCheck {
initDelayMillis = 1000