You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2016/12/16 07:04:07 UTC
incubator-eagle git commit: [EAGLE-834] Update Daily Job Summery
Report
Repository: incubator-eagle
Updated Branches:
refs/heads/master 229d7b907 -> a89275bfc
[EAGLE-834] Update Daily Job Summery Report
https://issues.apache.org/jira/browse/EAGLE-834
* update mail subject & title
* adjust mail template and enrich the summary data
* fix a query bug in the mr job list API (GET /mrJobs)
Author: Zhao, Qingwen <qi...@apache.org>
Closes #749 from qingwen220/EAGLE-834.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/a89275bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/a89275bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/a89275bf
Branch: refs/heads/master
Commit: a89275bfc86c4db63688e28ff12196ee97a78207
Parents: 229d7b9
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Fri Dec 16 15:04:00 2016 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Fri Dec 16 15:04:00 2016 +0800
----------------------------------------------------------------------
.../mr/history/MRHistoryJobDailyReporter.java | 169 +++++++++----------
.../src/main/resources/JobReportTemplate.vm | 118 +++++++------
.../history/MRHistoryJobDailyReporterTest.java | 43 ++---
.../service/jpm/MRJobExecutionResource.java | 26 +--
4 files changed, 177 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
index 6b38244..0dc6c5f 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporter.java
@@ -39,19 +39,20 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
+import static org.apache.eagle.common.config.EagleConfigConstants.EAGLE_TIME_ZONE;
import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_HOST;
import static org.apache.eagle.common.config.EagleConfigConstants.SERVICE_PORT;
public class MRHistoryJobDailyReporter extends AbstractScheduledService {
private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobDailyReporter.class);
- private static final String TIMEZONE_PATH = "service.timezone";
+
private static final String DAILY_SENT_HOUROFDAY = "application.dailyJobReport.reportHourTime";
private static final String DAILY_SENT_PERIOD = "application.dailyJobReport.reportPeriodInHour";
private static final String NUM_TOP_USERS = "application.dailyJobReport.numTopUsers";
private static final String JOB_OVERTIME_LIMIT_HOUR = "application.dailyJobReport.jobOvertimeLimitInHour";
public static final String SERVICE_PATH = "application.dailyJobReport";
- public static final String APP_TYPE = "MR_HISTORY_JOB_APP";
+ protected static final String APP_TYPE = "MR_HISTORY_JOB_APP";
// alert context keys
protected static final String NUM_TOP_USERS_KEY = "numTopUsers";
@@ -64,6 +65,12 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
protected static final String FINISHED_JOB_USERS_KEY = "finishedJobUsers";
protected static final String EAGLE_JOB_LINK_KEY = "eagleJobLink";
+ // queries
+ private static final String STATUS_QUERY = "%s[@site=\"%s\" and @endTime<=%...@currentState>{count}.{count desc}";
+ private static final String FAILED_JOBS_QUERY = "%s[@site=\"%s\" and @currentState=\"FAILED\" and @endTime<=%...@user>{count}.{count desc}";
+ private static final String SUCCEEDED_JOB_QUERY = "%s[@site=\"%s\" and @currentState=\"SUCCEEDED\" and @durationTime>%s and @endTime<=%...@user>{count}.{count desc}";
+ private static final String FINISHED_JOB_QUERY = "%s[@site=\"%s\" and @endTime<=%...@user>{count}.{count desc}";
+
private Config config;
private IEagleServiceClient client;
private ApplicationEntityService applicationResource;
@@ -77,13 +84,13 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
private int jobOvertimeLimit = 6;
// scheduler
- private int initialDelayMin = 10;
+ private int initialDelayMin = 5;
private int periodInMin = 60;
private TimeZone timeZone;
@Inject
public MRHistoryJobDailyReporter(Config config, ApplicationEntityService applicationEntityService) {
- this.timeZone = TimeZone.getTimeZone(config.getString(TIMEZONE_PATH));
+ this.timeZone = TimeZone.getTimeZone(config.getString(EAGLE_TIME_ZONE));
if (config.hasPath(SERVICE_PATH) && config.hasPath(AlertEmailConstants.EAGLE_APPLICATION_EMAIL_SERVICE)) {
this.emailService = new ApplicationEmailService(config, SERVICE_PATH);
@@ -139,8 +146,10 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
for (String site : sites) {
int reportHour = currentHour / dailySentPeriod * dailySentPeriod;
calendar.set(Calendar.HOUR_OF_DAY, reportHour);
- String subject = String.format("%s %s", site.toUpperCase(), config.getString(SERVICE_PATH + "." + AlertEmailConstants.SUBJECT));
- Map<String, Object> alertData = buildAlertData(site, calendar.getTimeInMillis());
+ long endTime = calendar.getTimeInMillis() / DateTimeUtil.ONEHOUR * DateTimeUtil.ONEHOUR;
+ long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod;
+ String subject = buildAlertSubject(site, startTime, endTime);
+ Map<String, Object> alertData = buildAlertData(site, startTime, endTime);
sendByEmailWithSubject(alertData, subject);
}
} catch (Exception ex) {
@@ -159,27 +168,30 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
emailService.onAlert(alertContext, alertData);
}
- private Map<String, Object> buildAlertData(String site, long endTime) {
+ protected String buildAlertSubject(String site, long startTime, long endTime) {
+ String subjectFormat = "[%s] Job Report by %s";
+ String date = DateTimeUtil.format(endTime, "yyyyMMdd HH:mm");
+ //String startHour = DateTimeUtil.format(startTime, "HH:mm");
+ //String endHour = DateTimeUtil.format(endTime, "kk:mm");
+ return String.format(subjectFormat, site.toUpperCase(), date);
+ }
+
+ private Map<String, Object> buildAlertData(String site, long startTime, long endTime) {
StopWatch watch = new StopWatch();
Map<String, Object> data = new HashMap<>();
this.client = new EagleServiceClientImpl(config);
- long startTime = endTime - DateTimeUtil.ONEHOUR * dailySentPeriod;
- LOG.info("Going to report job summery info for site {} from {} to {}", site,
- DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
- DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime));
+ String startTimeStr = DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime);
+ String endTimeStr = DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime);
+ LOG.info("Going to report job summery info for site {} from {} to {}", site, startTimeStr, endTimeStr);
try {
watch.start();
data.putAll(buildJobSummery(site, startTime, endTime));
- data.putAll(buildFailedJobInfo(site, startTime, endTime));
- data.putAll(buildSucceededJobInfo(site, startTime, endTime));
- data.putAll(buildFinishedJobInfo(site, startTime, endTime));
data.put(NUM_TOP_USERS_KEY, numTopUsers);
data.put(JOB_OVERTIME_LIMIT_KEY, jobOvertimeLimit);
- data.put(ALERT_TITLE_KEY, String.format("[%s] Daily Job Report", site.toUpperCase()));
- data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s",
- DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
- DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime),
- DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+ data.put(ALERT_TITLE_KEY, String.format("[%s] Job Report for 12 Hours", site.toUpperCase()));
+ data.put(REPORT_RANGE_KEY, String.format("%s ~ %s %s", startTimeStr, endTimeStr, DateTimeUtil.CURRENT_TIME_ZONE.getID()));
+ data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s",
+ config.getString(SERVICE_HOST), config.getInt(SERVICE_PORT), site, startTimeStr, endTimeStr));
watch.stop();
LOG.info("Fetching DailyJobReport tasks {} seconds", watch.getTime() / DateTimeUtil.ONESECOND);
} finally {
@@ -192,6 +204,48 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
return data;
}
+ private Map<String, Object> buildJobSummery(String site, long startTime, long endTime) {
+ Map<String, Object> data = new HashMap<>();
+
+ String query = String.format(STATUS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+ Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE);
+ if (jobSummery == null || jobSummery.isEmpty()) {
+ LOG.warn("Result set is empty for query={}", query);
+ return data;
+ }
+ Long totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b).get();
+ String finishedJobQuery = String.format(FINISHED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+ String failedJobQuery = String.format(FAILED_JOBS_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
+ String succeededJobQuery = String.format(SUCCEEDED_JOB_QUERY, Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, jobOvertimeLimit * DateTimeUtil.ONEHOUR, endTime);
+ data.put(SUMMARY_INFO_KEY, processResult(jobSummery, totalJobs));
+ data.put(FAILED_JOB_USERS_KEY, buildJobSummery(failedJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.FAILED.toString())));
+ data.put(SUCCEEDED_JOB_USERS_KEY, buildJobSummery(succeededJobQuery, startTime, endTime, jobSummery.get(Constants.JobState.SUCCEEDED.toString())));
+ data.put(FINISHED_JOB_USERS_KEY, buildJobSummery(finishedJobQuery, startTime, endTime, totalJobs));
+
+ return data;
+ }
+
+ private List<JobSummaryInfo> buildJobSummery(String query, long startTime, long endTime, long totalJobs) {
+ Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
+ if (jobUsers == null || jobUsers.isEmpty()) {
+ LOG.warn("Result set is empty for query={}", query);
+ return null;
+ }
+ return processResult(jobUsers, totalJobs);
+ }
+
+ private List<JobSummaryInfo> processResult(Map<String, Long> parsedResult, long totalJobs) {
+ List<JobSummaryInfo> summaryInfoList = new ArrayList<>();
+ for (Map.Entry<String, Long> entry : parsedResult.entrySet()) {
+ JobSummaryInfo summaryInfo = new JobSummaryInfo();
+ summaryInfo.key = entry.getKey();
+ summaryInfo.numOfJobs = entry.getValue();
+ summaryInfo.ratio = Double.parseDouble(String.format("%.2f", summaryInfo.numOfJobs * 100d / totalJobs));
+ summaryInfoList.add(summaryInfo);
+ }
+ return summaryInfoList;
+ }
+
private Map<String, Long> parseQueryResult(List<Map<List<String>, List<Double>>> result, int limit) {
Map<String, Long> stateCount = new LinkedHashMap<>();
for (Map<List<String>, List<Double>> map : result) {
@@ -213,6 +267,7 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
.startTime(startTime)
.endTime(endTime).send();
if (!response.isSuccess()) {
+ LOG.error(response.getException());
return null;
}
List<Map<List<String>, List<Double>>> result = response.getObj();
@@ -223,86 +278,18 @@ public class MRHistoryJobDailyReporter extends AbstractScheduledService {
}
}
- private Map<String, Object> buildJobSummery(String site, long startTime, long endTime) {
- Map<String, Object> data = new HashMap<>();
- List<JobSummeryInfo> statusCount = new ArrayList<>();
- String query = String.format("%s[@site=\"%s\" and @endTime<=%...@currentState>{count}",
- Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
- Map<String, Long> jobSummery = queryGroupByMetrics(query, startTime, endTime, Integer.MAX_VALUE);
- if (jobSummery == null || jobSummery.isEmpty()) {
- return data;
- }
- Optional<Long> totalJobs = jobSummery.values().stream().reduce((a, b) -> a + b);
- for (Map.Entry<String, Long> entry : jobSummery.entrySet()) {
- JobSummeryInfo summeryInfo = new JobSummeryInfo();
- summeryInfo.status = entry.getKey();
- summeryInfo.numOfJobs = entry.getValue();
- summeryInfo.ratio = Double.parseDouble(String.format("%.2f", entry.getValue() * 1d / totalJobs.get()));
- statusCount.add(summeryInfo);
- }
- data.put(SUMMARY_INFO_KEY, statusCount);
- return data;
- }
-
- private Map<String, Object> buildFailedJobInfo(String site, long startTime, long endTime) {
- Map<String, Object> data = new HashMap<>();
- String query = String.format("%s[@site=\"%s\" and @currentState=\"FAILED\" and @endTime<=%...@user>{count}.{count desc}",
- Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
-
- Map<String, Long> failedJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
- if (failedJobUsers == null || failedJobUsers.isEmpty()) {
- LOG.warn("Result set is empty for query={}", query);
- return data;
- }
- data.put(FAILED_JOB_USERS_KEY, failedJobUsers);
- data.put(EAGLE_JOB_LINK_KEY, String.format("http://%s:%d/#/site/%s/jpm/list?startTime=%s&endTime=%s",
- config.getString(SERVICE_HOST),
- config.getInt(SERVICE_PORT),
- site,
- DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime),
- DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime)));
- return data;
- }
-
- private Map<String, Object> buildSucceededJobInfo(String site, long startTime, long endTime) {
- Map<String, Object> data = new HashMap<>();
- long overtimeLimit = jobOvertimeLimit * DateTimeUtil.ONEHOUR;
- String query = String.format("%s[@site=\"%s\" and @currentState=\"SUCCEEDED\" and @durationTime>%s and @endTime<=%...@user>{count}.{count desc}",
- Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, overtimeLimit, endTime);
- Map<String, Long> succeededJobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
- if (succeededJobUsers == null || succeededJobUsers.isEmpty()) {
- LOG.warn("Result set is empty for query={}", query);
- return data;
- }
- data.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
- return data;
- }
-
- private Map<String, Object> buildFinishedJobInfo(String site, long startTime, long endTime) {
- Map<String, Object> data = new HashMap<>();
- String query = String.format("%s[@site=\"%s\" and @endTime<=%...@user>{count}.{count desc}",
- Constants.JPA_JOB_EXECUTION_SERVICE_NAME, site, endTime);
- Map<String, Long> jobUsers = queryGroupByMetrics(query, startTime, endTime, numTopUsers);
- if (jobUsers == null || jobUsers.isEmpty()) {
- LOG.warn("Result set is empty for query={}", query);
- return data;
- }
- data.put(FINISHED_JOB_USERS_KEY, jobUsers);
- return data;
- }
-
@Override
protected Scheduler scheduler() {
return Scheduler.newFixedRateSchedule(initialDelayMin, periodInMin, TimeUnit.MINUTES);
}
- public static class JobSummeryInfo {
- public String status;
+ public static class JobSummaryInfo {
+ public String key;
public long numOfJobs;
public double ratio;
- public String getStatus() {
- return status;
+ public String getKey() {
+ return key;
}
public long getNumOfJobs() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
index c3afa58..c878d6e 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/resources/JobReportTemplate.vm
@@ -139,7 +139,7 @@
</tr>
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<td class="content-block"
- style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+ style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
valign="top">
<h2 class="aligncenter" style="font-family: 'Helvetica Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 20px; color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
Summary
@@ -176,12 +176,12 @@
<td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
valign="top">
#set($statusColor = "#337ab7")
- #if($item.status == "SUCCEEDED")
+ #if($item.key == "SUCCEEDED")
#set($statusColor = "green")
- #elseif($item.status == "FAILED" || $item.status == "KILLED")
+ #elseif($item.key == "FAILED" || $item.key == "KILLED")
#set($statusColor = "red")
#end
- <span style="color: $statusColor">$item.status</span>
+ <span style="color: $statusColor">$item.key</span>
</td>
<td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
valign="top">
@@ -189,8 +189,9 @@
</td>
<td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
valign="top">
- #set ($ratio_percentage = $item.ratio * 100)
- $ratio_percentage %
+## #set ($ratio_percentage = $item.ratio * 100)
+## $ratio_percentage %
+ $item.ratio %
</td>
</tr>
#end
@@ -203,11 +204,11 @@
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<td class="content-block"
- style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+ style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
valign="top">
## Top $alert["numTopUsers"] Users (Order by Number of Failed Jobs)
<h2 class="aligncenter" style="font-family: 'Helvetica Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px; color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
- Top $alert["numTopUsers"] Failed Job Users
+ Top $alert["numTopUsers"] Users by Failed Jobs
</h2>
</td>
</tr>
@@ -236,19 +237,21 @@
Ratio
</th>
</tr>
- #foreach($userItem in $alert["failedJobUsers"].entrySet())
- <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
- <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">
- $userItem.key
- </td>
- <td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">$userItem.value
- </td>
- <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">$TODO_RATIO
- </td>
- </tr>
+ #foreach($item in $alert["failedJobUsers"])
+ <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+ <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.key
+ </td>
+ <td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.numOfJobs
+ </td>
+ <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.ratio %
+ </td>
+ </tr>
#end
</table>
</td>
@@ -261,15 +264,15 @@
<td class="content-block"
style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
valign="top">
- <a href="$alert["eagleJobLink"]" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #999; text-decoration: underline; margin: 0;">
- View Jobs on Eagle
+ <a href="$alert["eagleJobLink"]" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; color: #337ab7; text-decoration: underline; margin: 0;">
+ View Failed Jobs on Eagle
</a>
</td>
</tr>
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
- <td class="content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;" valign="top">
+ <td class="content-block" style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;" valign="top">
<h2 class="aligncenter" style="font-family: 'Helvetica Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px; color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
- Top $alert["numTopUsers"] Finished* Job Users
+ Top $alert["numTopUsers"] Users by Finished* Job
</h2>
</td>
</tr>
@@ -298,20 +301,21 @@
Ratio
</th>
</tr>
- #foreach($userItem in $alert["finishedJobUsers"].entrySet())
- <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
- <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">
- $userItem.key
- </td>
- <td style="text-align:center; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">
- $userItem.value
- </td>
- <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">$TODO_RATIO
- </td>
- </tr>
+ #foreach($item in $alert["finishedJobUsers"])
+ <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+ <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.key
+ </td>
+ <td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.numOfJobs
+ </td>
+ <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.ratio %
+ </td>
+ </tr>
#end
</table>
</td>
@@ -322,10 +326,10 @@
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
<td class="content-block"
- style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
+ style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 0px;"
valign="top">
<h2 class="aligncenter" style="font-family: 'Helvetica Neue',Helvetica,Arial,'Lucida Grande',sans-serif; box-sizing: border-box; font-size: 18px; color: #000; line-height: 1.2em; font-weight: 400; text-align: left; margin: 0 0 0;" align="center">
- Top $alert["numTopUsers"] Succeeded Long Running* Job Users
+ Top $alert["numTopUsers"] Users by Long Running* Job
</h2>
</td>
</tr>
@@ -355,19 +359,21 @@
Ratio
</th>
</tr>
- #foreach($userItem in $alert["succeededJobUsers"].entrySet())
- <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
- <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">
- $userItem.key
- </td>
- <td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">$userItem.value
- </td>
- <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
- valign="top">$TODO_RATIO
- </td>
- </tr>
+ #foreach($item in $alert["succeededJobUsers"])
+ <tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
+ <td style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.key
+ </td>
+ <td style="text-align: center;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.numOfJobs
+ </td>
+ <td style="text-align: right;font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; border-top-width: 1px; border-top-color: #eee; border-top-style: solid; margin: 0; padding: 5px 0;"
+ valign="top">
+ $item.ratio %
+ </td>
+ </tr>
#end
</table>
</td>
@@ -380,8 +386,8 @@
style="text-align:left; font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
valign="top">
* Notes:<br/>
- 1) Finished jobs include those <i>SUCCEEDED</i>, <i>FAILED</i> and <i>KILLED</i>.<br/>
- 2) Long running jobs mean those duration over $alert["jobOvertimeLimit"] hours.
+ 1) Finished jobs include those <i>SUCCEEDED</i>, <i>FAILED</i>, <i>KILLED</i>, etc.<br/>
+ 2) Long running jobs mean those succeeded jobs with duration over $alert["jobOvertimeLimit"] hours.
</td>
</tr>
</table>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
index d80ae37..3b297ae 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/MRHistoryJobDailyReporterTest.java
@@ -52,6 +52,7 @@ public class MRHistoryJobDailyReporterTest {
server.stop();
}
}
+
@Test
public void test() throws Exception {
MRHistoryJobDailyReporter reporter = new MRHistoryJobDailyReporter(config, null);
@@ -65,33 +66,26 @@ public class MRHistoryJobDailyReporterTest {
private Map<String, Object> mockAlertData() {
Map<String, Object> alertData = new HashMap<>();
- List<MRHistoryJobDailyReporter.JobSummeryInfo> summeryInfos = new ArrayList<>();
- MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo1 = new MRHistoryJobDailyReporter.JobSummeryInfo();
- summeryInfo1.status = "FAILED";
- summeryInfo1.numOfJobs = 10;
- summeryInfo1.ratio = 0.1;
- MRHistoryJobDailyReporter.JobSummeryInfo summeryInfo2 = new MRHistoryJobDailyReporter.JobSummeryInfo();
- summeryInfo2.status = "SUCCEEDED";
- summeryInfo2.numOfJobs = 90;
- summeryInfo2.ratio = 0.9;
- summeryInfos.add(summeryInfo1);
- summeryInfos.add(summeryInfo2);
+ List<JobSummaryInfo> summeryInfos = new ArrayList<>();
+ summeryInfos.add(buildJobSummaryInfo("FAILED", 8, 8));
+ summeryInfos.add(buildJobSummaryInfo("SUCCEEDED", 90, 89.9));
+ summeryInfos.add(buildJobSummaryInfo("KILLED", 2, 2));
alertData.put(SUMMARY_INFO_KEY, summeryInfos);
- Map<String,Double> failedJobUsers = new TreeMap<>();
- failedJobUsers.put("alice", 100d);
- failedJobUsers.put("bob", 97d);
+ List<JobSummaryInfo> failedJobUsers = new ArrayList<>();
+ failedJobUsers.add(buildJobSummaryInfo("alice", 100L, 98d));
+ failedJobUsers.add(buildJobSummaryInfo("bob", 97L, 2));
alertData.put(FAILED_JOB_USERS_KEY, failedJobUsers);
- Map<String,Double> succeededJobUsers = new TreeMap<>();
- succeededJobUsers.put("alice1", 100d);
- succeededJobUsers.put("bob1", 97d);
+ List<JobSummaryInfo> succeededJobUsers = new ArrayList<>();
+ succeededJobUsers.add(buildJobSummaryInfo("alice1", 100L, 98));
+ succeededJobUsers.add(buildJobSummaryInfo("bob1", 97, 2));
alertData.put(SUCCEEDED_JOB_USERS_KEY, succeededJobUsers);
- Map<String,Double> finishedJobUsers = new TreeMap<>();
- finishedJobUsers.put("alice2", 100d);
- finishedJobUsers.put("bob2", 97d);
+ List<JobSummaryInfo> finishedJobUsers = new ArrayList<>();
+ finishedJobUsers.add(buildJobSummaryInfo("alice2", 100L, 98));
+ finishedJobUsers.add(buildJobSummaryInfo("bob2", 97, 2));
alertData.put(FINISHED_JOB_USERS_KEY, finishedJobUsers);
alertData.put(ALERT_TITLE_KEY, "[TEST_CLUSTER] Daily Job Report");
@@ -104,4 +98,13 @@ public class MRHistoryJobDailyReporterTest {
alertData.put(EAGLE_JOB_LINK_KEY, "http://localhost:9090/#/site/sandbox/jpm/statistics");
return alertData;
}
+
+ private JobSummaryInfo buildJobSummaryInfo(String key, long number, double ratio) {
+ JobSummaryInfo jobSummaryInfo = new JobSummaryInfo();
+ jobSummaryInfo.numOfJobs = number;
+ jobSummaryInfo.ratio = ratio;
+ jobSummaryInfo.key = key;
+ return jobSummaryInfo;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/a89275bf/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
index 42b6584..28e6bb3 100644
--- a/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
+++ b/eagle-jpm/eagle-jpm-service/src/main/java/org/apache/eagle/service/jpm/MRJobExecutionResource.java
@@ -59,30 +59,32 @@ public class MRJobExecutionResource {
@QueryParam("filterIfMissing") boolean filterIfMissing,
@QueryParam("parallel") int parallel,
@QueryParam("metricName") String metricName,
- @QueryParam("verbose") Boolean verbose) {
+ @QueryParam("verbose") Boolean verbose) throws ParseException {
GenericServiceAPIResponseEntity response = new GenericServiceAPIResponseEntity();
List<TaggedLogAPIEntity> jobs = new ArrayList<>();
- List<TaggedLogAPIEntity> finishedJobs = new ArrayList<>();
+ List<JobExecutionAPIEntity> finishedJobs = new ArrayList<>();
Set<String> jobIds = new HashSet<>();
final Map<String, Object> meta = new HashMap<>();
StopWatch stopWatch = new StopWatch();
stopWatch.start();
String jobQuery = String.format(query, Constants.JPA_JOB_EXECUTION_SERVICE_NAME);
- GenericServiceAPIResponseEntity<TaggedLogAPIEntity> res =
- resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin,
- top, filterIfMissing, parallel, metricName, verbose);
+ GenericServiceAPIResponseEntity<JobExecutionAPIEntity> res =
+ resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose);
if (res.isSuccess() && res.getObj() != null) {
- for (TaggedLogAPIEntity o : res.getObj()) {
- finishedJobs.add(o);
- jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
+ long maxFinishedTime = DateTimeUtil.humanDateToSeconds(endTime) * DateTimeUtil.ONESECOND;
+ for (JobExecutionAPIEntity o : res.getObj()) {
+ if (o.getEndTime() <= maxFinishedTime) {
+ finishedJobs.add(o);
+ jobIds.add(o.getTags().get(MRJobTagName.JOB_ID.toString()));
+ }
}
jobQuery = String.format(query, Constants.JPA_RUNNING_JOB_EXECUTION_SERVICE_NAME);
- res = resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin,
- top, filterIfMissing, parallel, metricName, verbose);
- if (res.isSuccess() && res.getObj() != null) {
- for (TaggedLogAPIEntity o : res.getObj()) {
+ GenericServiceAPIResponseEntity<org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity> runningRes =
+ resource.search(jobQuery, startTime, endTime, pageSize, startRowkey, treeAgg, timeSeries, intervalmin, top, filterIfMissing, parallel, metricName, verbose);
+ if (runningRes.isSuccess() && runningRes.getObj() != null) {
+ for (org.apache.eagle.jpm.mr.runningentity.JobExecutionAPIEntity o : runningRes.getObj()) {
String key = o.getTags().get(MRJobTagName.JOB_ID.toString());
if (!ResourceUtils.isDuplicate(jobIds, key)) {
jobs.add(o);