You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2018/11/26 20:27:43 UTC
[incubator-pinot] branch master updated: [TE] Display Holidays only
from Top Countries (#3544)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 234abc6 [TE] Display Holidays only from Top Countries (#3544)
234abc6 is described below
commit 234abc6f8cc1d8e3aa7adfa2a170fca002122b74
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Mon Nov 26 12:27:39 2018 -0800
[TE] Display Holidays only from Top Countries (#3544)
Instead of listing all the holidays in the email, display holidays from a whitelist of countries.
---
.../alert/content/BaseEmailContentFormatter.java | 16 ++++------------
.../content/EmailContentFormatterConfiguration.java | 11 +++++++++++
.../HierarchicalAnomaliesEmailContentFormatter.java | 7 ++++---
.../content/MultipleAnomaliesEmailContentFormatter.java | 9 +++------
.../thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java | 10 ++++++++++
.../linkedin/thirdeye/anomaly/events/EventFilter.java | 7 ++++---
.../thirdeye/anomaly/events/HolidayEventProvider.java | 17 +++++++++++------
.../eventprovider/TestHolidayEventProvider.java | 1 +
8 files changed, 48 insertions(+), 30 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/BaseEmailContentFormatter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/BaseEmailContentFormatter.java
index d0c7657..9f62687 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/BaseEmailContentFormatter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/BaseEmailContentFormatter.java
@@ -450,7 +450,7 @@ public abstract class BaseEmailContentFormatter implements EmailContentFormatter
, String metricName, String serviceName, Map<String, List<String>> targetDimensions) {
List<EventDTO> relatedEvents = new ArrayList<>();
for (EventType eventType : eventTypes) {
- relatedEvents.addAll(getRelatedEvents(eventType, start, end, metricName, serviceName, targetDimensions));
+ relatedEvents.addAll(getHolidayEvents(start, end, targetDimensions));
}
return relatedEvents;
}
@@ -509,27 +509,19 @@ public abstract class BaseEmailContentFormatter implements EmailContentFormatter
/**
* Taking advantage of event data provider, extract the events around the given start and end time
- * @param eventType a given event type
* @param start the start time of the event, preEventCrawlOffset is added before the given date time
* @param end the end time of the event, postEventCrawlOffset is added after the given date time
- * @param metricName the affected metric name
- * @param serviceName the affected service name
* @param targetDimensions the affected dimensions
* @return a list of related events
*/
- public List<EventDTO> getRelatedEvents(EventType eventType, DateTime start, DateTime end
- , String metricName, String serviceName, Map<String, List<String>> targetDimensions) {
- List<EventDTO> relatedEvents = new ArrayList<>();
-
- // Set Event Filters
+ public List<EventDTO> getHolidayEvents(DateTime start, DateTime end, Map<String, List<String>> targetDimensions) {
EventFilter eventFilter = new EventFilter();
+ eventFilter.setEventType(EventType.HOLIDAY.name());
eventFilter.setStartTime(start.minus(preEventCrawlOffset).getMillis());
eventFilter.setEndTime(end.plus(postEventCrawlOffset).getMillis());
- eventFilter.setMetricName(metricName);
- eventFilter.setServiceName(serviceName);
eventFilter.setTargetDimensionMap(targetDimensions);
- eventFilter.setEventType(eventType.toString());
+ LOG.info("Fetching holidays with preEventCrawlOffset {} and postEventCrawlOffset {}", preEventCrawlOffset, postEventCrawlOffset);
return new HolidayEventProvider().getEvents(eventFilter);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/EmailContentFormatterConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/EmailContentFormatterConfiguration.java
index fa93496..f904803 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/EmailContentFormatterConfiguration.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/EmailContentFormatterConfiguration.java
@@ -18,6 +18,7 @@ package com.linkedin.thirdeye.alert.content;
import com.linkedin.thirdeye.anomaly.SmtpConfiguration;
import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
+import java.util.List;
import static com.linkedin.thirdeye.anomaly.SmtpConfiguration.SMTP_CONFIG_KEY;
@@ -32,6 +33,7 @@ public class EmailContentFormatterConfiguration{
private String phantomJsPath = "";
private String failureFromAddress;
private String failureToAddress;
+ private List<String> holidayCountriesWhitelist;
public String getFunctionConfigPath() {
return functionConfigPath;
@@ -97,6 +99,14 @@ public class EmailContentFormatterConfiguration{
this.failureToAddress = failureToAddress;
}
+ public List<String> getHolidayCountriesWhitelist() {
+ return holidayCountriesWhitelist;
+ }
+
+ public void setHolidayCountriesWhitelist(List<String> holidayCountriesWhitelist) {
+ this.holidayCountriesWhitelist = holidayCountriesWhitelist;
+ }
+
public static EmailContentFormatterConfiguration fromThirdEyeAnomalyConfiguration(ThirdEyeAnomalyConfiguration thirdeyeConfig) {
EmailContentFormatterConfiguration emailConfig = new EmailContentFormatterConfiguration();
emailConfig.setDashboardHost(thirdeyeConfig.getDashboardHost());
@@ -106,6 +116,7 @@ public class EmailContentFormatterConfiguration{
emailConfig.setFunctionConfigPath(thirdeyeConfig.getFunctionConfigPath());
emailConfig.setAlertFilterConfigPath(thirdeyeConfig.getAlertFilterConfigPath());
emailConfig.setPhantomJsPath(thirdeyeConfig.getPhantomJsPath());
+ emailConfig.setHolidayCountriesWhitelist(thirdeyeConfig.getHolidayCountriesWhitelist());
emailConfig.setSmtpConfiguration(
SmtpConfiguration.createFromProperties(thirdeyeConfig.getAlerterConfiguration().get(SMTP_CONFIG_KEY)));
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/HierarchicalAnomaliesEmailContentFormatter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/HierarchicalAnomaliesEmailContentFormatter.java
index 9008529..3d2b1dd 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/HierarchicalAnomaliesEmailContentFormatter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/HierarchicalAnomaliesEmailContentFormatter.java
@@ -153,9 +153,10 @@ public class HierarchicalAnomaliesEmailContentFormatter extends BaseEmailContent
if (affectedCountries.size() > 0) { // if the anomaly is on country level
Map<String, List<String>> targetDimensions = new HashMap<>();
targetDimensions.put(EVENT_FILTER_COUNTRY, affectedCountries);
- relatedEvents.addAll(getRelatedEvents(EventType.HOLIDAY,
- new DateTime(anomaly.getStartTime(), dateTimeZone), new DateTime(anomaly.getEndTime(), dateTimeZone),
- null, null, targetDimensions));
+ relatedEvents.addAll(getHolidayEvents(
+ new DateTime(anomaly.getStartTime(), dateTimeZone),
+ new DateTime(anomaly.getEndTime(), dateTimeZone),
+ targetDimensions));
}
return anomalyReport;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/MultipleAnomaliesEmailContentFormatter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/MultipleAnomaliesEmailContentFormatter.java
index 24c5b27..dd82847 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/MultipleAnomaliesEmailContentFormatter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/alert/content/MultipleAnomaliesEmailContentFormatter.java
@@ -52,12 +52,9 @@ public class MultipleAnomaliesEmailContentFormatter extends BaseEmailContentForm
private static final Logger LOG = LoggerFactory.getLogger(MultipleAnomaliesEmailContentFormatter.class);
public static final String EMAIL_TEMPLATE = "emailTemplate";
- private static final String DATE_PATTERN = "MMM dd, HH:mm";
public static final String DEFAULT_EMAIL_TEMPLATE = "holiday-anomaly-report.ftl";
- private static final long EVENT_TIME_TOLERANCE = TimeUnit.DAYS.toMillis(2);
- private EventManager eventDAO = null;
private DetectionConfigManager configDAO = null;
public MultipleAnomaliesEmailContentFormatter(){
@@ -68,7 +65,6 @@ public class MultipleAnomaliesEmailContentFormatter extends BaseEmailContentForm
public void init(Properties properties, EmailContentFormatterConfiguration configuration) {
super.init(properties, configuration);
this.emailTemplate = properties.getProperty(EMAIL_TEMPLATE, DEFAULT_EMAIL_TEMPLATE);
- this.eventDAO = DAORegistry.getInstance().getEventDAO();
this.configDAO = DAORegistry.getInstance().getDetectionConfigManager();
}
@@ -163,8 +159,9 @@ public class MultipleAnomaliesEmailContentFormatter extends BaseEmailContentForm
// holidays
final DateTime eventStart = windowStart.minus(preEventCrawlOffset);
final DateTime eventEnd = windowEnd.plus(postEventCrawlOffset);
- List<EventDTO> holidays = eventDAO.findEventsBetweenTimeRange(EventType.HOLIDAY.toString(),
- eventStart.getMillis(), eventEnd.getMillis());
+ Map<String, List<String>> targetDimensions = new HashMap<>();
+ targetDimensions.put(EVENT_FILTER_COUNTRY, emailContentFormatterConfiguration.getHolidayCountriesWhitelist());
+ List<EventDTO> holidays = getHolidayEvents(eventStart, eventEnd, targetDimensions);
Collections.sort(holidays, new Comparator<EventDTO>() {
@Override
public int compare(EventDTO o1, EventDTO o2) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
index e6f985b..d166a5c 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/ThirdEyeAnomalyConfiguration.java
@@ -22,6 +22,7 @@ import com.linkedin.thirdeye.auto.onboard.AutoOnboardConfiguration;
import com.linkedin.thirdeye.common.ThirdEyeConfiguration;
import java.util.Collection;
import java.util.HashSet;
+import java.util.List;
public class ThirdEyeAnomalyConfiguration extends ThirdEyeConfiguration {
@@ -46,6 +47,7 @@ public class ThirdEyeAnomalyConfiguration extends ThirdEyeConfiguration {
private TaskDriverConfiguration taskDriverConfiguration = new TaskDriverConfiguration();
private String failureFromAddress;
private String failureToAddress;
+ private List<String> holidayCountriesWhitelist;
public HolidayEventsLoaderConfiguration getHolidayEventsLoaderConfiguration() {
return holidayEventsLoaderConfiguration;
@@ -198,4 +200,12 @@ public class ThirdEyeAnomalyConfiguration extends ThirdEyeConfiguration {
public void setFailureToAddress(String failureToAddress) {
this.failureToAddress = failureToAddress;
}
+
+ public List<String> getHolidayCountriesWhitelist() {
+ return holidayCountriesWhitelist;
+ }
+
+ public void setHolidayCountriesWhitelist(List<String> holidayCountriesWhitelist) {
+ this.holidayCountriesWhitelist = holidayCountriesWhitelist;
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/EventFilter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/EventFilter.java
index a29eaef..f61826f 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/EventFilter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/EventFilter.java
@@ -105,7 +105,6 @@ public class EventFilter {
// if filter map not empty, filter events
if (MapUtils.isNotEmpty(eventFilterDimensionMap)) {
-
// go over each event
for (EventDTO event : allEvents) {
boolean eventAdded = false;
@@ -163,8 +162,10 @@ public class EventFilter {
private static List<String> transformDimensionValues(List<String> dimensionValues) {
List<String> dimensionValuesTransformed = new ArrayList<>();
- for (String value : dimensionValues) {
- dimensionValuesTransformed.add(value.toLowerCase());
+ if (dimensionValues != null) {
+ for (String value : dimensionValues) {
+ dimensionValuesTransformed.add(value.toLowerCase());
+ }
}
return dimensionValuesTransformed;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/HolidayEventProvider.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/HolidayEventProvider.java
index b172617..6455823 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/HolidayEventProvider.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/events/HolidayEventProvider.java
@@ -21,19 +21,24 @@ import com.linkedin.thirdeye.datalayer.dto.EventDTO;
import com.linkedin.thirdeye.datasource.DAORegistry;
import java.util.List;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
public class HolidayEventProvider implements EventDataProvider<EventDTO> {
+ private static final Logger LOG = LoggerFactory.getLogger(HolidayEventProvider.class);
+
private EventManager eventDAO = DAORegistry.getInstance().getEventDAO();
@Override
public List<EventDTO> getEvents(EventFilter eventFilter) {
+ LOG.info("Fetching all {} events between {} and {}", eventFilter.getEventType(), eventFilter.getStartTime(), eventFilter.getEndTime());
+ List<EventDTO> allEventsBetweenTimeRange = eventDAO.findEventsBetweenTimeRange(
+ eventFilter.getEventType(),
+ eventFilter.getStartTime(),
+ eventFilter.getEndTime());
- List<EventDTO> allEventsBetweenTimeRange =
- eventDAO.findEventsBetweenTimeRange(EventType.HOLIDAY.name(), eventFilter.getStartTime(),
- eventFilter.getEndTime());
-
- List<EventDTO> holidayEvents = EventFilter.applyDimensionFilter(allEventsBetweenTimeRange, eventFilter.getTargetDimensionMap());
- return holidayEvents;
+ return EventFilter.applyDimensionFilter(allEventsBetweenTimeRange, eventFilter.getTargetDimensionMap());
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/eventprovider/TestHolidayEventProvider.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/eventprovider/TestHolidayEventProvider.java
index 0a0af8b..b5ef670 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/eventprovider/TestHolidayEventProvider.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/eventprovider/TestHolidayEventProvider.java
@@ -109,6 +109,7 @@ public class TestHolidayEventProvider {
// check that it gets all HOLIDAY events in time range, and only HOLIDAY events
eventFilter.setStartTime(hoursAgo5);
eventFilter.setEndTime(hoursAgo3);
+ eventFilter.setEventType(EventType.HOLIDAY.name());
events = holidayEventProvider.getEvents(eventFilter);
Assert.assertEquals(events.size(), 2);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org