You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/04/04 22:41:31 UTC
[incubator-pinot] branch master updated: [TE] detection - handle
day light savings time in moving windows (#4072)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b08eccc [TE] detection - handle day light savings time in moving windows (#4072)
b08eccc is described below
commit b08eccc79ca2139954a60153cfdcf25ddf62b1cb
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Thu Apr 4 15:41:26 2019 -0700
[TE] detection - handle day light savings time in moving windows (#4072)
---
.../pinot/thirdeye/detection/DetectionUtils.java | 18 ++++++++
.../thirdeye/detection/algorithm/MergeWrapper.java | 1 -
.../detection/wrapper/AnomalyDetectorWrapper.java | 29 +++++++------
.../yaml/CompositePipelineConfigTranslator.java | 48 +++++++++++-----------
.../wrapper/AnomalyDetectorWrapperTest.java | 25 +++++++++++
5 files changed, 81 insertions(+), 40 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
index f2ca286..8da6ed3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionUtils.java
@@ -26,6 +26,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.dataframe.BooleanSeries;
@@ -42,6 +43,7 @@ import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
+import org.joda.time.Duration;
import org.joda.time.Period;
import org.joda.time.PeriodType;
@@ -219,4 +221,20 @@ public class DetectionUtils {
return TimeGranularity.fromString(monitoringGranularity).toPeriod();
}
+ public static Period periodFromTimeUnit(int size, TimeUnit unit) {
+ switch (unit) {
+ case DAYS:
+ return Period.days(size);
+ case HOURS:
+ return Period.hours(size);
+ case MINUTES:
+ return Period.minutes(size);
+ case SECONDS:
+ return Period.seconds(size);
+ case MILLISECONDS:
+ return Period.millis(size);
+ default:
+ return new Period(TimeUnit.MILLISECONDS.convert(size, unit));
+ }
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
index 83be36c..5acf958 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/algorithm/MergeWrapper.java
@@ -38,7 +38,6 @@ import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DetectionPipeline;
-import org.apache.pinot.thirdeye.detection.DetectionPipelineJob;
import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
index b549427..99ba6dd 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
@@ -43,7 +43,6 @@ import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.spi.components.AnomalyDetector;
import org.apache.pinot.thirdeye.detection.spi.exception.DetectorDataInsufficientException;
-import org.apache.pinot.thirdeye.detection.spi.exception.DetectorException;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
import org.joda.time.DateTime;
@@ -76,7 +75,7 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
private static final String PROP_TIMEZONE = "timezone";
private static final String PROP_BUCKET_PERIOD = "bucketPeriod";
private static final String PROP_CACHE_PERIOD_LOOKBACK = "cachingPeriodLookback";
- private static final long DEFAULT_CACHING_PERIOD_LOOKBACK = TimeUnit.DAYS.toMillis(-1);
+ private static final long DEFAULT_CACHING_PERIOD_LOOKBACK = -1;
private static final long CACHING_PERIOD_LOOKBACK_DAILY = TimeUnit.DAYS.toMillis(90);
private static final long CACHING_PERIOD_LOOKBACK_HOURLY = TimeUnit.DAYS.toMillis(60);
// disable minute level cache warm up
@@ -99,7 +98,6 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
// need to specify run frequency for minute level detection. Used for moving monitoring window alignment, default to be 15 minutes.
private final TimeGranularity functionFrequency;
private final String detectorName;
- private long windowSizeMillis;
private final DatasetConfigDTO dataset;
private final DateTimeZone dateTimeZone;
private Period bucketPeriod;
@@ -125,8 +123,8 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
this.windowDelayUnit = TimeUnit.valueOf(MapUtils.getString(config.getProperties(), PROP_WINDOW_DELAY_UNIT, "DAYS"));
// detection window size
this.windowSize = MapUtils.getIntValue(config.getProperties(), PROP_WINDOW_SIZE, 1);
+ // detection window unit
this.windowUnit = TimeUnit.valueOf(MapUtils.getString(config.getProperties(), PROP_WINDOW_UNIT, "DAYS"));
- this.windowSizeMillis = TimeUnit.MILLISECONDS.convert(windowSize, windowUnit);
// run frequency, used to determine moving windows for minute-level detection
Map<String, Object> frequency = MapUtils.getMap(config.getProperties(), PROP_FREQUENCY, Collections.emptyMap());
this.functionFrequency = new TimeGranularity(MapUtils.getIntValue(frequency, "size", 15), TimeUnit.valueOf(MapUtils.getString(frequency, "unit", "MINUTES")));
@@ -151,7 +149,7 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
// 1. get the last time stamp for the time series.
// 2. to calculate current values and baseline values for the anomalies detected
// 3. anomaly detection current and baseline time series value
- if( this.cachingPeriodLookback > 0) {
+ if (this.cachingPeriodLookback >= 0) {
MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - cachingPeriodLookback, endTime,
this.metricEntity.getFilters());
this.provider.fetchTimeseries(Collections.singleton(cacheSlice));
@@ -233,12 +231,14 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
List<Interval> getMonitoringWindows() {
if (this.isMovingWindowDetection) {
try{
+ Period windowDelayPeriod = DetectionUtils.periodFromTimeUnit(windowDelay, windowDelayUnit);
+ Period windowSizePeriod = DetectionUtils.periodFromTimeUnit(windowSize, windowUnit);
List<Interval> monitoringWindows = new ArrayList<>();
- List<Long> monitoringWindowEndTimes = getMonitoringWindowEndTimes();
- for (long monitoringEndTime : monitoringWindowEndTimes) {
- long endTime = monitoringEndTime - TimeUnit.MILLISECONDS.convert(windowDelay, windowDelayUnit);
- long startTime = endTime - this.windowSizeMillis;
- monitoringWindows.add(new Interval(startTime, endTime, dateTimeZone));
+ List<DateTime> monitoringWindowEndTimes = getMonitoringWindowEndTimes();
+ for (DateTime monitoringEndTime : monitoringWindowEndTimes) {
+ DateTime endTime = monitoringEndTime.minus(windowDelayPeriod);
+ DateTime startTime = endTime.minus(windowSizePeriod);
+ monitoringWindows.add(new Interval(startTime, endTime));
}
for (Interval window : monitoringWindows){
LOG.info("Will run detection in window {}", window);
@@ -270,8 +270,8 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
}
// get the list of monitoring window end times
- private List<Long> getMonitoringWindowEndTimes() {
- List<Long> endTimes = new ArrayList<>();
+ private List<DateTime> getMonitoringWindowEndTimes() {
+ List<DateTime> endTimes = new ArrayList<>();
// get current hour/day, depending on granularity of dataset,
DateTime currentEndTime = new DateTime(getBoundaryAlignedTimeForDataset(new DateTime(endTime, dateTimeZone)), dateTimeZone);
@@ -279,7 +279,7 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
DateTime lastDateTime = new DateTime(getBoundaryAlignedTimeForDataset(new DateTime(startTime, dateTimeZone)), dateTimeZone);
while (lastDateTime.isBefore(currentEndTime)) {
lastDateTime = lastDateTime.plus(this.bucketPeriod);
- endTimes.add(lastDateTime.getMillis());
+ endTimes.add(lastDateTime);
}
return endTimes;
}
@@ -368,7 +368,6 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
bucketPeriod = Period.days(1);
windowSize = 1;
windowUnit = TimeUnit.DAYS;
- windowSizeMillis = TimeUnit.MILLISECONDS.convert(windowSize, windowUnit);
}
}
-}
\ No newline at end of file
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 9124da3..fd637d2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -309,30 +309,30 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
properties.put(PROP_WINDOW_SIZE, 6);
properties.put(PROP_WINDOW_UNIT, TimeUnit.HOURS);
}
- // override from yaml
- if (yamlConfig.containsKey(PROP_WINDOW_SIZE)) {
- properties.put(PROP_MOVING_WINDOW_DETECTION, true);
- properties.put(PROP_WINDOW_SIZE, MapUtils.getString(yamlConfig, PROP_WINDOW_SIZE));
- }
- if (yamlConfig.containsKey(PROP_WINDOW_UNIT)) {
- properties.put(PROP_MOVING_WINDOW_DETECTION, true);
- properties.put(PROP_WINDOW_UNIT, MapUtils.getString(yamlConfig, PROP_WINDOW_UNIT));
- }
- if (yamlConfig.containsKey(PROP_WINDOW_DELAY)) {
- properties.put(PROP_WINDOW_DELAY, MapUtils.getString(yamlConfig, PROP_WINDOW_DELAY));
- }
- if (yamlConfig.containsKey(PROP_WINDOW_DELAY_UNIT)) {
- properties.put(PROP_WINDOW_DELAY_UNIT, MapUtils.getString(yamlConfig, PROP_WINDOW_DELAY_UNIT));
- }
- if (yamlConfig.containsKey(PROP_TIMEZONE)){
- properties.put(PROP_TIMEZONE, MapUtils.getString(yamlConfig, PROP_TIMEZONE));
- }
- if (yamlConfig.containsKey(PROP_BUCKET_PERIOD)){
- properties.put(PROP_BUCKET_PERIOD, MapUtils.getString(yamlConfig, PROP_BUCKET_PERIOD));
- }
- if (yamlConfig.containsKey(PROP_CACHE_PERIOD_LOOKBACK)) {
- properties.put(PROP_CACHE_PERIOD_LOOKBACK, MapUtils.getString(yamlConfig, PROP_CACHE_PERIOD_LOOKBACK));
- }
+ }
+ // override from yaml
+ if (yamlConfig.containsKey(PROP_WINDOW_SIZE)) {
+ properties.put(PROP_MOVING_WINDOW_DETECTION, true);
+ properties.put(PROP_WINDOW_SIZE, MapUtils.getString(yamlConfig, PROP_WINDOW_SIZE));
+ }
+ if (yamlConfig.containsKey(PROP_WINDOW_UNIT)) {
+ properties.put(PROP_MOVING_WINDOW_DETECTION, true);
+ properties.put(PROP_WINDOW_UNIT, MapUtils.getString(yamlConfig, PROP_WINDOW_UNIT));
+ }
+ if (yamlConfig.containsKey(PROP_WINDOW_DELAY)) {
+ properties.put(PROP_WINDOW_DELAY, MapUtils.getString(yamlConfig, PROP_WINDOW_DELAY));
+ }
+ if (yamlConfig.containsKey(PROP_WINDOW_DELAY_UNIT)) {
+ properties.put(PROP_WINDOW_DELAY_UNIT, MapUtils.getString(yamlConfig, PROP_WINDOW_DELAY_UNIT));
+ }
+ if (yamlConfig.containsKey(PROP_TIMEZONE)){
+ properties.put(PROP_TIMEZONE, MapUtils.getString(yamlConfig, PROP_TIMEZONE));
+ }
+ if (yamlConfig.containsKey(PROP_BUCKET_PERIOD)){
+ properties.put(PROP_BUCKET_PERIOD, MapUtils.getString(yamlConfig, PROP_BUCKET_PERIOD));
+ }
+ if (yamlConfig.containsKey(PROP_CACHE_PERIOD_LOOKBACK)) {
+ properties.put(PROP_CACHE_PERIOD_LOOKBACK, MapUtils.getString(yamlConfig, PROP_CACHE_PERIOD_LOOKBACK));
}
}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
index 69389f1..8cbd5e1 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapperTest.java
@@ -125,6 +125,31 @@ public class AnomalyDetectorWrapperTest {
}
@Test
+ public void testMovingMonitoringWindowDayLightSaving() {
+ DateTimeZone timeZone = DateTimeZone.forID("America/Los_Angeles");
+ this.properties.put(PROP_MOVING_WINDOW_DETECTION, true);
+ this.properties.put(PROP_TIMEZONE, timeZone.toString());
+ AnomalyDetectorWrapper detectionPipeline =
+ new AnomalyDetectorWrapper(this.provider, this.config, 1552118400000L, 1552287600000L);
+ List<Interval> monitoringWindows = detectionPipeline.getMonitoringWindows();
+ Assert.assertEquals(monitoringWindows,
+ Arrays.asList(new Interval(1552118400000L, 1552204800000L, timeZone), new Interval(1552204800000L, 1552287600000L, timeZone)));
+ }
+
+ @Test
+ public void testMovingMonitoringWindowDayLightSaving2() {
+ DateTimeZone timeZone = DateTimeZone.forID("America/Los_Angeles");
+ this.properties.put(PROP_MOVING_WINDOW_DETECTION, true);
+ this.properties.put(PROP_TIMEZONE, timeZone.toString());
+ AnomalyDetectorWrapper detectionPipeline =
+ new AnomalyDetectorWrapper(this.provider, this.config, 1541228400000L, 1541404800000L);
+ List<Interval> monitoringWindows = detectionPipeline.getMonitoringWindows();
+ Assert.assertEquals(monitoringWindows,
+ Arrays.asList(new Interval(1541228400000L, 1541314800000L, timeZone), new Interval(1541314800000L, 1541404800000L, timeZone)));
+ }
+
+
+ @Test
public void testGetLastTimestampWithEstimate() {
AnomalyDetectorWrapper detectionPipeline =
new AnomalyDetectorWrapper(this.provider, this.config, 1546300800000L, 1546560000000L);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org