You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/08/01 19:15:31 UTC
[incubator-pinot] branch master updated: [TE] Adding Mean Variance
Rule Detector (#4466)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7fde375 [TE] Adding Mean Variance Rule Detector (#4466)
7fde375 is described below
commit 7fde37594bc644c9c868a0850c4af7103a4d178b
Author: Horace Choi <ho...@gmail.com>
AuthorDate: Thu Aug 1 12:15:25 2019 -0700
[TE] Adding Mean Variance Rule Detector (#4466)
Forecast using history mean and standard deviation.
---
.../components/MeanVarianceRuleDetector.java | 290 +++++++++++++++++++++
.../spec/MeanVarianceRuleDetectorSpec.java | 60 +++++
.../components/MeanVarianceRuleDetectorTest.java | 175 +++++++++++++
.../thirdeye/detection/algorithm/timeseries-2y.csv | 107 ++++++++
4 files changed, 632 insertions(+)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java
new file mode 100644
index 0000000..4eed71c
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.components;
+
+import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+import org.apache.pinot.thirdeye.dataframe.BooleanSeries;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
+import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
+import org.apache.pinot.thirdeye.dataframe.LongSeries;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.detection.DetectionUtils;
+import org.apache.pinot.thirdeye.detection.InputDataFetcher;
+import org.apache.pinot.thirdeye.detection.Pattern;
+import org.apache.pinot.thirdeye.detection.annotation.Components;
+import org.apache.pinot.thirdeye.detection.annotation.DetectionTag;
+import org.apache.pinot.thirdeye.detection.annotation.Param;
+import org.apache.pinot.thirdeye.detection.spec.MeanVarianceRuleDetectorSpec;
+import org.apache.pinot.thirdeye.detection.spi.components.AnomalyDetector;
+import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
+import org.apache.pinot.thirdeye.detection.spi.model.DetectionResult;
+import org.apache.pinot.thirdeye.detection.spi.model.InputData;
+import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
+import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.pinot.thirdeye.dataframe.DoubleSeries.*;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+
+@Components(title = "History mean and standard deviation based forecasting and detection.",
+ type = "MEAN_VARIANCE_RULE",
+ tags = {DetectionTag.RULE_DETECTION},
+ description = "Forecast using history mean and standard deviation.",
+ params = {
+ @Param(name = "offset", defaultValue = "wo1w"),
+ @Param(name = "pattern", allowableValues = {"up", "down"}),
+ @Param(name = "lookback"),
+ @Param(name = "sensitivity")})
+public class MeanVarianceRuleDetector implements AnomalyDetector<MeanVarianceRuleDetectorSpec>,
+ BaselineProvider<MeanVarianceRuleDetectorSpec> {
+ private static final Logger LOG = LoggerFactory.getLogger(MeanVarianceRuleDetector.class);
+ private InputDataFetcher dataFetcher;
+ private Pattern pattern;
+ private String monitoringGranularity;
+ private TimeGranularity timeGranularity;
+ private double sensitivity;
+ private int lookback;
+
+ private static final String COL_CURR = "current";
+ private static final String COL_ANOMALY = "anomaly";
+ private static final String COL_PATTERN = "pattern";
+ private static final String COL_DIFF = "diff";
+ private static final String COL_DIFF_VIOLATION = "diff_violation";
+ private static final String COL_ERROR = "error";
+ private static final String COL_CHANGE = "change";
+
+ @Override
+ public void init(MeanVarianceRuleDetectorSpec spec, InputDataFetcher dataFetcher) {
+ this.dataFetcher = dataFetcher;
+ this.pattern = spec.getPattern();
+ this.lookback = spec.getLookback();
+ this.sensitivity = spec.getSensitivity();
+ this.monitoringGranularity = spec.getMonitoringGranularity();
+
+ if (this.monitoringGranularity.equals("1_MONTHS")) {
+ this.timeGranularity = MetricSlice.NATIVE_GRANULARITY;
+ } else {
+ this.timeGranularity = TimeGranularity.fromString(spec.getMonitoringGranularity());
+ }
+
+ //Lookback spec validation
+ //Minimum lookback set to 9. That's 8 change data points.
+ if (this.lookback < 9) {
+ throw new IllegalArgumentException(String.format("Lookback of %d is too small. Please increase to greater than 9.", this.lookback));
+ }
+
+ }
+
+ @Override
+ public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
+ MetricEntity metricEntity = MetricEntity.fromSlice(slice, 0);
+ Interval window = new Interval(slice.getStart(), slice.getEnd());
+ DateTime trainStart;
+
+ if (isMultiDayGranularity()) {
+ trainStart = window.getStart().minusDays(timeGranularity.getSize() * lookback);
+ } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+ trainStart = window.getStart().minusMonths(lookback);
+ } else {
+ trainStart = window.getStart().minusWeeks(lookback);
+ }
+
+ DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
+ .withMetricIdsForDataset(Collections.singleton(metricEntity.getId()))).getDatasetForMetricId()
+ .get(metricEntity.getId());
+ DataFrame inputDf = fetchData(metricEntity, trainStart.getMillis(), window.getEndMillis());
+ DataFrame resultDF = computePredictionInterval(inputDf, window.getStartMillis(), datasetConfig.getTimezone());
+ resultDF = resultDF.joinLeft(inputDf.renameSeries(COL_VALUE, COL_CURR), COL_TIME);
+
+ // Exclude the end because baseline calculation should not contain the end
+ if (resultDF.size() > 1) {
+ resultDF = resultDF.head(resultDF.size() - 1);
+ }
+
+ return TimeSeries.fromDataFrame(resultDF);
+ }
+
+ @Override
+ public DetectionResult runDetection(Interval window, String metricUrn) {
+ MetricEntity me = MetricEntity.fromURN(metricUrn);
+ DateTime fetchStart;
+ //get historical data
+ if (isMultiDayGranularity()) {
+ fetchStart = window.getStart().minusDays(timeGranularity.getSize() * lookback);
+ } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+ fetchStart = window.getStart().minusMonths(lookback);
+ } else {
+ fetchStart = window.getStart().minusWeeks(lookback);
+ }
+
+ MetricSlice slice = MetricSlice.from(me.getId(), fetchStart.getMillis(), window.getEndMillis(), me.getFilters(), timeGranularity);
+ DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
+ .withMetricIdsForDataset(Collections.singleton(me.getId()))).getDatasetForMetricId()
+ .get(me.getId());
+ // getting data (window + earliest lookback) all at once.
+ LOG.info("Getting data for" + slice.toString());
+ DataFrame dfInput = fetchData(me, fetchStart.getMillis(), window.getEndMillis());
+ DataFrame dfCurr = new DataFrame(dfInput).renameSeries(COL_VALUE, COL_CURR);
+ DataFrame dfBase = computePredictionInterval(dfInput, window.getStartMillis(), datasetConfig.getTimezone());
+ DataFrame df = new DataFrame(dfCurr).addSeries(dfBase, COL_VALUE, COL_ERROR);
+ df.addSeries(COL_DIFF, df.getDoubles(COL_CURR).subtract(df.get(COL_VALUE)));
+ df.addSeries(COL_ANOMALY, BooleanSeries.fillValues(df.size(), false));
+
+ // Filter pattern
+ if (pattern.equals(Pattern.UP_OR_DOWN) ) {
+ df.addSeries(COL_PATTERN, BooleanSeries.fillValues(df.size(), true));
+ } else {
+ df.addSeries(COL_PATTERN, pattern.equals(Pattern.UP) ? df.getDoubles(COL_DIFF).gt(0) :
+ df.getDoubles(COL_DIFF).lt(0));
+ }
+ df.addSeries(COL_DIFF_VIOLATION, df.getDoubles(COL_DIFF).abs().gte(df.getDoubles(COL_ERROR)));
+ df.mapInPlace(BooleanSeries.ALL_TRUE, COL_ANOMALY, COL_PATTERN, COL_DIFF_VIOLATION);
+
+ // Anomalies
+ List<MergedAnomalyResultDTO> anomalyResults = DetectionUtils.makeAnomalies(slice, df, COL_ANOMALY,
+ window.getEndMillis(),
+ DetectionUtils.getMonitoringGranularityPeriod(timeGranularity.toAggregationGranularityString(),
+ datasetConfig), datasetConfig);
+ dfBase = dfBase.joinRight(df.retainSeries(COL_TIME, COL_CURR), COL_TIME);
+ return DetectionResult.from(anomalyResults, TimeSeries.fromDataFrame(dfBase));
+ }
+
+ private DataFrame computePredictionInterval(DataFrame inputDF, long windowStartTime, String timezone) {
+
+ DataFrame resultDF = new DataFrame();
+ //filter the data inside window for current values.
+ DataFrame forecastDF = inputDF.filter(new LongConditional() {
+ @Override
+ public boolean apply(long... values) {
+ return values[0] >= windowStartTime;
+ }
+ }, COL_TIME).dropNull();
+
+ int size = forecastDF.size();
+ double[] baselineArray = new double[size];
+ double[] upperBoundArray = new double[size];
+ double[] lowerBoundArray = new double[size];
+ long[] resultTimeArray = new long[size];
+ double[] errorArray = new double[size];
+ double[] std = new double[size];
+ double[] mean = new double[size];
+
+ //get the trainingDF for each week, which is the number of lookback to 1 week before the each week predict start time
+ for (int k = 0; k < size; k++) {
+ DataFrame trainingDF;
+ trainingDF = getLookbackDF(inputDF, forecastDF.getLong(COL_TIME, k));
+ //the get historical WoW mean and std.
+ std[k]= trainingDF.getDoubles(COL_CHANGE).std().value();
+ mean[k] = trainingDF.getDoubles(COL_CHANGE).mean().value();
+
+ //calculate baseline, error , upper and lower bound for prediction window.
+ resultTimeArray[k] = forecastDF.getLong(COL_TIME, k);
+ baselineArray[k] = trainingDF.getDouble(COL_VALUE,trainingDF.size()-1) * (1 + mean[k]);
+ errorArray[k] = trainingDF.getDouble(COL_VALUE,trainingDF.size()-1) * sensitivityToSigma(this.sensitivity) * std[k];
+ upperBoundArray[k] = baselineArray[k] + errorArray[k];
+ lowerBoundArray[k] = baselineArray[k] - errorArray[k];
+ }
+ //Construct the dataframe.
+ resultDF.addSeries(COL_TIME, LongSeries.buildFrom(resultTimeArray)).setIndex(COL_TIME);
+ resultDF.addSeries(COL_VALUE, DoubleSeries.buildFrom(baselineArray));
+ resultDF.addSeries(COL_UPPER_BOUND, DoubleSeries.buildFrom(upperBoundArray));
+ resultDF.addSeries(COL_LOWER_BOUND, DoubleSeries.buildFrom(lowerBoundArray));
+ resultDF.addSeries(COL_ERROR, DoubleSeries.buildFrom(errorArray));
+ return resultDF;
+ }
+
+ /**
+ * Fetch data from metric
+ *
+ * @param metricEntity metric entity
+ * @param start start timestamp
+ * @param end end timestamp
+ * @return Data Frame that has data from start to end
+ */
+ private DataFrame fetchData(MetricEntity metricEntity, long start, long end) {
+ List<MetricSlice> slices = new ArrayList<>();
+ MetricSlice sliceData = MetricSlice.from(metricEntity.getId(), start, end,
+ metricEntity.getFilters(), timeGranularity);
+ slices.add(sliceData);
+ LOG.info("Getting data for" + sliceData.toString());
+ InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(slices));
+ return data.getTimeseries().get(sliceData);
+ }
+
+ /**
+ * Returns a data frame containing lookback number of data before prediction time
+ * @param originalDF the original dataframe
+ * @param time the prediction time, in unix timestamp
+ * @return DataFrame containing lookback number of data
+ */
+ private DataFrame getLookbackDF(DataFrame originalDF, Long time) {
+ LongSeries longSeries = (LongSeries) originalDF.get(COL_TIME);
+ int indexEnd = longSeries.find(time);
+ DataFrame df = DataFrame.builder(COL_TIME, COL_VALUE).build();
+
+ if (indexEnd != -1) {
+ int indexStart = Math.max(0, indexEnd - lookback);
+ df = df.append(originalDF.slice(indexStart, indexEnd));
+ }
+ // calculate percentage change
+ df.addSeries(COL_CURR,df.getDoubles(COL_VALUE).shift(-1));
+ df.addSeries(COL_CHANGE, map((DoubleFunction) values -> {
+ if (Double.compare(values[1], 0.0) == 0) {
+ // divide by zero handling
+ return 0.0;
+ }
+ return (values[0] - values[1]) / values[1];
+ }, df.getDoubles(COL_CURR), df.get(COL_VALUE)));
+ return df;
+ }
+
+ // Check whether monitoring timeGranularity is multiple days
+ private boolean isMultiDayGranularity() {
+ return !timeGranularity.equals(MetricSlice.NATIVE_GRANULARITY) && timeGranularity.getUnit() == TimeUnit.DAYS;
+ }
+
+ /**
+ * Mapping of sensitivity to sigma on range of 0.5 - 1.5
+ * @param sensitivity double from 0 to 10
+ * @return sigma
+ */
+ private static double sensitivityToSigma(double sensitivity) {
+ // If out of bound, use boundary sensitivity
+ if (sensitivity < 0) {
+ sensitivity = 0;
+ } else if (sensitivity > 10) {
+ sensitivity = 10;
+ }
+ double sigma = 0.5 + 0.1 * (10 - sensitivity);
+ return sigma;
+ }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java
new file mode 100644
index 0000000..0ea1cba
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.spec;
+
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.detection.Pattern;
+
+
+public class MeanVarianceRuleDetectorSpec extends AbstractSpec {
+ private String monitoringGranularity = MetricSlice.NATIVE_GRANULARITY.toAggregationGranularityString(); // use native granularity by default
+ private int lookback = 52; //default look back of 52 units
+ private double sensitivity = 5; //default sensitivity of 5, equals +/- 1 sigma
+ private Pattern pattern = Pattern.UP_OR_DOWN;
+
+ public String getMonitoringGranularity() {
+ return monitoringGranularity;
+ }
+
+ public void setMonitoringGranularity(String monitoringGranularity) {
+ this.monitoringGranularity = monitoringGranularity;
+ }
+
+ public Pattern getPattern() {
+ return pattern;
+ }
+
+ public void setPattern(Pattern pattern) {
+ this.pattern = pattern;
+ }
+
+ public int getLookback() { return lookback;}
+
+ public void setLookback(int lookback) {this.lookback = lookback;}
+
+ public double getSensitivity() {
+ return sensitivity;
+ }
+
+ public void setSensitivity(double sensitivity) {
+ this.sensitivity = sensitivity;
+ }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java
new file mode 100644
index 0000000..d844629
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.components;
+
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.lang.Math;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
+import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
+import org.apache.pinot.thirdeye.detection.MockDataProvider;
+import org.apache.pinot.thirdeye.detection.Pattern;
+import org.apache.pinot.thirdeye.detection.algorithm.AlgorithmUtils;
+import org.apache.pinot.thirdeye.detection.spec.MeanVarianceRuleDetectorSpec;
+import org.apache.pinot.thirdeye.detection.spi.model.DetectionResult;
+import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
+import org.joda.time.Interval;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
+
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+
+
+public class MeanVarianceRuleDetectorTest {
+
+ private DataProvider provider;
+ private DataFrame data;
+ private Multimap<String, String> filters = TreeMultimap.create();
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ try (Reader dataReader = new InputStreamReader(AlgorithmUtils.class.getResourceAsStream("timeseries-2y.csv"))) {
+ this.data = DataFrame.fromCsv(dataReader);
+ this.data.setIndex(COL_TIME);
+ }
+
+ MetricConfigDTO dailyMetricConfig = new MetricConfigDTO();
+ dailyMetricConfig.setId(1L);
+ dailyMetricConfig.setName("thirdeye-test-daily");
+ dailyMetricConfig.setDataset("thirdeye-test-dataset-daily");
+
+ DatasetConfigDTO dailyDatasetConfig = new DatasetConfigDTO();
+ dailyDatasetConfig.setTimeUnit(TimeUnit.DAYS);
+ dailyDatasetConfig.setDataset("thirdeye-test-dataset-daily");
+ dailyDatasetConfig.setTimeDuration(1);
+
+ Map<MetricSlice, DataFrame> timeseries = new HashMap<>();
+ timeseries.put(MetricSlice.from(1L, 1499126400000L, 1562630400000L, filters, TimeGranularity.fromString("7_DAYS")), this.data);
+
+ this.provider = new MockDataProvider()
+ .setTimeseries(timeseries)
+ .setMetrics(Collections.singletonList(dailyMetricConfig))
+ .setDatasets(Collections.singletonList(dailyDatasetConfig));
+ }
+
+ @Test
+ public void testComputePredictedTimeSeriesDaily() {
+ MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+ MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+ spec.setMonitoringGranularity("7_DAYS");
+ spec.setLookback(52);
+ spec.setSensitivity(5);
+ detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+ Interval window = new Interval(1530576000000L, 1562630400000L);
+ String metricUrn = "thirdeye:metric:1";
+ MetricEntity me = MetricEntity.fromURN(metricUrn);
+ MetricSlice slice = MetricSlice.from(me.getId(), window.getStartMillis(), window.getEndMillis(), me.getFilters(), TimeGranularity
+ .fromString(spec.getMonitoringGranularity()));
+ TimeSeries timeSeries = detector.computePredictedTimeSeries(slice);
+ Assert.assertEquals(timeSeries.getPredictedBaseline().size(), 52);
+ // prediction assertion. Expected value calculated offline using Python.
+ Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(0) * 1000.0) / 1000.0,174431.234);
+ Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(1) * 1000.0) / 1000.0,147184.878);
+ Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(2) * 1000.0) / 1000.0,153660.665);
+ }
+
+ @Test
+ public void testWeekOverWeekLookbackChange() {
+ MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+ MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+ spec.setMonitoringGranularity("7_DAYS");
+ spec.setLookback(52);
+ spec.setSensitivity(5);
+ detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+ DetectionResult detectionResult = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+ List<MergedAnomalyResultDTO> anomalies = detectionResult.getAnomalies();
+ Assert.assertEquals(anomalies.size(), 6);
+ Assert.assertEquals(anomalies.get(0).getStartTime(), 1530576000000L);
+ Assert.assertEquals(anomalies.get(0).getEndTime(), 1531180800000L);
+ Assert.assertEquals(anomalies.get(1).getStartTime(), 1540252800000L);
+ Assert.assertEquals(anomalies.get(1).getEndTime(), 1541462400000L);
+ Assert.assertEquals(anomalies.get(2).getStartTime(), 1545696000000L);
+ Assert.assertEquals(anomalies.get(2).getEndTime(), 1546300800000L);
+ Assert.assertEquals(anomalies.get(3).getStartTime(), 1546905600000L);
+ Assert.assertEquals(anomalies.get(3).getEndTime(), 1547510400000L);
+ Assert.assertEquals(anomalies.get(4).getStartTime(), 1553558400000L);
+ Assert.assertEquals(anomalies.get(4).getEndTime(), 1554768000000L);
+ Assert.assertEquals(anomalies.get(5).getStartTime(), 1555372800000L);
+ Assert.assertEquals(anomalies.get(5).getEndTime(), 1557187200000L);
+ }
+
+ @Test
+ public void testWeekOverWeekLookbackChangeDown() {
+ MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+ MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+ spec.setMonitoringGranularity("7_DAYS");
+ spec.setLookback(52);
+ spec.setSensitivity(5);
+ spec.setPattern(Pattern.DOWN);
+ detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+ DetectionResult result = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+ List<MergedAnomalyResultDTO> anomalies = result.getAnomalies();
+ Assert.assertEquals(anomalies.size(), 4);
+ Assert.assertEquals(anomalies.get(0).getStartTime(), 1530576000000L);
+ Assert.assertEquals(anomalies.get(0).getEndTime(), 1531180800000L);
+ Assert.assertEquals(anomalies.get(1).getStartTime(), 1540857600000L);
+ Assert.assertEquals(anomalies.get(1).getEndTime(), 1541462400000L);
+ Assert.assertEquals(anomalies.get(2).getStartTime(), 1545696000000L);
+ Assert.assertEquals(anomalies.get(2).getEndTime(), 1546300800000L);
+ Assert.assertEquals(anomalies.get(3).getStartTime(), 1554163200000L);
+ Assert.assertEquals(anomalies.get(3).getEndTime(), 1554768000000L);
+ }
+
+ @Test
+ public void testWeekOverWeekLookbackChangeUp() {
+ MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+ MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+ spec.setMonitoringGranularity("7_DAYS");
+ spec.setLookback(52);
+ spec.setSensitivity(5);
+ spec.setPattern(Pattern.UP);
+ detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+ DetectionResult result = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+ List<MergedAnomalyResultDTO> anomalies = result.getAnomalies();
+ Assert.assertEquals(anomalies.size(), 4);
+ Assert.assertEquals(anomalies.get(0).getStartTime(), 1540252800000L);
+ Assert.assertEquals(anomalies.get(0).getEndTime(), 1540857600000L);
+ Assert.assertEquals(anomalies.get(1).getStartTime(),1546905600000L);
+ Assert.assertEquals(anomalies.get(1).getEndTime(),1547510400000L);
+ Assert.assertEquals(anomalies.get(2).getStartTime(),1553558400000L);
+ Assert.assertEquals(anomalies.get(2).getEndTime(),1554163200000L);
+ Assert.assertEquals(anomalies.get(3).getStartTime(),1555372800000L);
+ Assert.assertEquals(anomalies.get(3).getEndTime(),1557187200000L);
+ }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv
new file mode 100644
index 0000000..1af53a8
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv
@@ -0,0 +1,107 @@
+timestamp,value
+1499126400000,100237
+1499731200000,103952
+1500336000000,111500
+1500940800000,117761
+1501545600000,123124
+1502150400000,132955
+1502755200000,124202
+1503360000000,117803
+1503964800000,112117
+1504569600000,106698
+1505174400000,110367
+1505779200000,109243
+1506384000000,108994
+1506988800000,116122
+1507593600000,108620
+1508198400000,108490
+1508803200000,104060
+1509408000000,108019
+1510012800000,106021
+1510617600000,109581
+1511222400000,104792
+1511827200000,97852
+1512432000000,99984
+1513036800000,97621
+1513641600000,92522
+1514246400000,77013
+1514851200000,82625
+1515456000000,104407
+1516060800000,104543
+1516665600000,109124
+1517270400000,112433
+1517875200000,112776
+1518480000000,115295
+1519084800000,116233
+1519689600000,118773
+1520294400000,109849
+1520899200000,118393
+1521504000000,130142
+1522108800000,132692
+1522713600000,122168
+1523318400000,127751
+1523923200000,130663
+1524528000000,141285
+1525132800000,143513
+1525737600000,158803
+1526342400000,174864
+1526947200000,198051
+1527552000000,160028
+1528156800000,170869
+1528761600000,179357
+1529366400000,169122
+1529971200000,172133
+1530576000000,145782
+1531180800000,152279
+1531785600000,148278
+1532390400000,146696
+1532995200000,144465
+1533600000000,151209
+1534204800000,144277
+1534809600000,139179
+1535414400000,147519
+1536019200000,138942
+1536624000000,141822
+1537228800000,136290
+1537833600000,144448
+1538438400000,148539
+1539043200000,147423
+1539648000000,155035
+1540252800000,175620
+1540857600000,148494
+1541462400000,141305
+1542067200000,144111
+1542672000000,144678
+1543276800000,137476
+1543881600000,145511
+1544486400000,141762
+1545091200000,142075
+1545696000000,114929
+1546300800000,107373
+1546905600000,136891
+1547510400000,142291
+1548115200000,146492
+1548720000000,153866
+1549324800000,150707
+1549929600000,157428
+1550534400000,153784
+1551139200000,158622
+1551744000000,162204
+1552348800000,162323
+1552953600000,171853
+1553558400000,256515
+1554163200000,194456
+1554768000000,191069
+1555372800000,255471
+1555977600000,300109
+1556582400000,347975
+1557187200000,346512
+1557792000000,367724
+1558396800000,375144
+1559001600000,366047
+1559606400000,358787
+1560211200000,369429
+1560816000000,379284
+1561420800000,369440
+1562025600000,352345
+1562630400000,338757
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org