You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/08/01 19:15:31 UTC

[incubator-pinot] branch master updated: [TE] Adding Mean Variance Rule Detector (#4466)

This is an automated email from the ASF dual-hosted git repository.

jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 7fde375  [TE] Adding Mean Variance Rule Detector (#4466)
7fde375 is described below

commit 7fde37594bc644c9c868a0850c4af7103a4d178b
Author: Horace Choi <ho...@gmail.com>
AuthorDate: Thu Aug 1 12:15:25 2019 -0700

    [TE] Adding Mean Variance Rule Detector (#4466)
    
    Forecast using history mean and standard deviation.
---
 .../components/MeanVarianceRuleDetector.java       | 290 +++++++++++++++++++++
 .../spec/MeanVarianceRuleDetectorSpec.java         |  60 +++++
 .../components/MeanVarianceRuleDetectorTest.java   | 175 +++++++++++++
 .../thirdeye/detection/algorithm/timeseries-2y.csv | 107 ++++++++
 4 files changed, 632 insertions(+)

diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java
new file mode 100644
index 0000000..4eed71c
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetector.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.components;
+
+import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+import org.apache.pinot.thirdeye.dataframe.BooleanSeries;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
+import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
+import org.apache.pinot.thirdeye.dataframe.LongSeries;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.detection.DetectionUtils;
+import org.apache.pinot.thirdeye.detection.InputDataFetcher;
+import org.apache.pinot.thirdeye.detection.Pattern;
+import org.apache.pinot.thirdeye.detection.annotation.Components;
+import org.apache.pinot.thirdeye.detection.annotation.DetectionTag;
+import org.apache.pinot.thirdeye.detection.annotation.Param;
+import org.apache.pinot.thirdeye.detection.spec.MeanVarianceRuleDetectorSpec;
+import org.apache.pinot.thirdeye.detection.spi.components.AnomalyDetector;
+import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
+import org.apache.pinot.thirdeye.detection.spi.model.DetectionResult;
+import org.apache.pinot.thirdeye.detection.spi.model.InputData;
+import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
+import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import static org.apache.pinot.thirdeye.dataframe.DoubleSeries.*;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+
+@Components(title = "History mean and standard deviation based forecasting and detection.",
+    type = "MEAN_VARIANCE_RULE",
+    tags = {DetectionTag.RULE_DETECTION},
+    description = "Forecast using history mean and standard deviation.",
+    params = {
+        @Param(name = "offset", defaultValue = "wo1w"),
+        @Param(name = "pattern", allowableValues = {"up", "down"}),
+        @Param(name = "lookback"),
+        @Param(name = "sensitivity")})
+public class MeanVarianceRuleDetector implements AnomalyDetector<MeanVarianceRuleDetectorSpec>,
+                                                 BaselineProvider<MeanVarianceRuleDetectorSpec> {
+  private static final Logger LOG = LoggerFactory.getLogger(MeanVarianceRuleDetector.class);
+  private InputDataFetcher dataFetcher;
+  private Pattern pattern;
+  private String monitoringGranularity;
+  private TimeGranularity timeGranularity;
+  private double sensitivity;
+  private int lookback;
+
+  private static final String COL_CURR = "current";
+  private static final String COL_ANOMALY = "anomaly";
+  private static final String COL_PATTERN = "pattern";
+  private static final String COL_DIFF = "diff";
+  private static final String COL_DIFF_VIOLATION = "diff_violation";
+  private static final String COL_ERROR = "error";
+  private static final String COL_CHANGE = "change";
+
+  @Override
+  public void init(MeanVarianceRuleDetectorSpec spec, InputDataFetcher dataFetcher) {
+    this.dataFetcher = dataFetcher;
+    this.pattern = spec.getPattern();
+    this.lookback = spec.getLookback();
+    this.sensitivity = spec.getSensitivity();
+    this.monitoringGranularity = spec.getMonitoringGranularity();
+
+    if (this.monitoringGranularity.equals("1_MONTHS")) {
+      this.timeGranularity = MetricSlice.NATIVE_GRANULARITY;
+    } else {
+      this.timeGranularity = TimeGranularity.fromString(spec.getMonitoringGranularity());
+    }
+
+    //Lookback spec validation
+    //Minimum lookback set to 9. That's 8 change data points.
+    if (this.lookback < 9) {
+      throw new IllegalArgumentException(String.format("Lookback of %d is too small. Please increase to greater than 9.", this.lookback));
+    }
+
+  }
+
+  @Override
+  public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
+    MetricEntity metricEntity = MetricEntity.fromSlice(slice, 0);
+    Interval window = new Interval(slice.getStart(), slice.getEnd());
+    DateTime trainStart;
+
+    if (isMultiDayGranularity()) {
+      trainStart = window.getStart().minusDays(timeGranularity.getSize() * lookback);
+    } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+      trainStart = window.getStart().minusMonths(lookback);
+    } else {
+      trainStart = window.getStart().minusWeeks(lookback);
+    }
+
+    DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
+        .withMetricIdsForDataset(Collections.singleton(metricEntity.getId()))).getDatasetForMetricId()
+        .get(metricEntity.getId());
+    DataFrame inputDf = fetchData(metricEntity, trainStart.getMillis(), window.getEndMillis());
+    DataFrame resultDF = computePredictionInterval(inputDf, window.getStartMillis(), datasetConfig.getTimezone());
+    resultDF = resultDF.joinLeft(inputDf.renameSeries(COL_VALUE, COL_CURR), COL_TIME);
+
+    // Exclude the end because baseline calculation should not contain the end
+    if (resultDF.size() > 1) {
+      resultDF = resultDF.head(resultDF.size() - 1);
+    }
+
+    return TimeSeries.fromDataFrame(resultDF);
+  }
+
+  @Override
+  public DetectionResult runDetection(Interval window, String metricUrn) {
+    MetricEntity me = MetricEntity.fromURN(metricUrn);
+    DateTime fetchStart;
+    //get historical data
+    if (isMultiDayGranularity()) {
+      fetchStart = window.getStart().minusDays(timeGranularity.getSize() * lookback);
+    } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+      fetchStart = window.getStart().minusMonths(lookback);
+    } else {
+      fetchStart = window.getStart().minusWeeks(lookback);
+    }
+
+    MetricSlice slice = MetricSlice.from(me.getId(), fetchStart.getMillis(), window.getEndMillis(), me.getFilters(), timeGranularity);
+    DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
+        .withMetricIdsForDataset(Collections.singleton(me.getId()))).getDatasetForMetricId()
+        .get(me.getId());
+    // getting data (window + earliest lookback) all at once.
+    LOG.info("Getting data for" + slice.toString());
+    DataFrame dfInput = fetchData(me, fetchStart.getMillis(), window.getEndMillis());
+    DataFrame dfCurr = new DataFrame(dfInput).renameSeries(COL_VALUE, COL_CURR);
+    DataFrame dfBase = computePredictionInterval(dfInput, window.getStartMillis(), datasetConfig.getTimezone());
+    DataFrame df = new DataFrame(dfCurr).addSeries(dfBase, COL_VALUE, COL_ERROR);
+    df.addSeries(COL_DIFF, df.getDoubles(COL_CURR).subtract(df.get(COL_VALUE)));
+    df.addSeries(COL_ANOMALY, BooleanSeries.fillValues(df.size(), false));
+
+    // Filter pattern
+    if (pattern.equals(Pattern.UP_OR_DOWN) ) {
+      df.addSeries(COL_PATTERN, BooleanSeries.fillValues(df.size(), true));
+    } else {
+      df.addSeries(COL_PATTERN, pattern.equals(Pattern.UP) ? df.getDoubles(COL_DIFF).gt(0) :
+          df.getDoubles(COL_DIFF).lt(0));
+    }
+    df.addSeries(COL_DIFF_VIOLATION, df.getDoubles(COL_DIFF).abs().gte(df.getDoubles(COL_ERROR)));
+    df.mapInPlace(BooleanSeries.ALL_TRUE, COL_ANOMALY, COL_PATTERN, COL_DIFF_VIOLATION);
+
+    // Anomalies
+    List<MergedAnomalyResultDTO> anomalyResults = DetectionUtils.makeAnomalies(slice, df, COL_ANOMALY,
+        window.getEndMillis(),
+        DetectionUtils.getMonitoringGranularityPeriod(timeGranularity.toAggregationGranularityString(),
+            datasetConfig), datasetConfig);
+    dfBase = dfBase.joinRight(df.retainSeries(COL_TIME, COL_CURR), COL_TIME);
+    return DetectionResult.from(anomalyResults, TimeSeries.fromDataFrame(dfBase));
+  }
+
+  private DataFrame computePredictionInterval(DataFrame inputDF, long windowStartTime, String timezone) {
+
+    DataFrame resultDF = new DataFrame();
+    //filter the data inside window for current values.
+    DataFrame forecastDF = inputDF.filter(new LongConditional() {
+      @Override
+      public boolean apply(long... values) {
+        return values[0] >= windowStartTime;
+      }
+    }, COL_TIME).dropNull();
+
+    int size = forecastDF.size();
+    double[] baselineArray = new double[size];
+    double[] upperBoundArray = new double[size];
+    double[] lowerBoundArray = new double[size];
+    long[] resultTimeArray = new long[size];
+    double[] errorArray = new double[size];
+    double[] std = new double[size];
+    double[] mean = new double[size];
+
+    //get the trainingDF for each week, which is the number of lookback to 1 week before the each week predict start time
+    for (int k = 0; k < size; k++) {
+      DataFrame trainingDF;
+      trainingDF = getLookbackDF(inputDF, forecastDF.getLong(COL_TIME, k));
+      //the get historical WoW mean and std.
+      std[k]= trainingDF.getDoubles(COL_CHANGE).std().value();
+      mean[k] = trainingDF.getDoubles(COL_CHANGE).mean().value();
+
+      //calculate baseline, error , upper and lower bound for prediction window.
+      resultTimeArray[k] = forecastDF.getLong(COL_TIME, k);
+      baselineArray[k] = trainingDF.getDouble(COL_VALUE,trainingDF.size()-1) * (1 + mean[k]);
+      errorArray[k] = trainingDF.getDouble(COL_VALUE,trainingDF.size()-1) * sensitivityToSigma(this.sensitivity) * std[k];
+      upperBoundArray[k] = baselineArray[k] + errorArray[k];
+      lowerBoundArray[k] = baselineArray[k] - errorArray[k];
+    }
+    //Construct the dataframe.
+    resultDF.addSeries(COL_TIME, LongSeries.buildFrom(resultTimeArray)).setIndex(COL_TIME);
+    resultDF.addSeries(COL_VALUE, DoubleSeries.buildFrom(baselineArray));
+    resultDF.addSeries(COL_UPPER_BOUND, DoubleSeries.buildFrom(upperBoundArray));
+    resultDF.addSeries(COL_LOWER_BOUND, DoubleSeries.buildFrom(lowerBoundArray));
+    resultDF.addSeries(COL_ERROR, DoubleSeries.buildFrom(errorArray));
+    return resultDF;
+  }
+
+  /**
+   * Fetch data from metric
+   *
+   * @param metricEntity metric entity
+   * @param start start timestamp
+   * @param end end timestamp
+   * @return Data Frame that has data from start to end
+   */
+  private DataFrame fetchData(MetricEntity metricEntity, long start, long end) {
+    List<MetricSlice> slices = new ArrayList<>();
+    MetricSlice sliceData = MetricSlice.from(metricEntity.getId(), start, end,
+        metricEntity.getFilters(), timeGranularity);
+    slices.add(sliceData);
+    LOG.info("Getting data for" + sliceData.toString());
+    InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(slices));
+    return data.getTimeseries().get(sliceData);
+  }
+
+  /**
+   * Returns a data frame containing lookback number of data before prediction time
+   * @param originalDF the original dataframe
+   * @param time the prediction time, in unix timestamp
+   * @return DataFrame containing lookback number of data
+   */
+  private DataFrame getLookbackDF(DataFrame originalDF, Long time) {
+    LongSeries longSeries = (LongSeries) originalDF.get(COL_TIME);
+    int indexEnd = longSeries.find(time);
+    DataFrame df = DataFrame.builder(COL_TIME, COL_VALUE).build();
+
+    if (indexEnd != -1) {
+      int indexStart = Math.max(0, indexEnd - lookback);
+      df = df.append(originalDF.slice(indexStart, indexEnd));
+    }
+    // calculate percentage change
+    df.addSeries(COL_CURR,df.getDoubles(COL_VALUE).shift(-1));
+    df.addSeries(COL_CHANGE, map((DoubleFunction) values -> {
+      if (Double.compare(values[1], 0.0) == 0) {
+        // divide by zero handling
+        return 0.0;
+      }
+      return (values[0] - values[1]) / values[1];
+    }, df.getDoubles(COL_CURR), df.get(COL_VALUE)));
+    return df;
+  }
+
+  // Check whether monitoring timeGranularity is multiple days
+  private boolean isMultiDayGranularity() {
+    return !timeGranularity.equals(MetricSlice.NATIVE_GRANULARITY) && timeGranularity.getUnit() == TimeUnit.DAYS;
+  }
+
+  /**
+   * Mapping of sensitivity to sigma on range of 0.5 - 1.5
+   * @param sensitivity double from 0 to 10
+   * @return sigma
+   */
+  private static double sensitivityToSigma(double sensitivity) {
+    // If out of bound, use boundary sensitivity
+    if (sensitivity < 0) {
+      sensitivity = 0;
+    } else if (sensitivity > 10) {
+      sensitivity = 10;
+    }
+    double sigma = 0.5 + 0.1 * (10 - sensitivity);
+    return sigma;
+  }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java
new file mode 100644
index 0000000..0ea1cba
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/MeanVarianceRuleDetectorSpec.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.spec;
+
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.detection.Pattern;
+
+
+public class MeanVarianceRuleDetectorSpec extends AbstractSpec {
+  private String monitoringGranularity = MetricSlice.NATIVE_GRANULARITY.toAggregationGranularityString(); // use native granularity by default
+  private int lookback = 52; //default look back of 52 units
+  private double sensitivity = 5; //default sensitivity of 5, equals +/- 1 sigma
+  private Pattern pattern = Pattern.UP_OR_DOWN;
+
+  public String getMonitoringGranularity() {
+    return monitoringGranularity;
+  }
+
+  public void setMonitoringGranularity(String monitoringGranularity) {
+    this.monitoringGranularity = monitoringGranularity;
+  }
+
+  public Pattern getPattern() {
+    return pattern;
+  }
+
+  public void setPattern(Pattern pattern) {
+    this.pattern = pattern;
+  }
+
+  public int getLookback() { return lookback;}
+
+  public void setLookback(int lookback) {this.lookback = lookback;}
+
+  public double getSensitivity() {
+    return sensitivity;
+  }
+
+  public void setSensitivity(double sensitivity) {
+    this.sensitivity = sensitivity;
+  }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java
new file mode 100644
index 0000000..d844629
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/MeanVarianceRuleDetectorTest.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *         http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pinot.thirdeye.detection.components;
+
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.lang.Math;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.TreeMultimap;
+import java.util.concurrent.TimeUnit;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
+import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
+import org.apache.pinot.thirdeye.detection.MockDataProvider;
+import org.apache.pinot.thirdeye.detection.Pattern;
+import org.apache.pinot.thirdeye.detection.algorithm.AlgorithmUtils;
+import org.apache.pinot.thirdeye.detection.spec.MeanVarianceRuleDetectorSpec;
+import org.apache.pinot.thirdeye.detection.spi.model.DetectionResult;
+import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
+import org.joda.time.Interval;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
+
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+
+
+public class MeanVarianceRuleDetectorTest {
+
+  private DataProvider provider;
+  private DataFrame data;
+  private Multimap<String, String> filters =  TreeMultimap.create();
+
+  @BeforeMethod
+  public void setUp() throws Exception {
+    try (Reader dataReader = new InputStreamReader(AlgorithmUtils.class.getResourceAsStream("timeseries-2y.csv"))) {
+      this.data = DataFrame.fromCsv(dataReader);
+      this.data.setIndex(COL_TIME);
+    }
+
+    MetricConfigDTO dailyMetricConfig = new MetricConfigDTO();
+    dailyMetricConfig.setId(1L);
+    dailyMetricConfig.setName("thirdeye-test-daily");
+    dailyMetricConfig.setDataset("thirdeye-test-dataset-daily");
+
+    DatasetConfigDTO dailyDatasetConfig = new DatasetConfigDTO();
+    dailyDatasetConfig.setTimeUnit(TimeUnit.DAYS);
+    dailyDatasetConfig.setDataset("thirdeye-test-dataset-daily");
+    dailyDatasetConfig.setTimeDuration(1);
+
+    Map<MetricSlice, DataFrame> timeseries = new HashMap<>();
+    timeseries.put(MetricSlice.from(1L, 1499126400000L, 1562630400000L, filters, TimeGranularity.fromString("7_DAYS")), this.data);
+
+    this.provider = new MockDataProvider()
+        .setTimeseries(timeseries)
+        .setMetrics(Collections.singletonList(dailyMetricConfig))
+        .setDatasets(Collections.singletonList(dailyDatasetConfig));
+  }
+
+  @Test
+  public void testComputePredictedTimeSeriesDaily() {
+    MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+    MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+    spec.setMonitoringGranularity("7_DAYS");
+    spec.setLookback(52);
+    spec.setSensitivity(5);
+    detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+    Interval window = new Interval(1530576000000L, 1562630400000L);
+    String metricUrn = "thirdeye:metric:1";
+    MetricEntity me = MetricEntity.fromURN(metricUrn);
+    MetricSlice slice = MetricSlice.from(me.getId(), window.getStartMillis(), window.getEndMillis(), me.getFilters(), TimeGranularity
+        .fromString(spec.getMonitoringGranularity()));
+    TimeSeries timeSeries = detector.computePredictedTimeSeries(slice);
+    Assert.assertEquals(timeSeries.getPredictedBaseline().size(), 52);
+    // prediction assertion. Expected value calculated offline using Python.
+    Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(0) * 1000.0) / 1000.0,174431.234);
+    Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(1) * 1000.0) / 1000.0,147184.878);
+    Assert.assertEquals(Math.round(timeSeries.getPredictedBaseline().getDouble(2) * 1000.0) / 1000.0,153660.665);
+  }
+
+  @Test
+  public void testWeekOverWeekLookbackChange() {
+    MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+    MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+    spec.setMonitoringGranularity("7_DAYS");
+    spec.setLookback(52);
+    spec.setSensitivity(5);
+    detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+    DetectionResult detectionResult = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+    List<MergedAnomalyResultDTO> anomalies = detectionResult.getAnomalies();
+    Assert.assertEquals(anomalies.size(), 6);
+    Assert.assertEquals(anomalies.get(0).getStartTime(), 1530576000000L);
+    Assert.assertEquals(anomalies.get(0).getEndTime(), 1531180800000L);
+    Assert.assertEquals(anomalies.get(1).getStartTime(), 1540252800000L);
+    Assert.assertEquals(anomalies.get(1).getEndTime(), 1541462400000L);
+    Assert.assertEquals(anomalies.get(2).getStartTime(), 1545696000000L);
+    Assert.assertEquals(anomalies.get(2).getEndTime(), 1546300800000L);
+    Assert.assertEquals(anomalies.get(3).getStartTime(), 1546905600000L);
+    Assert.assertEquals(anomalies.get(3).getEndTime(), 1547510400000L);
+    Assert.assertEquals(anomalies.get(4).getStartTime(), 1553558400000L);
+    Assert.assertEquals(anomalies.get(4).getEndTime(), 1554768000000L);
+    Assert.assertEquals(anomalies.get(5).getStartTime(), 1555372800000L);
+    Assert.assertEquals(anomalies.get(5).getEndTime(), 1557187200000L);
+  }
+
+  @Test
+  public void testWeekOverWeekLookbackChangeDown() {
+    MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+    MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+    spec.setMonitoringGranularity("7_DAYS");
+    spec.setLookback(52);
+    spec.setSensitivity(5);
+    spec.setPattern(Pattern.DOWN);
+    detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+    DetectionResult result = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+    List<MergedAnomalyResultDTO> anomalies = result.getAnomalies();
+    Assert.assertEquals(anomalies.size(), 4);
+    Assert.assertEquals(anomalies.get(0).getStartTime(), 1530576000000L);
+    Assert.assertEquals(anomalies.get(0).getEndTime(), 1531180800000L);
+    Assert.assertEquals(anomalies.get(1).getStartTime(), 1540857600000L);
+    Assert.assertEquals(anomalies.get(1).getEndTime(), 1541462400000L);
+    Assert.assertEquals(anomalies.get(2).getStartTime(), 1545696000000L);
+    Assert.assertEquals(anomalies.get(2).getEndTime(), 1546300800000L);
+    Assert.assertEquals(anomalies.get(3).getStartTime(), 1554163200000L);
+    Assert.assertEquals(anomalies.get(3).getEndTime(), 1554768000000L);
+  }
+
+  @Test
+  public void testWeekOverWeekLookbackChangeUp() {
+    MeanVarianceRuleDetector detector = new MeanVarianceRuleDetector();
+    MeanVarianceRuleDetectorSpec spec = new MeanVarianceRuleDetectorSpec();
+    spec.setMonitoringGranularity("7_DAYS");
+    spec.setLookback(52);
+    spec.setSensitivity(5);
+    spec.setPattern(Pattern.UP);
+    detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+    DetectionResult result = detector.runDetection(new Interval(1530576000000L, 1562630400000L), "thirdeye:metric:1");
+    List<MergedAnomalyResultDTO> anomalies = result.getAnomalies();
+    Assert.assertEquals(anomalies.size(), 4);
+    Assert.assertEquals(anomalies.get(0).getStartTime(), 1540252800000L);
+    Assert.assertEquals(anomalies.get(0).getEndTime(), 1540857600000L);
+    Assert.assertEquals(anomalies.get(1).getStartTime(),1546905600000L);
+    Assert.assertEquals(anomalies.get(1).getEndTime(),1547510400000L);
+    Assert.assertEquals(anomalies.get(2).getStartTime(),1553558400000L);
+    Assert.assertEquals(anomalies.get(2).getEndTime(),1554163200000L);
+    Assert.assertEquals(anomalies.get(3).getStartTime(),1555372800000L);
+    Assert.assertEquals(anomalies.get(3).getEndTime(),1557187200000L);
+  }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv
new file mode 100644
index 0000000..1af53a8
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/algorithm/timeseries-2y.csv
@@ -0,0 +1,107 @@
+timestamp,value
+1499126400000,100237
+1499731200000,103952
+1500336000000,111500
+1500940800000,117761
+1501545600000,123124
+1502150400000,132955
+1502755200000,124202
+1503360000000,117803
+1503964800000,112117
+1504569600000,106698
+1505174400000,110367
+1505779200000,109243
+1506384000000,108994
+1506988800000,116122
+1507593600000,108620
+1508198400000,108490
+1508803200000,104060
+1509408000000,108019
+1510012800000,106021
+1510617600000,109581
+1511222400000,104792
+1511827200000,97852
+1512432000000,99984
+1513036800000,97621
+1513641600000,92522
+1514246400000,77013
+1514851200000,82625
+1515456000000,104407
+1516060800000,104543
+1516665600000,109124
+1517270400000,112433
+1517875200000,112776
+1518480000000,115295
+1519084800000,116233
+1519689600000,118773
+1520294400000,109849
+1520899200000,118393
+1521504000000,130142
+1522108800000,132692
+1522713600000,122168
+1523318400000,127751
+1523923200000,130663
+1524528000000,141285
+1525132800000,143513
+1525737600000,158803
+1526342400000,174864
+1526947200000,198051
+1527552000000,160028
+1528156800000,170869
+1528761600000,179357
+1529366400000,169122
+1529971200000,172133
+1530576000000,145782
+1531180800000,152279
+1531785600000,148278
+1532390400000,146696
+1532995200000,144465
+1533600000000,151209
+1534204800000,144277
+1534809600000,139179
+1535414400000,147519
+1536019200000,138942
+1536624000000,141822
+1537228800000,136290
+1537833600000,144448
+1538438400000,148539
+1539043200000,147423
+1539648000000,155035
+1540252800000,175620
+1540857600000,148494
+1541462400000,141305
+1542067200000,144111
+1542672000000,144678
+1543276800000,137476
+1543881600000,145511
+1544486400000,141762
+1545091200000,142075
+1545696000000,114929
+1546300800000,107373
+1546905600000,136891
+1547510400000,142291
+1548115200000,146492
+1548720000000,153866
+1549324800000,150707
+1549929600000,157428
+1550534400000,153784
+1551139200000,158622
+1551744000000,162204
+1552348800000,162323
+1552953600000,171853
+1553558400000,256515
+1554163200000,194456
+1554768000000,191069
+1555372800000,255471
+1555977600000,300109
+1556582400000,347975
+1557187200000,346512
+1557792000000,367724
+1558396800000,375144
+1559001600000,366047
+1559606400000,358787
+1560211200000,369429
+1560816000000,379284
+1561420800000,369440
+1562025600000,352345
+1562630400000,338757


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org