You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/04/22 21:09:17 UTC
[incubator-pinot] branch master updated: [TE] support monitoring
granularity for Holt Winters (#4149)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new b6ecc83 [TE] support monitoring granularity for Holt Winters (#4149)
b6ecc83 is described below
commit b6ecc83ca0d6fb25a5ad2b10bd3231f53d5bea8b
Author: Dian Tang <de...@gmail.com>
AuthorDate: Mon Apr 22 14:09:12 2019 -0700
[TE] support monitoring granularity for Holt Winters (#4149)
---
.../detection/components/HoltWintersDetector.java | 74 ++++++++++++++++++----
.../detection/spec/HoltWintersDetectorSpec.java | 10 +++
2 files changed, 72 insertions(+), 12 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/HoltWintersDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/HoltWintersDetector.java
index 8b9d183..6f0b6e2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/HoltWintersDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/HoltWintersDetector.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.commons.math3.analysis.MultivariateFunction;
import org.apache.commons.math3.optim.PointValuePair;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.dataframe.BooleanSeries;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
@@ -32,7 +33,6 @@ import org.apache.pinot.thirdeye.dataframe.Series;
import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.Pattern;
@@ -50,7 +50,6 @@ import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
-import org.joda.time.Period;
import org.apache.commons.math3.optim.MaxIter;
import org.apache.commons.math3.optim.nonlinear.scalar.ObjectiveFunction;
import org.apache.commons.math3.optim.MaxEval;
@@ -100,7 +99,8 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
private Pattern pattern;
private double sensitivity;
private boolean smoothing;
- private Period lookbackPeriod = ConfigUtils.parsePeriod(LOOKBACK + "DAYS");
+ private String monitoringGranularity;
+ private TimeGranularity timeGranularity;
@Override
public void init(HoltWintersDetectorSpec spec, InputDataFetcher dataFetcher) {
@@ -112,13 +112,28 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
this.pattern = spec.getPattern();
this.smoothing = spec.getSmoothing();
this.sensitivity = spec.getSensitivity();
+ this.monitoringGranularity = spec.getMonitoringGranularity();
+
+ if (this.monitoringGranularity.equals("1_MONTHS")) {
+ this.timeGranularity = MetricSlice.NATIVE_GRANULARITY;
+ } else {
+ this.timeGranularity = TimeGranularity.fromString(spec.getMonitoringGranularity());
+ }
}
@Override
public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
MetricEntity metricEntity = MetricEntity.fromSlice(slice, 0);
Interval window = new Interval(slice.getStart(), slice.getEnd());
- DateTime trainStart = window.getStart().minus(lookbackPeriod);
+ DateTime trainStart;
+ if (isMultiDayGranularity()) {
+ trainStart = window.getStart().minusDays(timeGranularity.getSize() * LOOKBACK);
+ } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+ trainStart = window.getStart().minusMonths(LOOKBACK);
+ } else {
+ trainStart = window.getStart().minusDays(LOOKBACK);
+ }
+
DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
.withMetricIdsForDataset(Collections.singleton(metricEntity.getId()))).getDatasetForMetricId()
.get(metricEntity.getId());
@@ -137,7 +152,15 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
@Override
public List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn) {
MetricEntity metricEntity = MetricEntity.fromURN(metricUrn);
- DateTime trainStart = window.getStart().minus(lookbackPeriod);
+ DateTime trainStart;
+ if (isMultiDayGranularity()) {
+ trainStart = window.getStart().minusDays(timeGranularity.getSize() * LOOKBACK);
+ } else if (this.monitoringGranularity.equals("1_MONTHS")) {
+ trainStart = window.getStart().minusMonths(LOOKBACK);
+ } else {
+ trainStart = window.getStart().minusDays(LOOKBACK);
+ }
+
DatasetConfigDTO datasetConfig = this.dataFetcher.fetchData(new InputDataSpec()
.withMetricIdsForDataset(Collections.singleton(metricEntity.getId()))).getDatasetForMetricId()
.get(metricEntity.getId());
@@ -145,7 +168,6 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
metricEntity.getFilters());
DataFrame dfInput = fetchData(metricEntity, trainStart.getMillis(), window.getEndMillis());
-
// Kernel smoothing
if (smoothing && !TimeUnit.DAYS.equals(datasetConfig.bucketTimeGranularity().getUnit())) {
int kernelSize = (int) (KERNEL_PERIOD / datasetConfig.bucketTimeGranularity().toMillis());
@@ -180,7 +202,7 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
// Anomalies
List<MergedAnomalyResultDTO> results = DetectionUtils.makeAnomalies(sliceData, df, COL_ANOMALY,
window.getEndMillis(),
- DetectionUtils.getMonitoringGranularityPeriod(MetricSlice.NATIVE_GRANULARITY.toAggregationGranularityString(),
+ DetectionUtils.getMonitoringGranularityPeriod(timeGranularity.toAggregationGranularityString(),
datasetConfig), datasetConfig);
return results;
@@ -198,7 +220,7 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
List<MetricSlice> slices = new ArrayList<>();
MetricSlice sliceData = MetricSlice.from(metricEntity.getId(), start, end,
- metricEntity.getFilters(), MetricSlice.NATIVE_GRANULARITY);
+ metricEntity.getFilters(), timeGranularity);
slices.add(sliceData);
LOG.info("Getting data for" + sliceData.toString());
InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(slices)
@@ -208,9 +230,27 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
}
/**
+ * Returns a data frame containing lookback number of data before prediction time
+ * @param originalDF the original dataframe
+ * @param time the prediction time, in unix timestamp
+ * @return DataFrame containing lookback number of data
+ */
+ private DataFrame getLookbackDF(DataFrame originalDF, Long time) {
+ LongSeries longSeries = (LongSeries) originalDF.get(COL_TIME);
+ int indexFinish = longSeries.find(time);
+ DataFrame df = DataFrame.builder(COL_TIME, COL_VALUE).build();
+
+ if (indexFinish != -1) {
+ int indexStart = Math.max(0, indexFinish - LOOKBACK);
+ df = df.append(originalDF.slice(indexStart, indexFinish));
+ }
+ return df;
+ }
+
+ /**
* Returns a data frame containing the same time daily data, based on input time
* @param originalDF the original dataframe
- * @param time the epoch time of the start of the day
+ * @param time the prediction time, in unix timestamp
* @return DataFrame containing same time of daily data for LOOKBACK number of days
*/
private DataFrame getDailyDF(DataFrame originalDF, Long time, String timezone) {
@@ -394,16 +434,21 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
double lastGamma = gamma;
for (int k = 0; k < size; k++) {
- DataFrame dailyDF = getDailyDF(inputDF, forecastDF.getLong(COL_TIME, k), timezone);
+ DataFrame trainingDF;
+ if (timeGranularity.equals(MetricSlice.NATIVE_GRANULARITY) && !this.monitoringGranularity.equals("1_MONTHS")) {
+ trainingDF = getDailyDF(inputDF, forecastDF.getLong(COL_TIME, k), timezone);
+ } else {
+ trainingDF = getLookbackDF(inputDF, forecastDF.getLong(COL_TIME, k));
+ }
// We need at least 2 periods of data
- if (dailyDF.size() < 2 * period) {
+ if (trainingDF.size() < 2 * period) {
continue;
}
resultTimeArray[k] = forecastDF.getLong(COL_TIME, k);
- double[] y = dailyDF.getDoubles(COL_VALUE).values();
+ double[] y = trainingDF.getDoubles(COL_VALUE).values();
HoltWintersParams params;
if (alpha < 0 && beta < 0 && gamma < 0) {
params = fitModelWithBOBYQA(y, lastAlpha, lastBeta, lastGamma);
@@ -514,6 +559,11 @@ public class HoltWintersDetector implements BaselineProvider<HoltWintersDetector
return z;
}
+ // Check whether monitoring timeGranularity is multiple days
+ private boolean isMultiDayGranularity() {
+ return !timeGranularity.equals(MetricSlice.NATIVE_GRANULARITY) && timeGranularity.getUnit() == TimeUnit.DAYS;
+ }
+
/**
* Container class to store holt winters parameters
*/
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/HoltWintersDetectorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/HoltWintersDetectorSpec.java
index e38067e..0541e67 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/HoltWintersDetectorSpec.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spec/HoltWintersDetectorSpec.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.thirdeye.detection.spec;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.detection.Pattern;
@@ -30,6 +31,7 @@ public class HoltWintersDetectorSpec extends AbstractSpec {
private double sensitivity = 5;
private Pattern pattern = Pattern.UP_OR_DOWN;
private boolean smoothing = true;
+ private String monitoringGranularity = MetricSlice.NATIVE_GRANULARITY.toAggregationGranularityString(); // use native granularity by default
public boolean getSmoothing() {
return smoothing;
@@ -59,6 +61,10 @@ public class HoltWintersDetectorSpec extends AbstractSpec {
return period;
}
+ public String getMonitoringGranularity() {
+ return monitoringGranularity;
+ }
+
public void setAlpha(double alpha) {
this.alpha = alpha;
}
@@ -86,4 +92,8 @@ public class HoltWintersDetectorSpec extends AbstractSpec {
public void setSmoothing(boolean smoothing) {
this.smoothing = smoothing;
}
+
+ public void setMonitoringGranularity(String monitoringGranularity) {
+ this.monitoringGranularity = monitoringGranularity;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org