You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/05/30 22:28:28 UTC
[incubator-pinot] branch master updated: [TE] add padding to
baseline endpoints (#4260)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3b9ebea [TE] add padding to baseline endpoints (#4260)
3b9ebea is described below
commit 3b9ebea18ca18e1dbd18c6c12a414162c6c9ceff
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Thu May 30 15:28:23 2019 -0700
[TE] add padding to baseline endpoints (#4260)
- add padding option to the predicted baseline endpoints.
- refactor rule-based baseline providers to return predicted baselines and boundaries.
---
.../thirdeye/detection/DetectionResource.java | 37 +++++++++++++++++-----
.../components/AbsoluteChangeRuleDetector.java | 5 ++-
.../components/PercentageChangeRuleDetector.java | 5 ++-
.../detection/components/RuleBaselineProvider.java | 13 ++++++--
.../components/ThresholdRuleDetector.java | 1 -
.../components/RuleBaselineProviderTest.java | 6 ++--
6 files changed, 46 insertions(+), 21 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
index cb7e761..9104593 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
@@ -52,6 +52,8 @@ import org.apache.pinot.thirdeye.constant.AnomalyFeedbackType;
import org.apache.pinot.thirdeye.constant.AnomalyResultSource;
import org.apache.pinot.thirdeye.dashboard.resources.v2.ResourceUtils;
import org.apache.pinot.thirdeye.dashboard.resources.v2.rootcause.AnomalyEventFormatter;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
@@ -60,6 +62,7 @@ import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
@@ -75,12 +78,18 @@ import org.apache.pinot.thirdeye.detection.finetune.GridSearchTuningAlgorithm;
import org.apache.pinot.thirdeye.detection.finetune.TuningAlgorithm;
import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
+import org.apache.pinot.thirdeye.detector.function.BaseAnomalyFunction;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
+import org.apache.pinot.thirdeye.util.AnomalyOffset;
import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+
@Path("/detection")
@Produces(MediaType.APPLICATION_JSON)
@@ -518,23 +527,35 @@ public class DetectionResource {
}
@GET
- @ApiOperation("get the predicted baseline for an anomaly within a time range")
+ @ApiOperation("get the current time series and predicted baselines for an anomaly within a time range")
@Path(value = "/predicted-baseline/{anomalyId}")
public Response getPredictedBaseline(
@PathParam("anomalyId") @ApiParam("anomalyId") long anomalyId,
- @QueryParam("start") long start,
- @QueryParam("end") long end
+ @ApiParam("Start time for the predicted baselines") @QueryParam("start") long start,
+ @ApiParam("End time for the predicted baselines") @QueryParam("end") long end,
+ @ApiParam("Add padding to the window based on metric granularity") @QueryParam("padding") @DefaultValue("false") boolean padding
) throws Exception {
MergedAnomalyResultDTO anomaly = anomalyDAO.findById(anomalyId);
if (anomaly == null) {
throw new IllegalArgumentException(String.format("Could not resolve anomaly id %d", anomalyId));
}
- MetricConfigDTO metric = this.metricDAO.findByMetricAndDataset(anomaly.getMetric(), anomaly.getCollection());
- if (metric == null) {
- throw new IllegalArgumentException(String.format("Could not resolve metric '%s' in dataset '%s' for anomaly id %d", anomaly.getMetric(), anomaly.getCollection(), anomaly.getId()));
+ if (padding) {
+ // add paddings for the time range
+ DatasetConfigDTO dataset = this.datasetDAO.findByDataset(anomaly.getCollection());
+ if (dataset == null) {
+ throw new IllegalArgumentException(String.format("Could not resolve dataset '%s' for anomaly id %d", anomaly.getCollection(), anomalyId));
+ }
+ AnomalyOffset offsets = BaseAnomalyFunction.getDefaultOffsets(dataset);
+ DateTimeZone dataTimeZone = DateTimeZone.forID(dataset.getTimezone());
+ start = new DateTime(start, dataTimeZone).minus(offsets.getPreOffsetPeriod()).getMillis();
+ end = new DateTime(end, dataTimeZone).plus(offsets.getPostOffsetPeriod()).getMillis();
}
- TimeSeries ts = DetectionUtils.getBaselineTimeseries(anomaly, Multimaps.forMap(anomaly.getDimensions()), metric.getId(), configDAO.findById(anomaly.getDetectionConfigId()), start, end, loader, provider);
- return Response.ok(ts.getDataFrame()).build();
+ MetricEntity me = MetricEntity.fromURN(anomaly.getMetricUrn());
+ TimeSeries baselineTimeseries = DetectionUtils.getBaselineTimeseries(anomaly, me.getFilters(), me.getId(), configDAO.findById(anomaly.getDetectionConfigId()), start, end, loader, provider);
+ // add current time series
+ MetricSlice currentSlice = MetricSlice.from(me.getId(), start, end, me.getFilters());
+ DataFrame dfCurrent = this.provider.fetchTimeseries(Collections.singleton(currentSlice)).get(currentSlice).renameSeries(COL_VALUE, COL_CURRENT);
+ return Response.ok(dfCurrent.joinOuter(baselineTimeseries.getDataFrame())).build();
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/AbsoluteChangeRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
index 9b80912..e008748 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
@@ -117,9 +117,8 @@ public class AbsoluteChangeRuleDetector implements AnomalyDetector<AbsoluteChang
@Override
public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
- InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(this.baseline.scatter(slice)));
- DataFrame dfBase = this.baseline.gather(slice, data.getTimeseries());
- return TimeSeries.fromDataFrame(constructAbsoluteChangeBoundaries(dfBase));
+ DataFrame df = RuleBaselineProvider.buildBaselines(slice, this.baseline, this.dataFetcher);
+ return TimeSeries.fromDataFrame(constructAbsoluteChangeBoundaries(df));
}
private DataFrame constructAbsoluteChangeBoundaries(DataFrame dfBase) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetector.java
index bd4101a..8632b6d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetector.java
@@ -121,9 +121,8 @@ public class PercentageChangeRuleDetector implements AnomalyDetector<PercentageC
@Override
public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
- InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(this.baseline.scatter(slice)));
- DataFrame dfBase = this.baseline.gather(slice, data.getTimeseries());
- return TimeSeries.fromDataFrame(constructPercentageChangeBoundaries(dfBase));
+ DataFrame df = RuleBaselineProvider.buildBaselines(slice, this.baseline, this.dataFetcher);
+ return TimeSeries.fromDataFrame(constructPercentageChangeBoundaries(df));
}
private DataFrame constructPercentageChangeBoundaries(DataFrame dfBase) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProvider.java
index 40ca80a..a5ec9c4 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProvider.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProvider.java
@@ -19,8 +19,10 @@
package org.apache.pinot.thirdeye.detection.components;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.pinot.thirdeye.dashboard.resources.v2.BaselineParsingUtils;
-import org.apache.pinot.thirdeye.dataframe.Series;
+import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.annotation.Components;
@@ -44,8 +46,13 @@ public class RuleBaselineProvider implements BaselineProvider<RuleBaselineProvid
@Override
public TimeSeries computePredictedTimeSeries(MetricSlice slice) {
- InputData data = this.dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(this.baseline.scatter(slice)));
- return TimeSeries.fromDataFrame(this.baseline.gather(slice, data.getTimeseries()));
+ return TimeSeries.fromDataFrame(buildBaselines(slice, this.baseline, this.dataFetcher));
+ }
+
+ static DataFrame buildBaselines(MetricSlice slice, Baseline baseline, InputDataFetcher dataFetcher) {
+ List<MetricSlice> slices = new ArrayList<>(baseline.scatter(slice));
+ InputData data = dataFetcher.fetchData(new InputDataSpec().withTimeseriesSlices(slices));
+ return baseline.gather(slice, data.getTimeseries());
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetector.java
index 3d7fd83..659f656 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetector.java
@@ -43,7 +43,6 @@ import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.joda.time.Interval;
-import org.joda.time.Period;
import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProviderTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProviderTest.java
index 0b3917f..f8fef7d 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProviderTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/RuleBaselineProviderTest.java
@@ -16,6 +16,9 @@
package org.apache.pinot.thirdeye.detection.components;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.DoubleSeries;
import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
@@ -23,9 +26,6 @@ import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.MockDataProvider;
import org.apache.pinot.thirdeye.detection.spec.RuleBaselineProviderSpec;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org