You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pinot.apache.org by GitBox <gi...@apache.org> on 2018/11/14 23:47:35 UTC
[GitHub] jihaozh closed pull request #3482: [TE] detection - site-wide
impact rule filter
jihaozh closed pull request #3482: [TE] detection - site-wide impact rule filter
URL: https://github.com/apache/incubator-pinot/pull/3482
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DefaultInputDataFetcher.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DefaultInputDataFetcher.java
index 5e3ce05ae1..2d4bb3d179 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DefaultInputDataFetcher.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/DefaultInputDataFetcher.java
@@ -50,10 +50,8 @@ public DefaultInputDataFetcher(DataProvider provider, long configId) {
*/
public InputData fetchData(InputDataSpec inputDataSpec) {
Map<MetricSlice, DataFrame> timeseries = provider.fetchTimeseries(inputDataSpec.getTimeseriesSlices());
- Map<MetricSlice, DataFrame> aggregates =
- provider.fetchAggregates(inputDataSpec.getAggregateSlices(), Collections.<String>emptyList());
- Multimap<AnomalySlice, MergedAnomalyResultDTO> existingAnomalies =
- provider.fetchAnomalies(inputDataSpec.getAnomalySlices(), configId);
+ Map<MetricSlice, DataFrame> aggregates = provider.fetchAggregates(inputDataSpec.getAggregateSlices(), Collections.<String>emptyList());
+ Multimap<AnomalySlice, MergedAnomalyResultDTO> existingAnomalies = provider.fetchAnomalies(inputDataSpec.getAnomalySlices(), configId);
Multimap<EventSlice, EventDTO> events = provider.fetchEvents(inputDataSpec.getEventSlices());
Map<Long, MetricConfigDTO> metrics = provider.fetchMetrics(inputDataSpec.getMetricIds());
Map<String, DatasetConfigDTO> datasets = provider.fetchDatasets(inputDataSpec.getDatasetNames());
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/Pattern.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/Pattern.java
new file mode 100644
index 0000000000..40b14bf03d
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/Pattern.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.linkedin.thirdeye.detection;
+
+/**
+ * Up or down detection pattern
+ */
+public enum Pattern {
+ UP,
+ DOWN
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
index a8008b8f27..0c812c7cba 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetector.java
@@ -24,6 +24,7 @@
import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.detection.DetectionUtils;
import com.linkedin.thirdeye.detection.InputDataFetcher;
+import com.linkedin.thirdeye.detection.Pattern;
import com.linkedin.thirdeye.detection.annotation.Components;
import com.linkedin.thirdeye.detection.annotation.DetectionTag;
import com.linkedin.thirdeye.detection.annotation.Param;
@@ -46,21 +47,24 @@
type = "ABSOLUTE_CHANGE_RULE",
tags = {DetectionTag.RULE_DETECTION},
presentation = {
- @PresentationOption(name = "absolute value", template = "comparing ${offset} is more than ${difference}"),
+ @PresentationOption(name = "absolute value", template = "comparing ${offset} is ${pattern} more than ${difference}"),
},
params = {
@Param(name = "offset", defaultValue = "wo1w"),
- @Param(name = "change", placeholder = "value")
+ @Param(name = "change", placeholder = "value"),
+ @Param(name = "pattern", allowableValues = {"up", "down"})
})
public class AbsoluteChangeRuleDetector implements AnomalyDetector<AbsoluteChangeRuleDetectorSpec> {
private double absoluteChange;
private InputDataFetcher dataFetcher;
private Baseline baseline;
+ private Pattern pattern;
private static final String COL_CURR = "current";
private static final String COL_BASE = "baseline";
- private static final String COL_CHANGE = "change";
private static final String COL_ANOMALY = "anomaly";
private static final String COL_DIFF = "diff";
+ private static final String COL_PATTERN = "pattern";
+ private static final String COL_DIFF_VIOLATION = "diff_violation";
@Override
public List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn) {
@@ -82,7 +86,9 @@
// absolute change
if (!Double.isNaN(this.absoluteChange)) {
- df.addSeries(COL_ANOMALY, df.getDoubles(COL_DIFF).abs().gte(this.absoluteChange));
+ df.addSeries(COL_PATTERN, this.pattern.equals(Pattern.UP) ? df.getDoubles(COL_DIFF).gt(0) : df.getDoubles(COL_DIFF).lt(0));
+ df.addSeries(COL_DIFF_VIOLATION, df.getDoubles(COL_DIFF).abs().gte(this.absoluteChange));
+ df.mapInPlace(BooleanSeries.ALL_TRUE, COL_ANOMALY, COL_PATTERN, COL_DIFF_VIOLATION);
}
// make anomalies
@@ -92,10 +98,11 @@
@Override
public void init(AbsoluteChangeRuleDetectorSpec spec, InputDataFetcher dataFetcher) {
- this.absoluteChange = spec.getAbsoluteChangeChange();
+ this.absoluteChange = spec.getAbsoluteChange();
this.dataFetcher = dataFetcher;
String timezone = spec.getTimezone();
String offset = spec.getOffset();
this.baseline = BaselineParsingUtils.parseOffset(offset, timezone);
+ this.pattern = Pattern.valueOf(spec.getPattern().toUpperCase());
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetector.java
index 1814e6c6be..14f09d5c71 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetector.java
@@ -25,6 +25,7 @@
import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import com.linkedin.thirdeye.detection.DetectionUtils;
import com.linkedin.thirdeye.detection.InputDataFetcher;
+import com.linkedin.thirdeye.detection.Pattern;
import com.linkedin.thirdeye.detection.annotation.Components;
import com.linkedin.thirdeye.detection.annotation.DetectionTag;
import com.linkedin.thirdeye.detection.annotation.Param;
@@ -49,20 +50,24 @@
description = "Computes a multi-week aggregate baseline and compares the current value "
+ "based on relative change.",
presentation = {
- @PresentationOption(name = "percentage change", template = "comparing ${offset} is more than ${change}")
+ @PresentationOption(name = "percentage change", template = "comparing ${offset} is ${pattern} more than ${change}")
},
params = {
@Param(name = "offset", defaultValue = "wo1w"),
- @Param(name = "change", placeholder = "value")
+ @Param(name = "change", placeholder = "value"),
+ @Param(name = "pattern", allowableValues = {"up", "down"})
})
public class PercentageChangeRuleDetector implements AnomalyDetector<PercentageChangeRuleDetectorSpec> {
private double percentageChange;
private InputDataFetcher dataFetcher;
private Baseline baseline;
+ private Pattern pattern;
private static final String COL_CURR = "current";
private static final String COL_BASE = "baseline";
private static final String COL_CHANGE = "change";
private static final String COL_ANOMALY = "anomaly";
+ private static final String COL_PATTERN = "pattern";
+ private static final String COL_CHANGE_VIOLATION = "change_violation";
@Override
public List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn) {
@@ -89,7 +94,9 @@
// relative change
if (!Double.isNaN(this.percentageChange)) {
- df.addSeries(COL_ANOMALY, df.getDoubles(COL_CHANGE).abs().gte(this.percentageChange));
+ df.addSeries(COL_PATTERN, this.pattern.equals(Pattern.UP) ? df.getDoubles(COL_CHANGE).gt(0) : df.getDoubles(COL_CHANGE).lt(0));
+ df.addSeries(COL_CHANGE_VIOLATION, df.getDoubles(COL_CHANGE).abs().gte(this.percentageChange));
+ df.mapInPlace(BooleanSeries.ALL_TRUE, COL_ANOMALY, COL_PATTERN, COL_CHANGE_VIOLATION);
}
// anomalies
@@ -104,5 +111,6 @@ public void init(PercentageChangeRuleDetectorSpec spec, InputDataFetcher dataFet
String timezone = spec.getTimezone();
String offset = spec.getOffset();
this.baseline = BaselineParsingUtils.parseOffset(offset, timezone);
+ this.pattern = Pattern.valueOf(spec.getPattern().toUpperCase());
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
new file mode 100644
index 0000000000..fa95b447d9
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilter.java
@@ -0,0 +1,67 @@
+package com.linkedin.thirdeye.detection.components;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.ArrayListMultimap;
+import com.linkedin.thirdeye.dashboard.resources.v2.BaselineParsingUtils;
+import com.linkedin.thirdeye.dataframe.DataFrame;
+import com.linkedin.thirdeye.dataframe.util.MetricSlice;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import com.linkedin.thirdeye.detection.InputDataFetcher;
+import com.linkedin.thirdeye.detection.annotation.Components;
+import com.linkedin.thirdeye.detection.spec.SitewideImpactRuleAnomalyFilterSpec;
+import com.linkedin.thirdeye.detection.spi.components.AnomalyFilter;
+import com.linkedin.thirdeye.detection.spi.model.InputDataSpec;
+import com.linkedin.thirdeye.rootcause.impl.MetricEntity;
+import com.linkedin.thirdeye.rootcause.timeseries.Baseline;
+import java.util.Arrays;
+import java.util.Map;
+
+import static com.linkedin.thirdeye.dataframe.util.DataFrameUtils.*;
+
+/**
+ * Site-wide impact anomaly filter
+ */
+@Components(type = "SITEWIDE_IMPACT_FILTER")
+public class SitewideImpactRuleAnomalyFilter implements AnomalyFilter<SitewideImpactRuleAnomalyFilterSpec> {
+ private double threshold;
+ private InputDataFetcher dataFetcher;
+ private Baseline baseline;
+ private String siteWideMetricUrn;
+
+ @Override
+ public boolean isQualified(MergedAnomalyResultDTO anomaly) {
+ MetricEntity me = MetricEntity.fromURN(anomaly.getMetricUrn());
+ MetricSlice currentSlice =
+ MetricSlice.from(me.getId(), anomaly.getStartTime(), anomaly.getEndTime(), me.getFilters());
+ MetricSlice baselineSlice = this.baseline.scatter(currentSlice).get(0);
+
+ String siteWideImpactMetricUrn = Strings.isNullOrEmpty(this.siteWideMetricUrn) ? anomaly.getMetricUrn() : this.siteWideMetricUrn;
+ MetricEntity siteWideEntity = MetricEntity.fromURN(siteWideImpactMetricUrn).withFilters(ArrayListMultimap.<String, String>create());
+ MetricSlice siteWideSlice = this.baseline.scatter(MetricSlice.from(siteWideEntity.getId(), anomaly.getStartTime(), anomaly.getEndTime())).get(0);
+
+ Map<MetricSlice, DataFrame> aggregates =
+ this.dataFetcher.fetchData(new InputDataSpec().withAggregateSlices(Arrays.asList(currentSlice, baselineSlice, siteWideSlice))).getAggregates();
+
+ double currentValue = getValueFromAggregates(currentSlice, aggregates);
+ double baselineValue = getValueFromAggregates(baselineSlice, aggregates);
+ double siteWideBaselineValue = getValueFromAggregates(siteWideSlice, aggregates);
+
+ if (siteWideBaselineValue != 0 && (Math.abs(currentValue - baselineValue) / siteWideBaselineValue) < this.threshold) {
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public void init(SitewideImpactRuleAnomalyFilterSpec spec, InputDataFetcher dataFetcher) {
+ this.dataFetcher = dataFetcher;
+ this.baseline = BaselineParsingUtils.parseOffset(spec.getOffset(), spec.getTimezone());
+ this.threshold = spec.getThreshold();
+ this.siteWideMetricUrn = spec.getSitewideMetricUrn();
+ }
+
+ private double getValueFromAggregates(MetricSlice slice, Map<MetricSlice, DataFrame> aggregates) {
+ return aggregates.get(slice).getDouble(COL_VALUE, 0);
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
index cc6f8cc17c..b496d5e21b 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleAnomalyFilter.java
@@ -40,7 +40,7 @@
*/
@Components(title = "Aggregate Threshold Filter", type = "THRESHOLD_RULE_FILTER", tags = {
DetectionTag.RULE_FILTER}, description = "Threshold rule filter. filters the anomalies if either the min or max thresholds do not satisfied.", presentation = {
- @PresentationOption(name = "absolute value", description = "aggregated absolute value within a time period", template = "is between ${min} and ${max}")}, params = {
+ @PresentationOption(name = "absolute value", description = "aggregated absolute value within a time period", template = "is higher than ${min} and lower than ${max}")}, params = {
@Param(name = "min", placeholder = "value"), @Param(name = "max", placeholder = "value")})
public class ThresholdRuleAnomalyFilter implements AnomalyFilter<ThresholdRuleFilterSpec> {
private double min;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleDetector.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleDetector.java
index 0fc223051e..40809e7ad9 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/components/ThresholdRuleDetector.java
@@ -41,7 +41,7 @@
@Components(title = "Threshold", type = "THRESHOLD", tags = {
DetectionTag.RULE_DETECTION}, description = "Simple threshold rule algorithm with (optional) upper and lower bounds on a metric value.", presentation = {
- @PresentationOption(name = "absolute value", description = "aggregated absolute value within a time period", template = "is lower than ${min} or higher than ${max}")}, params = {
+ @PresentationOption(name = "absolute value", template = "is lower than ${min} or higher than ${max}")}, params = {
@Param(name = "min", placeholder = "value"), @Param(name = "max", placeholder = "value")})
public class ThresholdRuleDetector implements AnomalyDetector<ThresholdRuleDetectorSpec> {
private final String COL_TOO_HIGH = "tooHigh";
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
index c69659ea85..c6b25aa512 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/AbsoluteChangeRuleDetectorSpec.java
@@ -16,10 +16,30 @@
package com.linkedin.thirdeye.detection.spec;
+import com.linkedin.thirdeye.detection.Pattern;
+
+
public class AbsoluteChangeRuleDetectorSpec extends AbstractSpec {
private double absoluteChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
+ private String pattern;
+
+ public double getAbsoluteChange() {
+ return absoluteChange;
+ }
+
+ public void setAbsoluteChange(double absoluteChange) {
+ this.absoluteChange = absoluteChange;
+ }
+
+ public String getPattern() {
+ return pattern;
+ }
+
+ public void setPattern(String pattern) {
+ this.pattern = pattern;
+ }
public String getTimezone() {
return timezone;
@@ -36,12 +56,4 @@ public String getOffset() {
public void setOffset(String offset) {
this.offset = offset;
}
-
- public double getAbsoluteChangeChange() {
- return absoluteChange;
- }
-
- public void setAbsoluteChangeChange(double absoluteChange) {
- this.absoluteChange = absoluteChange;
- }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
index c1e78fb9d6..d58533e39d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/PercentageChangeRuleDetectorSpec.java
@@ -20,6 +20,15 @@
private double percentageChange = Double.NaN;
private String offset = "wo1w";
private String timezone = "UTC";
+ private String pattern;
+
+ public String getPattern() {
+ return pattern;
+ }
+
+ public void setPattern(String pattern) {
+ this.pattern = pattern;
+ }
public String getTimezone() {
return timezone;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
new file mode 100644
index 0000000000..7da2799dea
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/detection/spec/SitewideImpactRuleAnomalyFilterSpec.java
@@ -0,0 +1,40 @@
+package com.linkedin.thirdeye.detection.spec;
+
+public class SitewideImpactRuleAnomalyFilterSpec extends AbstractSpec {
+ private String timezone = "UTC";
+ private String sitewideMetricUrn;
+ private double threshold = Double.NaN;
+ private String offset = "wo1w";
+
+ public String getOffset() {
+ return offset;
+ }
+
+ public void setOffset(String offset) {
+ this.offset = offset;
+ }
+
+ public String getTimezone() {
+ return timezone;
+ }
+
+ public void setTimezone(String timezone) {
+ this.timezone = timezone;
+ }
+
+ public String getSitewideMetricUrn() {
+ return sitewideMetricUrn;
+ }
+
+ public void setSitewideMetricUrn(String sitewideMetricUrn) {
+ this.sitewideMetricUrn = sitewideMetricUrn;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public void setThreshold(double threshold) {
+ this.threshold = threshold;
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetectorTest.java
index fc73e8d1d8..8a25904d4c 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetectorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/AbsoluteChangeRuleDetectorTest.java
@@ -78,10 +78,11 @@ public void beforeMethod() throws Exception {
}
@Test
- public void testWeekOverWeekDifference() throws Exception {
+ public void testWeekOverWeekDifference() {
AbsoluteChangeRuleDetector detector = new AbsoluteChangeRuleDetector();
AbsoluteChangeRuleDetectorSpec spec = new AbsoluteChangeRuleDetectorSpec();
- spec.setAbsoluteChangeChange(400);
+ spec.setAbsoluteChange(400);
+ spec.setPattern("up");
detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
List<MergedAnomalyResultDTO> anomalies = detector.runDetection(new Interval(1814400000L, 2419200000L), "thirdeye:metric:1");
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
index b1e878c7da..d3f4e6dcd3 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
@@ -78,9 +78,10 @@ public void beforeMethod() throws Exception {
}
@Test
- public void testWeekOverWeekChange() throws Exception {
+ public void testWeekOverWeekChange() {
PercentageChangeRuleDetector detector = new PercentageChangeRuleDetector();
PercentageChangeRuleDetectorSpec spec = new PercentageChangeRuleDetectorSpec();
+ spec.setPattern("up");
spec.setPercentageChange(0.4);
detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
List<MergedAnomalyResultDTO> anomalies = detector.runDetection(new Interval(1814400000L, 2419200000L), "thirdeye:metric:1");
@@ -92,23 +93,36 @@ public void testWeekOverWeekChange() throws Exception {
}
@Test
- public void testThreeWeekMedianChange() throws Exception {
+ public void testThreeWeekMedianChange() {
PercentageChangeRuleDetector detector = new PercentageChangeRuleDetector();
PercentageChangeRuleDetectorSpec spec = new PercentageChangeRuleDetectorSpec();
spec.setPercentageChange(0.3);
spec.setOffset("median3w");
+ spec.setPattern("up");
detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
List<MergedAnomalyResultDTO> anomalies = detector.runDetection(new Interval(1814400000L, 2419200000L), "thirdeye:metric:1");
- Assert.assertEquals(anomalies.size(), 5);
+ Assert.assertEquals(anomalies.size(), 4);
Assert.assertEquals(anomalies.get(0).getStartTime(), 2005200000L);
Assert.assertEquals(anomalies.get(0).getEndTime(), 2008800000L);
Assert.assertEquals(anomalies.get(1).getStartTime(), 2134800000L);
Assert.assertEquals(anomalies.get(1).getEndTime(), 2138400000L);
Assert.assertEquals(anomalies.get(2).getStartTime(), 2152800000L);
Assert.assertEquals(anomalies.get(2).getEndTime(), 2156400000L);
- Assert.assertEquals(anomalies.get(3).getStartTime(), 2181600000L);
- Assert.assertEquals(anomalies.get(3).getEndTime(), 2185200000L);
- Assert.assertEquals(anomalies.get(4).getStartTime(), 2322000000L);
- Assert.assertEquals(anomalies.get(4).getEndTime(), 2325600000L);
+ Assert.assertEquals(anomalies.get(3).getStartTime(), 2322000000L);
+ Assert.assertEquals(anomalies.get(3).getEndTime(), 2325600000L);
+ }
+
+ @Test
+ public void testThreeWeekMedianChangeDown() {
+ PercentageChangeRuleDetector detector = new PercentageChangeRuleDetector();
+ PercentageChangeRuleDetectorSpec spec = new PercentageChangeRuleDetectorSpec();
+ spec.setPercentageChange(0.3);
+ spec.setOffset("median3w");
+ spec.setPattern("down");
+ detector.init(spec, new DefaultInputDataFetcher(this.provider, -1));
+ List<MergedAnomalyResultDTO> anomalies = detector.runDetection(new Interval(1814400000L, 2419200000L), "thirdeye:metric:1");
+ Assert.assertEquals(anomalies.size(), 1);
+ Assert.assertEquals(anomalies.get(0).getStartTime(), 2181600000L);
+ Assert.assertEquals(anomalies.get(0).getEndTime(), 2185200000L);
}
}
\ No newline at end of file
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilterTest.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilterTest.java
new file mode 100644
index 0000000000..fb9f57ae89
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/detection/components/SitewideImpactRuleAnomalyFilterTest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.linkedin.thirdeye.detection.components;
+
+import com.linkedin.thirdeye.dataframe.DataFrame;
+import com.linkedin.thirdeye.dataframe.util.MetricSlice;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import com.linkedin.thirdeye.detection.DataProvider;
+import com.linkedin.thirdeye.detection.DefaultInputDataFetcher;
+import com.linkedin.thirdeye.detection.DetectionTestUtils;
+import com.linkedin.thirdeye.detection.MockDataProvider;
+import com.linkedin.thirdeye.detection.spec.SitewideImpactRuleAnomalyFilterSpec;
+import com.linkedin.thirdeye.rootcause.timeseries.Baseline;
+import com.linkedin.thirdeye.rootcause.timeseries.BaselineAggregate;
+import com.linkedin.thirdeye.rootcause.timeseries.BaselineAggregateType;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.joda.time.DateTimeZone;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import static com.linkedin.thirdeye.dataframe.util.DataFrameUtils.*;
+
+
+public class SitewideImpactRuleAnomalyFilterTest {
+ private static final String METRIC_URN = "thirdeye:metric:123";
+
+ private List<MergedAnomalyResultDTO> anomalies;
+ private DataProvider testDataProvider;
+ private Baseline baseline;
+
+ @BeforeMethod
+ public void beforeMethod() {
+ this.baseline = BaselineAggregate.fromWeekOverWeek(BaselineAggregateType.MEDIAN, 1, 1, DateTimeZone.forID("UTC"));
+
+ MetricSlice slice1 = MetricSlice.from(123L, 0, 2);
+ MetricSlice baselineSlice1 = this.baseline.scatter(slice1).get(0);
+ MetricSlice slice2 = MetricSlice.from(123L, 4, 6);
+ MetricSlice baselineSlice2 = this.baseline.scatter(slice2).get(0);
+
+ Map<MetricSlice, DataFrame> aggregates = new HashMap<>();
+ aggregates.put(slice1, new DataFrame().addSeries(COL_VALUE, 150));
+ aggregates.put(baselineSlice1, new DataFrame().addSeries(COL_VALUE, 200));
+ aggregates.put(slice2, new DataFrame().addSeries(COL_VALUE, 500));
+ aggregates.put(baselineSlice2, new DataFrame().addSeries(COL_VALUE, 1000));
+
+ this.anomalies = Arrays.asList(makeAnomaly(0, 2), makeAnomaly(4, 6));
+
+ this.testDataProvider = new MockDataProvider().setAggregates(aggregates);
+ }
+
+ @Test
+ public void testSiteWideImpactFilter() {
+ SitewideImpactRuleAnomalyFilterSpec spec = new SitewideImpactRuleAnomalyFilterSpec();
+ spec.setThreshold(0.5);
+ spec.setOffset("median3w");
+ SitewideImpactRuleAnomalyFilter filter = new SitewideImpactRuleAnomalyFilter();
+ filter.init(spec, new DefaultInputDataFetcher(this.testDataProvider, 125L));
+
+ List<Boolean> results =
+ this.anomalies.stream().map(anomaly -> filter.isQualified(anomaly)).collect(Collectors.toList());
+ Assert.assertEquals(results, Arrays.asList(false, true));
+ }
+
+ private static MergedAnomalyResultDTO makeAnomaly(long start, long end) {
+ Map<String, String> dimensions = new HashMap<>();
+ MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(125L, start, end, dimensions);
+ anomaly.setMetricUrn(METRIC_URN);
+ return anomaly;
+ }
+}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pinot.apache.org
For additional commands, e-mail: dev-help@pinot.apache.org