You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2020/03/30 20:52:52 UTC
[incubator-pinot] 01/01: [TE] fix the merger issue that it can't
merge historical anomaly generated by multiple rules.
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch merger-fix-multiple-rules
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0ce1f7213ad454c8f2622962a5e867be239a046c
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Mon Mar 30 13:51:31 2020 -0700
[TE] fix the merger issue that it can't merge historical anomaly generated by multiple rules.
---
.../wrapper/ChildKeepingMergeWrapper.java | 22 +++++++++++++++++++---
.../wrapper/ChildKeepingMergeWrapperTest.java | 5 +++--
2 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index e78cba0..5f7b692 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -26,11 +26,13 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.algorithm.MergeWrapper;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
@@ -48,9 +50,23 @@ public class ChildKeepingMergeWrapper extends BaselineFillingMergeWrapper {
}
@Override
- // does not fetch any anomalies from database
+ // retrieve the anomalies that are detected by multiple detectors
protected List<MergedAnomalyResultDTO> retrieveAnomaliesFromDatabase(List<MergedAnomalyResultDTO> generated) {
- return Collections.emptyList();
+ AnomalySlice effectiveSlice = this.slice.withDetectionId(this.config.getId())
+ .withStart(this.getStartTime(generated) - this.maxGap - 1)
+ .withEnd(this.getEndTime(generated) + this.maxGap + 1);
+
+ Collection<MergedAnomalyResultDTO> anomalies =
+ this.provider.fetchAnomalies(Collections.singleton(effectiveSlice)).get(effectiveSlice);
+
+ return anomalies.stream()
+ .filter(anomaly -> !anomaly.isChild() && isDetectedByMultipleComponents(anomaly))
+ .collect(Collectors.toList());
+ }
+
+ private boolean isDetectedByMultipleComponents(MergedAnomalyResultDTO anomaly) {
+ String componentName = anomaly.getProperties().getOrDefault(PROP_DETECTOR_COMPONENT_NAME, "");
+ return componentName.contains(",");
}
@Override
@@ -59,7 +75,7 @@ public class ChildKeepingMergeWrapper extends BaselineFillingMergeWrapper {
Map<Long, MergedAnomalyResultDTO> existingParentAnomalies = new HashMap<>();
for (MergedAnomalyResultDTO anomaly : input) {
if (anomaly.getId() != null && !anomaly.getChildren().isEmpty()) {
- existingParentAnomalies.put(anomaly.getId(), anomaly);
+ existingParentAnomalies.put(anomaly.getId(), copyAnomalyInfo(anomaly, new MergedAnomalyResultDTO()));
}
}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
index a5a5bda..cb78d8b 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
@@ -103,7 +103,8 @@ public class ChildKeepingMergeWrapperTest {
MockPipelineLoader mockLoader = new MockPipelineLoader(this.runs, this.outputs);
this.provider = new MockDataProvider()
- .setLoader(mockLoader);
+ .setLoader(mockLoader)
+ .setAnomalies(Collections.emptyList());
}
@Test
@@ -292,7 +293,7 @@ public class ChildKeepingMergeWrapperTest {
MockPipelineLoader mockLoader = new MockPipelineLoader(this.runs, Collections.singletonList(new MockPipelineOutput(Collections.singletonList(anomaly), -1L)));
DataProvider provider = new MockDataProvider()
- .setLoader(mockLoader);
+ .setLoader(mockLoader).setAnomalies(Collections.emptyList());
DetectionPipelineResult output = new ChildKeepingMergeWrapper(provider, config, 1000, 3000).run();
List<MergedAnomalyResultDTO> anomalyResults = output.getAnomalies();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org