You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2020/03/30 20:52:51 UTC
[incubator-pinot] branch merger-fix-multiple-rules created (now
0ce1f72)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a change to branch merger-fix-multiple-rules
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 0ce1f72 [TE] fix the merger issue that it can't merge historical anomaly generated by multiple rules.
This branch includes the following new commits:
new 0ce1f72 [TE] fix the merger issue that it can't merge historical anomaly generated by multiple rules.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: [TE] fix the merger issue that it can't
merge historical anomaly generated by multiple rules.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch merger-fix-multiple-rules
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0ce1f7213ad454c8f2622962a5e867be239a046c
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Mon Mar 30 13:51:31 2020 -0700
[TE] fix the merger issue that it can't merge historical anomaly generated by multiple rules.
---
.../wrapper/ChildKeepingMergeWrapper.java | 22 +++++++++++++++++++---
.../wrapper/ChildKeepingMergeWrapperTest.java | 5 +++--
2 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
index e78cba0..5f7b692 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapper.java
@@ -26,11 +26,13 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.algorithm.MergeWrapper;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
@@ -48,9 +50,23 @@ public class ChildKeepingMergeWrapper extends BaselineFillingMergeWrapper {
}
@Override
- // does not fetch any anomalies from database
+ // retrieve the anomalies that are detected by multiple detectors
protected List<MergedAnomalyResultDTO> retrieveAnomaliesFromDatabase(List<MergedAnomalyResultDTO> generated) {
- return Collections.emptyList();
+ AnomalySlice effectiveSlice = this.slice.withDetectionId(this.config.getId())
+ .withStart(this.getStartTime(generated) - this.maxGap - 1)
+ .withEnd(this.getEndTime(generated) + this.maxGap + 1);
+
+ Collection<MergedAnomalyResultDTO> anomalies =
+ this.provider.fetchAnomalies(Collections.singleton(effectiveSlice)).get(effectiveSlice);
+
+ return anomalies.stream()
+ .filter(anomaly -> !anomaly.isChild() && isDetectedByMultipleComponents(anomaly))
+ .collect(Collectors.toList());
+ }
+
+ private boolean isDetectedByMultipleComponents(MergedAnomalyResultDTO anomaly) {
+ String componentName = anomaly.getProperties().getOrDefault(PROP_DETECTOR_COMPONENT_NAME, "");
+ return componentName.contains(",");
}
@Override
@@ -59,7 +75,7 @@ public class ChildKeepingMergeWrapper extends BaselineFillingMergeWrapper {
Map<Long, MergedAnomalyResultDTO> existingParentAnomalies = new HashMap<>();
for (MergedAnomalyResultDTO anomaly : input) {
if (anomaly.getId() != null && !anomaly.getChildren().isEmpty()) {
- existingParentAnomalies.put(anomaly.getId(), anomaly);
+ existingParentAnomalies.put(anomaly.getId(), copyAnomalyInfo(anomaly, new MergedAnomalyResultDTO()));
}
}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
index a5a5bda..cb78d8b 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/wrapper/ChildKeepingMergeWrapperTest.java
@@ -103,7 +103,8 @@ public class ChildKeepingMergeWrapperTest {
MockPipelineLoader mockLoader = new MockPipelineLoader(this.runs, this.outputs);
this.provider = new MockDataProvider()
- .setLoader(mockLoader);
+ .setLoader(mockLoader)
+ .setAnomalies(Collections.emptyList());
}
@Test
@@ -292,7 +293,7 @@ public class ChildKeepingMergeWrapperTest {
MockPipelineLoader mockLoader = new MockPipelineLoader(this.runs, Collections.singletonList(new MockPipelineOutput(Collections.singletonList(anomaly), -1L)));
DataProvider provider = new MockDataProvider()
- .setLoader(mockLoader);
+ .setLoader(mockLoader).setAnomalies(Collections.emptyList());
DetectionPipelineResult output = new ChildKeepingMergeWrapper(provider, config, 1000, 3000).run();
List<MergedAnomalyResultDTO> anomalyResults = output.getAnomalies();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org