You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/06/28 18:52:03 UTC
[incubator-pinot] branch master updated: [TE] Inject entity name
into grouper to enrich the grouped anomalies (#4378)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 29bad86 [TE] Inject entity name into grouper to enrich the grouped anomalies (#4378)
29bad86 is described below
commit 29bad867cb9c8165a1cd8d0fb116e8c5dc608c21
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Fri Jun 28 11:51:57 2019 -0700
[TE] Inject entity name into grouper to enrich the grouped anomalies (#4378)
* [TE} Inject entity name into grouper to enrich the grouped anomalies
* Fixed the tests
---
.../components/TriggerConditionGrouper.java | 6 +-
.../thirdeye/detection/wrapper/GrouperWrapper.java | 6 ++
.../yaml/translator/DetectionConfigTranslator.java | 31 +++---
.../components/TriggerConditionGrouperTest.java | 2 +-
.../compositePipelineTranslatorTestResult-4.json | 28 +++---
.../compositePipelineTranslatorTestResult-5.json | 110 ++++++++++-----------
6 files changed, 88 insertions(+), 95 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
index a68ae86..1ea5233 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
@@ -54,7 +54,7 @@ public class TriggerConditionGrouper implements Grouper<TriggerConditionGrouperS
private Map<String, Object> rightOp;
private InputDataFetcher dataFetcher;
- static final String PROP_DETECTOR_COMPONENT_NAME = "detectorComponentName";
+ static final String PROP_ENTITY_NAME = "subEntityName";
static final String PROP_AND = "and";
static final String PROP_OR = "or";
static final String PROP_OPERATOR = "operator";
@@ -143,8 +143,8 @@ public class TriggerConditionGrouper implements Grouper<TriggerConditionGrouperS
String value = MapUtils.getString(operatorNode, "value");
if (value != null) {
return anomalies.stream().filter(anomaly ->
- anomaly.getProperties() != null && anomaly.getProperties().get(PROP_DETECTOR_COMPONENT_NAME) != null
- && anomaly.getProperties().get(PROP_DETECTOR_COMPONENT_NAME).startsWith(value)
+ anomaly.getProperties() != null && anomaly.getProperties().get(PROP_ENTITY_NAME) != null
+ && anomaly.getProperties().get(PROP_ENTITY_NAME).startsWith(value)
).collect(Collectors.toList());
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
index 86134ab..f953ffb 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
@@ -48,11 +48,13 @@ public class GrouperWrapper extends DetectionPipeline {
private static final String PROP_CLASS_NAME = "className";
private static final String PROP_GROUPER = "grouper";
private static final String PROP_DETECTOR_COMPONENT_NAME = "detectorComponentName";
+ private static final String PROP_ENTITY_NAME = "subEntityName";
private final List<Map<String, Object>> nestedProperties;
private final Grouper grouper;
private final String grouperName;
+ private final String entityName;
public GrouperWrapper(DataProvider provider, DetectionConfigDTO config, long startTime, long endTime)
throws Exception {
@@ -65,6 +67,9 @@ public class GrouperWrapper extends DetectionPipeline {
this.grouperName = DetectionUtils.getComponentKey(MapUtils.getString(config.getProperties(), PROP_GROUPER));
Preconditions.checkArgument(this.config.getComponents().containsKey(this.grouperName));
this.grouper = (Grouper) this.config.getComponents().get(this.grouperName);
+ Preconditions.checkArgument(this.config.getProperties().containsKey(PROP_ENTITY_NAME));
+ this.entityName = this.config.getProperties().get(PROP_ENTITY_NAME).toString();
+
}
/**
@@ -110,6 +115,7 @@ public class GrouperWrapper extends DetectionPipeline {
anomaly.setDetectionConfigId(this.config.getId());
anomaly.getProperties().put(PROP_DETECTOR_COMPONENT_NAME, this.grouperName);
+ anomaly.getProperties().put(PROP_ENTITY_NAME, this.entityName);
}
return new DetectionPipelineResult(anomalies, DetectionUtils.consolidateNestedLastTimeStamps(lastTimeStamps),
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 693e2fa..b383284 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -21,12 +21,10 @@ package org.apache.pinot.thirdeye.detection.yaml.translator;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
-import java.time.Period;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -132,9 +130,8 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
private static final String PROP_FILTERS = "filters";
private static final String PROP_TYPE = "type";
private static final String PROP_CLASS_NAME = "className";
+ private static final String PROP_ENTITY_NAME = "subEntityName";
private static final String PROP_PARAMS = "params";
- private static final String PROP_METRIC = "metric";
- private static final String PROP_DATASET = "dataset";
private static final String PROP_METRIC_URN = "metricUrn";
private static final String PROP_DIMENSION_FILTER_METRIC = "dimensionFilterMetric";
private static final String PROP_NESTED_METRIC_URNS = "nestedMetricUrns";
@@ -226,15 +223,16 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
// Wrap with metric level grouper, restricting to only 1 grouper
List<Map<String, Object>> grouperYamls = getList(metricAlertConfigMap.get(PROP_GROUPER));
+ String subEntityName = MapUtils.getString(metricAlertConfigMap, PROP_ENTITY_NAME);
if (!grouperYamls.isEmpty()) {
- properties = buildGroupWrapperProperties(metricUrn, grouperYamls.get(0), Collections.singletonList(properties));
+ properties = buildGroupWrapperProperties(subEntityName, metricUrn, grouperYamls.get(0), Collections.singletonList(properties));
}
return properties;
}
private Map<String, Object> translateCompositeAlert(Map<String, Object> compositeAlertConfigMap) {
- Map<String, Object> properties;
+ Map<String, Object> properties = new HashMap<>();
// Recursively translate all the sub-alerts
List<Map<String, Object>> subDetectionYamls = ConfigUtils.getList(compositeAlertConfigMap.get(PROP_ALERTS));
@@ -252,20 +250,17 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
// Wrap the entity level grouper, only 1 grouper is supported now
List<Map<String, Object>> grouperProps = ConfigUtils.getList(compositeAlertConfigMap.get(PROP_GROUPER));
+ String subEntityName = MapUtils.getString(compositeAlertConfigMap, PROP_ENTITY_NAME);
if (!grouperProps.isEmpty()) {
- properties = buildGroupWrapperProperties(grouperProps.get(0), nestedPropertiesList);
- } else {
- Map<String, Object> defaultGrouper = new HashMap<>();
- defaultGrouper.put(PROP_TYPE, "MOCK_GROUPER");
- defaultGrouper.put(PROP_NAME, "Default grouper");
- properties = buildGroupWrapperProperties(defaultGrouper, nestedPropertiesList);
+ properties = buildGroupWrapperProperties(subEntityName, grouperProps.get(0), nestedPropertiesList);
+ nestedPropertiesList = Collections.singletonList(properties);
}
// Wrap the entity level merger
Map<String, Object> mergerProperties = ConfigUtils.getMap(compositeAlertConfigMap.get(PROP_MERGER));
properties = buildWrapperProperties(
ChildKeepingMergeWrapper.class.getName(),
- Collections.singletonList(properties),
+ nestedPropertiesList,
mergerProperties);
return properties;
@@ -360,14 +355,18 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
return properties;
}
- private Map<String, Object> buildGroupWrapperProperties(Map<String, Object> grouperYaml, List<Map<String, Object>> nestedProps) {
- return buildGroupWrapperProperties(null, grouperYaml, nestedProps);
+ private Map<String, Object> buildGroupWrapperProperties(String entityName, Map<String, Object> grouperYaml, List<Map<String, Object>> nestedProps) {
+ return buildGroupWrapperProperties(entityName, null, grouperYaml, nestedProps);
}
- private Map<String, Object> buildGroupWrapperProperties(String metricUrn, Map<String, Object> grouperYaml, List<Map<String, Object>> nestedProps) {
+ private Map<String, Object> buildGroupWrapperProperties(String entityName, String metricUrn,
+ Map<String, Object> grouperYaml, List<Map<String, Object>> nestedProps) {
Map<String, Object> properties = new HashMap<>();
properties.put(PROP_CLASS_NAME, GrouperWrapper.class.getName());
properties.put(PROP_NESTED, nestedProps);
+ if (entityName != null) {
+ properties.put(PROP_ENTITY_NAME, entityName);
+ }
String grouperType = MapUtils.getString(grouperYaml, PROP_TYPE);
String grouperName = MapUtils.getString(grouperYaml, PROP_NAME);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouperTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouperTest.java
index 19870f3..0df68bd 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouperTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouperTest.java
@@ -40,7 +40,7 @@ public class TriggerConditionGrouperTest {
public static MergedAnomalyResultDTO makeAnomaly(long start, long end, String entity) {
MergedAnomalyResultDTO anomaly = DetectionTestUtils.makeAnomaly(1000l, start, end, null, null, Collections.<String, String>emptyMap());
Map<String, String> props = new HashMap<>();
- props.put(PROP_DETECTOR_COMPONENT_NAME, entity);
+ props.put(PROP_ENTITY_NAME, entity);
anomaly.setProperties(props);
return anomaly;
}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-4.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-4.json
index 2cc7cce..9885a70 100644
--- a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-4.json
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-4.json
@@ -2,26 +2,22 @@
"properties": {
"className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
"nested": [{
- "grouper": "$Default grouper:MOCK_GROUPER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
"nested": [{
- "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nestedMetricUrns": ["thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"],
+ "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested": [{
- "nestedMetricUrns": ["thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"],
- "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
- "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
+ "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested": [{
- "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "nested": [{
- "bucketPeriod": "P1D",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
- }],
- "detector": "$maxThreshold_1:THRESHOLD"
+ "bucketPeriod": "P1D",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
}],
- "minContribution": 0.05,
- "dimensions": ["D1", "D2"]
- }]
+ "detector": "$maxThreshold_1:THRESHOLD"
+ }],
+ "minContribution": 0.05,
+ "dimensions": ["D1", "D2"]
}]
}]
},
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-5.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-5.json
index 2a84f5a..3ecc10e 100644
--- a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-5.json
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-5.json
@@ -2,83 +2,75 @@
"properties": {
"className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
"nested": [{
- "grouper": "$Default grouper:MOCK_GROUPER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
"nested": [{
- "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nestedMetricUrns": ["thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"],
+ "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested": [{
- "nestedMetricUrns": ["thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"],
- "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
- "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
+ "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [{
- "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
+ "filter": "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
"className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [{
- "filter": "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
- "nested": [{
- "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "nested": [{
- "bucketPeriod": "P1D",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
- }],
- "detector": "$maxThreshold_1:THRESHOLD"
- }]
- }]
- },
- {
- "baselineValueProvider": "$maxThreshold_2:THRESHOLD",
+ "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
"className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested": [{
"bucketPeriod": "P1D",
"className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
}],
- "detector": "$maxThreshold_2:THRESHOLD"
+ "detector": "$maxThreshold_1:THRESHOLD"
+ }]
+ }]
+ },
+ {
+ "baselineValueProvider": "$maxThreshold_2:THRESHOLD",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "nested": [{
+ "bucketPeriod": "P1D",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
}],
- "minContribution": 0.05,
- "dimensions": ["D1", "D2"]
- }]
- },
- {
+ "detector": "$maxThreshold_2:THRESHOLD"
+ }],
+ "minContribution": 0.05,
+ "dimensions": ["D1", "D2"]
+ }]
+ },
+ {
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nested": [{
"className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
"nested": [{
- "grouper": "$Default grouper:MOCK_GROUPER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper",
+ "nestedMetricUrns": ["thirdeye:metric:1"],
+ "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
"nested": [{
- "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested": [{
- "nestedMetricUrns": ["thirdeye:metric:1"],
- "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
- "nested": [{
- "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "nested": [{
- "bucketPeriod": "P1D",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
- }],
- "detector": "$maxThreshold_1:THRESHOLD"
- }]
- }]
- },
- {
- "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "bucketPeriod": "P1D",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }],
+ "detector": "$maxThreshold_1:THRESHOLD"
+ }]
+ }]
+ },
+ {
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nested": [{
+ "nestedMetricUrns": ["thirdeye:metric:1"],
+ "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "nested": [{
+ "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"nested": [{
- "nestedMetricUrns": ["thirdeye:metric:1"],
- "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
- "nested": [{
- "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "nested": [{
- "bucketPeriod": "P1D",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
- }],
- "detector": "$maxThreshold_1:THRESHOLD"
- }]
- }]
+ "bucketPeriod": "P1D",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }],
+ "detector": "$maxThreshold_1:THRESHOLD"
}]
+ }]
}]
- }]
}]
},
"components": {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org