You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/07/09 21:14:14 UTC
[incubator-pinot] branch master updated: [TE] Add Merger after
metric grouper; other minor clean up (#4410)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 7df1a43 [TE] Add Merger after metric grouper; other minor clean up (#4410)
7df1a43 is described below
commit 7df1a43ab04fee969ce2d9dc80a42800b6a3bb22
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Tue Jul 9 14:14:10 2019 -0700
[TE] Add Merger after metric grouper; other minor clean up (#4410)
---
.../detection/alert/DetectionAlertTaskFactory.java | 13 ++--
.../alert/scheme/DetectionEmailAlerter.java | 1 -
.../components/TriggerConditionGrouper.java | 2 +-
.../thirdeye/detection/wrapper/GrouperWrapper.java | 2 +-
.../yaml/translator/DetectionConfigTranslator.java | 13 ++--
.../translator/SubscriptionConfigTranslator.java | 3 -
.../thirdeye/detection/alert/SendAlertTest.java | 2 +-
.../compositePipelineTranslatorTestResult-1.json | 91 ++++++++++++----------
8 files changed, 68 insertions(+), 59 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java
index 0737ab3..539bc75 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/DetectionAlertTaskFactory.java
@@ -53,6 +53,7 @@ public class DetectionAlertTaskFactory {
private static final Logger LOG = LoggerFactory.getLogger(DetectionAlertTaskFactory.class);
private static final String PROP_CLASS_NAME = "className";
+ private static final String PROP_EMAIL_SCHEME = "emailScheme";
private static final String DEFAULT_ALERT_SCHEME = "org.apache.pinot.thirdeye.detection.alert.scheme.DetectionEmailAlerter";
private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance();
@@ -90,15 +91,15 @@ public class DetectionAlertTaskFactory {
Map<String, Map<String, Object>> alertSchemes = alertConfig.getAlertSchemes();
if (alertSchemes == null || alertSchemes.isEmpty()) {
Map<String, Object> emailScheme = new HashMap<>();
- emailScheme.put("className", DEFAULT_ALERT_SCHEME);
- alertSchemes = Collections.singletonMap("EmailScheme", emailScheme);
+ emailScheme.put(PROP_CLASS_NAME, DEFAULT_ALERT_SCHEME);
+ alertSchemes = Collections.singletonMap(PROP_EMAIL_SCHEME, emailScheme);
}
Set<DetectionAlertScheme> detectionAlertSchemeSet = new HashSet<>();
for (String alertSchemeType : alertSchemes.keySet()) {
LOG.debug("Loading Alert Scheme : {}", alertSchemeType);
Preconditions.checkNotNull(alertSchemes.get(alertSchemeType));
- Preconditions.checkNotNull(alertSchemes.get(alertSchemeType).get("className"));
- Constructor<?> constructor = Class.forName(alertSchemes.get(alertSchemeType).get("className").toString().trim())
+ Preconditions.checkNotNull(alertSchemes.get(alertSchemeType).get(PROP_CLASS_NAME));
+ Constructor<?> constructor = Class.forName(alertSchemes.get(alertSchemeType).get(PROP_CLASS_NAME).toString().trim())
.getConstructor(DetectionAlertConfigDTO.class, ThirdEyeAnomalyConfiguration.class, DetectionAlertFilterResult.class);
detectionAlertSchemeSet.add((DetectionAlertScheme) constructor.newInstance(alertConfig,
thirdeyeConfig, result));
@@ -117,8 +118,8 @@ public class DetectionAlertTaskFactory {
for (String alertSuppressor : alertSuppressors.keySet()) {
LOG.debug("Loading Alert Suppressor : {}", alertSuppressor);
Preconditions.checkNotNull(alertSuppressors.get(alertSuppressor));
- Preconditions.checkNotNull(alertSuppressors.get(alertSuppressor).get("className"));
- Constructor<?> constructor = Class.forName(alertSuppressors.get(alertSuppressor).get("className").toString().trim())
+ Preconditions.checkNotNull(alertSuppressors.get(alertSuppressor).get(PROP_CLASS_NAME));
+ Constructor<?> constructor = Class.forName(alertSuppressors.get(alertSuppressor).get(PROP_CLASS_NAME).toString().trim())
.getConstructor(DetectionAlertConfigDTO.class);
detectionAlertSuppressors.add((DetectionAlertSuppressor) constructor.newInstance(alertConfig));
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
index d95b0ad..588a7b4 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
@@ -39,7 +39,6 @@ import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterResult;
import org.apache.pinot.thirdeye.detection.annotation.AlertScheme;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
index 1ea5233..5815cec 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/components/TriggerConditionGrouper.java
@@ -54,7 +54,7 @@ public class TriggerConditionGrouper implements Grouper<TriggerConditionGrouperS
private Map<String, Object> rightOp;
private InputDataFetcher dataFetcher;
- static final String PROP_ENTITY_NAME = "subEntityName";
+ static final String PROP_ENTITY_NAME = "entityName";
static final String PROP_AND = "and";
static final String PROP_OR = "or";
static final String PROP_OPERATOR = "operator";
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
index f953ffb..7d38bba 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/GrouperWrapper.java
@@ -48,7 +48,7 @@ public class GrouperWrapper extends DetectionPipeline {
private static final String PROP_CLASS_NAME = "className";
private static final String PROP_GROUPER = "grouper";
private static final String PROP_DETECTOR_COMPONENT_NAME = "detectorComponentName";
- private static final String PROP_ENTITY_NAME = "subEntityName";
+ private static final String PROP_ENTITY_NAME = "entityName";
private final List<Map<String, Object>> nestedProperties;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index b383284..68a7e54 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -130,7 +130,7 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
private static final String PROP_FILTERS = "filters";
private static final String PROP_TYPE = "type";
private static final String PROP_CLASS_NAME = "className";
- private static final String PROP_ENTITY_NAME = "subEntityName";
+ private static final String PROP_ENTITY_NAME = "entityName";
private static final String PROP_PARAMS = "params";
private static final String PROP_METRIC_URN = "metricUrn";
private static final String PROP_DIMENSION_FILTER_METRIC = "dimensionFilterMetric";
@@ -223,16 +223,19 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
// Wrap with metric level grouper, restricting to only 1 grouper
List<Map<String, Object>> grouperYamls = getList(metricAlertConfigMap.get(PROP_GROUPER));
- String subEntityName = MapUtils.getString(metricAlertConfigMap, PROP_ENTITY_NAME);
+ String subEntityName = MapUtils.getString(metricAlertConfigMap, PROP_NAME);
if (!grouperYamls.isEmpty()) {
- properties = buildGroupWrapperProperties(subEntityName, metricUrn, grouperYamls.get(0), Collections.singletonList(properties));
+ properties = buildWrapperProperties(
+ ChildKeepingMergeWrapper.class.getName(),
+ Collections.singletonList(buildGroupWrapperProperties(subEntityName, metricUrn, grouperYamls.get(0), Collections.singletonList(properties))),
+ mergerProperties);
}
return properties;
}
private Map<String, Object> translateCompositeAlert(Map<String, Object> compositeAlertConfigMap) {
- Map<String, Object> properties = new HashMap<>();
+ Map<String, Object> properties;
// Recursively translate all the sub-alerts
List<Map<String, Object>> subDetectionYamls = ConfigUtils.getList(compositeAlertConfigMap.get(PROP_ALERTS));
@@ -250,7 +253,7 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
// Wrap the entity level grouper, only 1 grouper is supported now
List<Map<String, Object>> grouperProps = ConfigUtils.getList(compositeAlertConfigMap.get(PROP_GROUPER));
- String subEntityName = MapUtils.getString(compositeAlertConfigMap, PROP_ENTITY_NAME);
+ String subEntityName = MapUtils.getString(compositeAlertConfigMap, PROP_NAME);
if (!grouperProps.isEmpty()) {
properties = buildGroupWrapperProperties(subEntityName, grouperProps.get(0), nestedPropertiesList);
nestedPropertiesList = Collections.singletonList(properties);
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
index d7416e1..9f0bd62 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
@@ -38,9 +38,6 @@ import java.util.Map;
import java.util.Set;
import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.detection.validators.SubscriptionConfigValidator;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.Yaml;
-
/**
* The translator converts the alert yaml config into a detection alert config
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/SendAlertTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/SendAlertTest.java
index 7c4fd53..56ed049 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/SendAlertTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/SendAlertTest.java
@@ -119,7 +119,7 @@ public class SendAlertTest {
Map<String, Object> emailScheme = new HashMap<>();
emailScheme.put("className", "org.apache.pinot.thirdeye.detection.alert.scheme.RandomAlerter");
- this.alertConfigDTO.setAlertSchemes(Collections.singletonMap("EmailScheme", emailScheme));
+ this.alertConfigDTO.setAlertSchemes(Collections.singletonMap("emailScheme", emailScheme));
this.alertConfigDTO.setProperties(properties);
this.alertConfigDTO.setFrom(FROM_ADDRESS_VALUE);
this.alertConfigDTO.setName(ALERT_NAME_VALUE);
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
index 31d11a9..6aa2bbf 100644
--- a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/translator/compositePipelineTranslatorTestResult-1.json
@@ -1,29 +1,55 @@
{
"properties": {
- "className": "org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper",
- "grouper": "$test_grouper:MOCK_GROUPER",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "maxGap": 0,
"nested": [
{
- "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
- "maxGap": 0,
+ "grouper": "$test_grouper:MOCK_GROUPER",
+ "entityName": "testPipeline",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.GrouperWrapper",
"nested": [
{
- "nestedMetricUrns": [
- "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"
- ],
- "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
- "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "maxGap": 0,
"nested": [
{
- "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
+ "nestedMetricUrns": [
+ "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3"
+ ],
+ "className": "org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "metricUrn": "thirdeye:metric:1:D1%3Dv1:D1%3Dv2:D2%3Dv3",
"nested": [
{
- "filter": "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+ "filter": "$thresholdFilter_2:THRESHOLD_RULE_FILTER",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
+ "nested": [
+ {
+ "filter": "$thresholdFilter_1:THRESHOLD_RULE_FILTER",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
+ "nested": [
+ {
+ "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "maxGap": 0,
+ "nested": [
+ {
+ "bucketPeriod": "P1D",
+ "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }
+ ],
+ "detector": "$maxThreshold_1:THRESHOLD",
+ "maxDuration": 100000000
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "filter": "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
"className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
"nested": [
{
- "baselineValueProvider": "$maxThreshold_1:THRESHOLD",
+ "baselineValueProvider": "$maxThreshold_2:THRESHOLD",
"className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
"maxGap": 0,
"nested": [
@@ -32,42 +58,25 @@
"className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
}
],
- "detector": "$maxThreshold_1:THRESHOLD",
+ "detector": "$maxThreshold_2:THRESHOLD",
"maxDuration": 100000000
}
]
}
- ]
- },
- {
- "filter": "$thresholdFilter_3:THRESHOLD_RULE_FILTER",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper",
- "nested": [
- {
- "baselineValueProvider": "$maxThreshold_2:THRESHOLD",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
- "maxGap": 0,
- "nested": [
- {
- "bucketPeriod": "P1D",
- "className": "org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
- }
- ],
- "detector": "$maxThreshold_2:THRESHOLD",
- "maxDuration": 100000000
- }
+ ],
+ "minContribution": 0.05,
+ "dimensions": [
+ "D1",
+ "D2"
]
}
],
- "minContribution": 0.05,
- "dimensions": [
- "D1",
- "D2"
- ]
+ "maxDuration": 100000000
}
- ],
- "maxDuration": 100000000
- }]
+ ]
+ }
+ ],
+ "maxDuration": 100000000
},
"components": {
"maxThreshold_2:THRESHOLD": {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org