You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/04/16 22:14:27 UTC
[incubator-pinot] branch master updated: [TE] Remove migrator,
partial reference and legacy alert (#4124)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new dab44a6 [TE] Remove migrator, partial reference and legacy alert (#4124)
dab44a6 is described below
commit dab44a696cea604f0e369d88ebb241a47da1ab2e
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Tue Apr 16 15:14:22 2019 -0700
[TE] Remove migrator, partial reference and legacy alert (#4124)
---
.../dashboard/ThirdEyeDashboardApplication.java | 6 -
.../datalayer/pojo/DetectionAlertConfigBean.java | 14 +-
.../datasource/sql/SqlResponseCacheLoader.java | 2 +-
.../sql/resources/SqlDataSourceResource.java | 2 +-
.../pinot/thirdeye/detection/DataProvider.java | 12 -
.../thirdeye/detection/DefaultDataProvider.java | 5 -
.../detection/DetectionMigrationResource.java | 730 ---------------------
.../alert/StatefulDetectionAlertFilter.java | 6 +-
.../detection/alert/filter/LegacyAlertFilter.java | 143 ----
.../yaml/YamlDetectionAlertConfigTranslator.java | 3 -
.../thirdeye/detection/yaml/YamlResource.java | 1 -
.../detection/DetectionMigrationResourceTest.java | 294 ---------
.../pinot/thirdeye/detection/MockDataProvider.java | 6 -
.../alert/filter/LegacyAlertFilterTest.java | 110 ----
.../ToAllRecipientsDetectionAlertFilterTest.java | 10 -
.../YamlDetectionAlertConfigTranslatorTest.java | 1 -
16 files changed, 5 insertions(+), 1340 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index 5c7357d..76b5403 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -71,7 +71,6 @@ import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
import org.apache.pinot.thirdeye.datasource.sql.resources.SqlDataSourceResource;
-import org.apache.pinot.thirdeye.detection.DetectionMigrationResource;
import org.apache.pinot.thirdeye.detection.DetectionResource;
import org.apache.pinot.thirdeye.detection.annotation.DetectionConfigurationResource;
import org.apache.pinot.thirdeye.detection.yaml.YamlResource;
@@ -173,11 +172,6 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new ThirdEyeResource());
env.jersey().register(new DataResource(anomalyFunctionFactory, alertFilterFactory));
env.jersey().register(new AnomaliesResource(anomalyFunctionFactory, alertFilterFactory));
- env.jersey().register(new DetectionMigrationResource(
- DAO_REGISTRY.getAnomalyFunctionDAO(), DAO_REGISTRY.getAlertConfigDAO(), DAO_REGISTRY.getApplicationDAO(),
- DAO_REGISTRY.getMetricConfigDAO(), DAO_REGISTRY.getDetectionConfigManager(),
- DAO_REGISTRY.getDetectionAlertConfigManager(), DAO_REGISTRY.getDatasetConfigDAO(),
- DAO_REGISTRY.getMergedAnomalyResultDAO()));
env.jersey().register(new OnboardResource(config));
env.jersey().register(new EntityMappingResource());
env.jersey().register(new OnboardDatasetMetricResource());
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java
index d05e624..8bbda0e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/DetectionAlertConfigBean.java
@@ -40,7 +40,6 @@ public class DetectionAlertConfigBean extends AbstractBean {
String cronExpression;
String application;
String yaml;
- boolean onlyFetchLegacyAnomalies;
Map<String, Map<String, Object>> alertSchemes;
Map<String, Map<String, Object>> alertSuppressors;
@@ -53,14 +52,6 @@ public class DetectionAlertConfigBean extends AbstractBean {
Map<String, String> refLinks;
- public boolean isOnlyFetchLegacyAnomalies() {
- return onlyFetchLegacyAnomalies;
- }
-
- public void setOnlyFetchLegacyAnomalies(boolean onlyFetchLegacyAnomalies) {
- this.onlyFetchLegacyAnomalies = onlyFetchLegacyAnomalies;
- }
-
public boolean isActive() {
return active;
}
@@ -179,13 +170,12 @@ public class DetectionAlertConfigBean extends AbstractBean {
&& subjectType == that.subjectType && Objects.equals(vectorClocks, that.vectorClocks) && Objects.equals(
highWaterMark, that.highWaterMark) && Objects.equals(properties, that.properties)
&& Objects.equals(alertSchemes, that.alertSchemes) && Objects.equals(alertSuppressors, that.alertSuppressors)
- && Objects.equals(refLinks, that.refLinks) && onlyFetchLegacyAnomalies == that.onlyFetchLegacyAnomalies
- && Objects.equals(yaml, that.yaml);
+ && Objects.equals(refLinks, that.refLinks) && Objects.equals(yaml, that.yaml);
}
@Override
public int hashCode() {
return Objects.hash(active, name, from, cronExpression, application, subjectType, vectorClocks,
- highWaterMark, properties, alertSchemes, alertSuppressors, refLinks, onlyFetchLegacyAnomalies, yaml);
+ highWaterMark, properties, alertSchemes, alertSuppressors, refLinks, yaml);
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
index ad4e61a..9f1953b 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlResponseCacheLoader.java
@@ -19,7 +19,7 @@
package org.apache.pinot.thirdeye.datasource.sql;
-import com.facebook.presto.jdbc.internal.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.cache.CacheLoader;
import java.io.File;
import java.sql.Connection;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java
index fba4b0a..4a61a86 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/resources/SqlDataSourceResource.java
@@ -1,6 +1,6 @@
package org.apache.pinot.thirdeye.datasource.sql.resources;
-import com.facebook.presto.jdbc.internal.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectMapper;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java
index d51187b..4d80598 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DataProvider.java
@@ -69,18 +69,6 @@ public interface DataProvider {
Map<MetricSlice, DataFrame> fetchAggregates(Collection<MetricSlice> slices, List<String> dimensions);
/**
- * Returns a multimap of anomalies (keyed by slice) generated by the legacy detection pipeline.
- *
- * @see MergedAnomalyResultDTO
- * @see AnomalySlice
- *
- * @param slices anomaly slice
- * @param configId configId
- * @return multimap of anomalies (keyed by slice)
- */
- Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices, long configId);
-
- /**
* Returns a multimap of anomalies (keyed by slice) for a given set of slices.
*
* @see MergedAnomalyResultDTO
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
index 0a2e200..a512974 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
@@ -247,11 +247,6 @@ public class DefaultDataProvider implements DataProvider {
}
@Override
- public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices, long configId) {
- return fetchAnomalies(slices, configId, true);
- }
-
- @Override
public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchAnomalies(Collection<AnomalySlice> slices, long configId) {
return fetchAnomalies(slices, configId, false);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
deleted file mode 100644
index 08fe6ae..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
+++ /dev/null
@@ -1,730 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Multimap;
-import com.wordnik.swagger.annotations.Api;
-import com.wordnik.swagger.annotations.ApiOperation;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.xml.bind.ValidationException;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.thirdeye.anomaly.detection.AnomalyDetectionInputContextBuilder;
-import org.apache.pinot.thirdeye.api.Constants;
-import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.util.Predicate;
-import org.apache.pinot.thirdeye.detection.alert.filter.ToAllRecipientsDetectionAlertFilter;
-import org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator;
-import org.apache.pinot.thirdeye.detection.yaml.YamlResource;
-import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.Yaml;
-
-import static org.apache.pinot.thirdeye.anomaly.merge.AnomalyMergeStrategy.*;
-import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
-
-
-/**
- * The Detection migration resource.
- */
-@Path("/migrate")
-@Api(tags = {Constants.YAML_TAG})
-public class DetectionMigrationResource {
- private static final Logger LOGGER = LoggerFactory.getLogger(DetectionMigrationResource.class);
- private static final String PROP_WINDOW_DELAY = "windowDelay";
- private static final String PROP_WINDOW_DELAY_UNIT = "windowDelayUnit";
- private static final String PROP_WINDOW_SIZE = "windowSize";
- private static final String PROP_WINDOW_UNIT = "windowUnit";
-
- static final String MIGRATED_TAG = "_thirdeye_migrated";
-
- private final AnomalyFunctionManager anomalyFunctionDAO;
- private final DetectionConfigManager detectionConfigDAO;
- private final DetectionAlertConfigManager detectionAlertConfigDAO;
- private final DatasetConfigManager datasetConfigDAO;
- private final MergedAnomalyResultManager mergedAnomalyResultDAO;
- private final AlertConfigManager alertConfigDAO;
- private final ApplicationManager appDAO;
- private final MetricConfigManager metricConfigDAO;
- private final Yaml yaml;
-
- /**
- * Instantiates a new Detection migration resource.
- */
- public DetectionMigrationResource(
- AnomalyFunctionManager anomalyFunctionDAO,
- AlertConfigManager alertConfigDAO,
- ApplicationManager appDAO,
- MetricConfigManager metricConfigDAO,
- DetectionConfigManager detectionConfigDAO,
- DetectionAlertConfigManager detectionAlertConfigDAO,
- DatasetConfigManager datasetConfigDAO,
- MergedAnomalyResultManager mergedAnomalyResultDAO) {
- this.anomalyFunctionDAO = anomalyFunctionDAO;
- this.detectionConfigDAO = detectionConfigDAO;
- this.detectionAlertConfigDAO = detectionAlertConfigDAO;
- this.alertConfigDAO = alertConfigDAO;
- this.appDAO = appDAO;
- this.metricConfigDAO = metricConfigDAO;
- this.datasetConfigDAO = datasetConfigDAO;
- this.mergedAnomalyResultDAO = mergedAnomalyResultDAO;
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
- options.setPrettyFlow(true);
- this.yaml = new Yaml(options);
- }
-
- private Map<String, Object> translateAnomalyFunctionToYaml(AnomalyFunctionDTO anomalyFunctionDTO) {
- Map<String, Object> yamlConfigs = new LinkedHashMap<>();
- yamlConfigs.put("detectionName", anomalyFunctionDTO.getFunctionName());
- yamlConfigs.put("description", "Please update description - If this alert fires then it means so-and-so and check so-and-so for irregularities");
- yamlConfigs.put("metric", anomalyFunctionDTO.getMetric());
- yamlConfigs.put("active", anomalyFunctionDTO.getIsActive());
- yamlConfigs.put("dataset", anomalyFunctionDTO.getCollection());
- yamlConfigs.put("pipelineType", "Composite");
- if (StringUtils.isNotBlank(anomalyFunctionDTO.getExploreDimensions())) {
- // dimension explore and data filter
- yamlConfigs.put("dimensionExploration",
- getDimensionExplorationParams(anomalyFunctionDTO));
- }
- if (anomalyFunctionDTO.getFilters() != null){
- yamlConfigs.put("filters",
- AnomalyDetectionInputContextBuilder.getFiltersForFunction(anomalyFunctionDTO.getFilters()).asMap());
- }
-
- Map<String, Object> ruleYaml = new LinkedHashMap<>();
-
- // detection
- if (anomalyFunctionDTO.getType().equals("WEEK_OVER_WEEK_RULE")){
- // wo1w change detector
- ruleYaml.put("detection", Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type", "PERCENTAGE_RULE",
- "params", getPercentageChangeRuleDetectorParams(anomalyFunctionDTO))));
- } else if (anomalyFunctionDTO.getType().equals("MIN_MAX_THRESHOLD")){
- // threshold detector
- ruleYaml.put("detection", Collections.singletonList(ImmutableMap.of("name", "detection_rule1", "type", "THRESHOLD",
- "params", getMinMaxThresholdRuleDetectorParams(anomalyFunctionDTO))));
- } else{
- // algorithm detector
- Map<String, Object> detectionProperties = new HashMap<>();
- if (anomalyFunctionDTO.getWindowDelay() != 0) {
- detectionProperties.put(PROP_WINDOW_DELAY, anomalyFunctionDTO.getWindowDelay());
- detectionProperties.put(PROP_WINDOW_DELAY_UNIT, anomalyFunctionDTO.getWindowDelayUnit().toString());
- }
- detectionProperties.put("name", "detection_rule1");
- detectionProperties.put("type", "MIGRATED_ALGORITHM");
- detectionProperties.put("params", getAlgorithmDetectorParams(anomalyFunctionDTO));
- detectionProperties.put(PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize());
- detectionProperties.put(PROP_WINDOW_UNIT, anomalyFunctionDTO.getWindowUnit().toString());
- detectionProperties.put("bucketPeriod", getBucketPeriod(anomalyFunctionDTO));
-
- ruleYaml.put("detection", Collections.singletonList(detectionProperties));
- }
-
- // filters
- Map<String, String> alertFilter = anomalyFunctionDTO.getAlertFilter();
-
- if (alertFilter != null && !alertFilter.isEmpty()){
- Map<String, Object> filterYaml = new LinkedHashMap<>();
- if (!alertFilter.containsKey("thresholdField")) {
- // algorithm alert filter
- filterYaml = ImmutableMap.of("name", "filter_rule1", "type", "MIGRATED_ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
- } else {
- // threshold filter migrate to rule filters
- // site wide impact filter migrate to rule based swi filter
- if (anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("impactToGlobal")){
- filterYaml.put("type", "SITEWIDE_IMPACT_FILTER");
- filterYaml.put("name", "filter_rule1");
- filterYaml.put("params", getSiteWideImpactFilterParams(anomalyFunctionDTO));
- }
- // weight filter migrate to rule based percentage change filter
- if (anomalyFunctionDTO.getAlertFilter().get("thresholdField").equals("weight")){
- filterYaml.put("name", "filter_rule1");
- filterYaml.put("type", "PERCENTAGE_CHANGE_FILTER");
- filterYaml.put("params", getPercentageChangeFilterParams(anomalyFunctionDTO));
- }
- }
- ruleYaml.put("filter", Collections.singletonList(filterYaml));
- }
-
- yamlConfigs.put("rules", Collections.singletonList(ruleYaml));
-
- // merger configs
- if (anomalyFunctionDTO.getAnomalyMergeConfig() != null ) {
- Map<String, Object> mergerYaml = new LinkedHashMap<>();
- if (anomalyFunctionDTO.getAnomalyMergeConfig().getMergeStrategy() == FUNCTION_DIMENSIONS){
- mergerYaml.put("maxGap", anomalyFunctionDTO.getAnomalyMergeConfig().getSequentialAllowedGap());
- mergerYaml.put("maxDuration", anomalyFunctionDTO.getAnomalyMergeConfig().getMaxMergeDurationLength());
- }
- yamlConfigs.put("merger", mergerYaml);
- }
-
- return yamlConfigs;
- }
-
- private Map<String, Object> getDimensionExplorationParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> dimensionExploreYaml = new LinkedHashMap<>();
- dimensionExploreYaml.put("dimensions", Collections.singletonList(functionDTO.getExploreDimensions()));
- if (functionDTO.getDataFilter() != null && !functionDTO.getDataFilter().isEmpty() && functionDTO.getDataFilter().get("type").equals("average_threshold")) {
- // migrate average threshold data filter
- dimensionExploreYaml.put("dimensionFilterMetric", functionDTO.getDataFilter().get("metricName"));
- dimensionExploreYaml.put("minValue", Double.valueOf(functionDTO.getDataFilter().get("threshold")));
- dimensionExploreYaml.put("minLiveZone", functionDTO.getDataFilter().get("minLiveZone"));
- }
- if (functionDTO.getType().equals("MIN_MAX_THRESHOLD")){
- // migrate volume threshold
- Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties());
- if (properties.containsKey("averageVolumeThreshold")){
- dimensionExploreYaml.put("minValue", properties.getProperty("averageVolumeThreshold"));
- }
- }
- return dimensionExploreYaml;
- }
-
- private Map<String, Object> getPercentageChangeFilterParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> filterYamlParams = new LinkedHashMap<>();
- filterYamlParams.put("threshold", Math.abs(Double.valueOf(functionDTO.getAlertFilter().get("maxThreshold"))));
- filterYamlParams.put("pattern", "up_or_down");
- return filterYamlParams;
- }
-
- private Map<String, Object> getSiteWideImpactFilterParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> filterYamlParams = new LinkedHashMap<>();
- filterYamlParams.put("threshold", Math.abs(Double.valueOf(functionDTO.getAlertFilter().get("maxThreshold"))));
- filterYamlParams.put("pattern", "up_or_down");
- filterYamlParams.put("sitewideMetricName", functionDTO.getGlobalMetric());
- filterYamlParams.put("sitewideCollection", functionDTO.getCollection());
- if (StringUtils.isNotBlank(functionDTO.getGlobalMetricFilters())) {
- filterYamlParams.put("filters",
- AnomalyDetectionInputContextBuilder.getFiltersForFunction(functionDTO.getGlobalMetricFilters()).asMap());
- }
- return filterYamlParams;
- }
-
- private Map<String, Object> getAlertFilterParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> filterYamlParams = new LinkedHashMap<>();
- Map<String, Object> params = new HashMap<>();
- filterYamlParams.put("configuration", params);
- params.putAll(functionDTO.getAlertFilter());
- params.put("bucketPeriod", getBucketPeriod(functionDTO));
- params.put("timeZone", getTimezone(functionDTO));
- return filterYamlParams;
- }
-
- private String getTimezone(AnomalyFunctionDTO functionDTO) {
- DatasetConfigDTO datasetConfigDTO = this.datasetConfigDAO.findByDataset(functionDTO.getCollection());
- return datasetConfigDTO.getTimezone();
- }
-
- private String getBucketPeriod(AnomalyFunctionDTO functionDTO) {
- if (functionDTO.getBucketUnit().equals(TimeUnit.DAYS)){
- return Period.days(functionDTO.getBucketSize()).toString();
- }
- return new Period(TimeUnit.MILLISECONDS.convert(functionDTO.getBucketSize(), functionDTO.getBucketUnit())).toString();
- }
-
- private Map<String, Object> getPercentageChangeRuleDetectorParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> detectorYaml = new LinkedHashMap<>();
- Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties());
- double threshold = Double.valueOf(properties.getProperty("changeThreshold"));
- if (properties.containsKey("changeThreshold")){
- detectorYaml.put("percentageChange", Math.abs(threshold));
- if (threshold > 0){
- detectorYaml.put("pattern", "UP");
- } else {
- detectorYaml.put("pattern", "DOWN");
- }
- }
- return detectorYaml;
- }
-
- private Map<String, Object> getMinMaxThresholdRuleDetectorParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> detectorYaml = new LinkedHashMap<>();
- Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties());
- if (properties.containsKey("min")){
- detectorYaml.put("min", properties.getProperty("min"));
- }
- if (properties.containsKey("max")){
- detectorYaml.put("max", properties.getProperty("max"));
- }
- return detectorYaml;
- }
-
- private Map<String, Object> getAlgorithmDetectorParams(AnomalyFunctionDTO functionDTO) {
- Map<String, Object> detectorYaml = new LinkedHashMap<>();
- Map<String, Object> params = new LinkedHashMap<>();
- detectorYaml.put("configuration", params);
- Properties properties = AnomalyFunctionDTO.toProperties(functionDTO.getProperties());
- for (Map.Entry<Object, Object> property : properties.entrySet()) {
- params.put((String) property.getKey(), property.getValue());
- }
- params.put("variables.bucketPeriod", getBucketPeriod(functionDTO));
- params.put("variables.timeZone", getTimezone(functionDTO));
- return detectorYaml;
- }
-
- long migrateLegacyAnomalyFunction(long anomalyFunctionId) throws ValidationException {
- AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionId);
-
- // Verify if function is still valid
- validateFunction(anomalyFunctionDTO);
-
- return migrateLegacyAnomalyFunction(anomalyFunctionDTO);
- }
-
- private long migrateLegacyAnomalyFunction(AnomalyFunctionDTO anomalyFunctionDTO) {
- DetectionConfigDTO detectionConfig;
-
- LOGGER.info(String.format("[MIG] Migrating anomaly function %d %s", anomalyFunctionDTO.getId(),
- anomalyFunctionDTO.getFunctionName()));
-
- // Check if this anomaly function is already migrated
- if (anomalyFunctionDTO.getFunctionName().contains(MIGRATED_TAG)) {
- LOGGER.info(String.format("[MIG] Anomaly function %d is already migrated.", anomalyFunctionDTO.getId()));
-
- // Fetch the migrated config id and return
- String funcName = anomalyFunctionDTO.getFunctionName();
- return Long.parseLong(funcName.substring(funcName.lastIndexOf("_") + 1, funcName.length()));
- }
-
- // Migrate anomaly function config to the detection config by converting to YAML and then to Detection Config
- try {
- Map<String, Object> detectionYAMLMap = translateAnomalyFunctionToYaml(anomalyFunctionDTO);
- detectionConfig = new YamlResource().translateToDetectionConfig(detectionYAMLMap);
- Preconditions.checkNotNull(detectionConfig);
- } catch (Exception e) {
- throw new RuntimeException("Error translating anomaly function config to the detection config" + e.getMessage());
- }
-
- // Save the migrated anomaly function
- detectionConfigDAO.save(detectionConfig);
- if (detectionConfig.getId() == null) {
- throw new RuntimeException("Error saving the new detection config.");
- }
-
- // Hack to retain Anomaly function owner
- detectionConfig.setCreatedBy(anomalyFunctionDTO.getCreatedBy());
- detectionConfigDAO.update(detectionConfig);
- if (detectionConfig.getId() == null) {
- throw new RuntimeException("Error saving the new detection config after updating the owner.");
- }
-
- // Point all the associated anomalies to the migrated anomaly function.
- List<MergedAnomalyResultDTO> mergedAnomalyResultDTOS = mergedAnomalyResultDAO.findByPredicate(Predicate.EQ("functionId", anomalyFunctionDTO.getId()));
- for (MergedAnomalyResultDTO anomaly : mergedAnomalyResultDTOS) {
- // Drop the baseline and current values from the anomalies.
- if (anomaly.getProperties() != null) {
- anomaly.getProperties().remove("anomalyTimelinesView");
- }
- anomaly.setMetricUrn(buildMetricUrn(anomaly));
- anomaly.setDetectionConfigId(detectionConfig.getId());
- int affectedRows = mergedAnomalyResultDAO.update(anomaly);
- if (affectedRows == 0) {
- throw new RuntimeException("Failed to update the anomaly " + anomaly.getId() + " with the new detection id"
- + " for anomaly function " + detectionConfig.getId());
- }
- }
-
- // Mark the old anomaly function as migrated
- anomalyFunctionDTO.setActive(false);
- anomalyFunctionDTO.setFunctionName(anomalyFunctionDTO.getFunctionName() + MIGRATED_TAG + "_" + detectionConfig.getId());
- int affectedRows = this.anomalyFunctionDAO.update(anomalyFunctionDTO);
- if (affectedRows == 0) {
- throw new RuntimeException("Anomaly function migrated successfully but failed to disable and update the"
- + " migration status of the old anomaly function. Recommend doing it manually. Migrated detection id "
- + detectionConfig.getId());
- }
-
- LOGGER.info(String.format("[MIG] Successfully migrated anomaly function %d %s", anomalyFunctionDTO.getId(),
- anomalyFunctionDTO.getFunctionName()));
- return detectionConfig.getId();
- }
-
- private String buildMetricUrn(MergedAnomalyResultDTO anomaly) {
- try {
- DimensionMap dimensionMap = anomaly.getDimensions();
- Multimap<String, String> filters = ArrayListMultimap.create();
- for (DimensionMap.Entry<String, String> entry : dimensionMap.entrySet()) {
- filters.put(entry.getKey(), entry.getValue());
- }
- MetricEntity me = MetricEntity.fromMetric(1.0, metricConfigDAO.findByMetricAndDataset(anomaly.getMetric(), anomaly.getCollection()).getId(), filters);
- return me.getUrn();
- } catch (Exception e) {
- throw new RuntimeException("Resolve metric urn failed for anomaly " + anomaly.getId(), e);
- }
- }
-
-
- private void migrateLegacyNotification(AlertConfigDTO alertConfigDTO) {
- int anomalyFailureCount = 0;
- int anomalyWarningCount = 0;
- String alertName = alertConfigDTO.getName();
-
- LOGGER.info(String.format("[MIG] Migrating alert %d %s", alertConfigDTO.getId(), alertName));
-
- // Skip if the alert is already migrated
- if (alertConfigDTO.getName().contains(MIGRATED_TAG)) {
- LOGGER.info(String.format("[MIG] Alert %d is already migrated. Skipping!", alertConfigDTO.getId()));
- return;
- }
-
- // Migrate all the subscribed anomaly functions. Note that this will update the state of old anomaly functions.
- List<Long> detectionIds = ConfigUtils.getLongs(alertConfigDTO.getEmailConfig().getFunctionIds());
- List<Long> filteredIds = new ArrayList<>();
- for (long detectionId : detectionIds) {
- try {
- migrateLegacyAnomalyFunction(detectionId);
- filteredIds.add(detectionId);
- } catch (ValidationException e) {
- anomalyWarningCount++;
- // Ignore those anomaly functions which are pointing to invalid entities
- LOGGER.warn("[MIG] Validation error while migrating anomaly function {}. Error ", detectionId, e.getMessage());
- } catch (Exception e) {
- anomalyFailureCount++;
- LOGGER.error("[MIG] Error while migrating anomaly function {}. Error ", detectionId, e);
- }
- }
- alertConfigDTO.getEmailConfig().setFunctionIds(filteredIds);
-
- // Translate the old alert and capture the state.
- Map<String, Object> detectionAlertYaml = translateAlertToYaml(alertConfigDTO);
-
- // Migrate the alert/notification group
- DetectionAlertConfigDTO alertConfig = new YamlDetectionAlertConfigTranslator(detectionConfigDAO).translate(detectionAlertYaml);
- List<DetectionAlertConfigDTO> alertDTOs = detectionAlertConfigDAO.findByPredicate(Predicate.EQ("name", alertConfig.getName()));
- if (!alertDTOs.isEmpty()) {
- LOGGER.warn("[MIG] Looks like this alert was already migrated. old id = " + alertConfig.getId() + " new id = "
- + alertDTOs.get(0).getId());
- } else {
- detectionAlertConfigDAO.save(alertConfig);
- if (alertConfig.getId() == null) {
- throw new RuntimeException("Error while saving the migrated alert config for " + alertName);
- }
-
- // Hack to retain subscription group owner
- alertConfig.setCreatedBy(alertConfigDTO.getCreatedBy());
- detectionAlertConfigDAO.update(alertConfig);
- if (alertConfig.getId() == null) {
- throw new RuntimeException("Error saving the migrated alert config after updating the owner.");
- }
- }
-
- // Update migration status and disable the old alert
- alertConfigDTO.setName(alertName + MIGRATED_TAG + "_" + alertConfig.getId());
- alertConfigDTO.setActive(false);
- int affectedRows = alertConfigDAO.update(alertConfigDTO);
- if (affectedRows == 0) {
- throw new RuntimeException(
- "Alert migrated successfully but failed to disable and update the migration status" + " of the old alert."
- + " Migrated alert id " + alertConfig.getId());
- }
-
- if (anomalyFailureCount == 0 && anomalyWarningCount == 0) {
- LOGGER.info(String.format("[MIG] Successfully migrated alert %d %s", alertConfigDTO.getId(), alertName));
- } else {
- throw new RuntimeException("Failures/Warnings found. anomalyFailureCount " + anomalyFailureCount + " and"
- + " anomalyWarningCount " + anomalyWarningCount);
- }
- }
-
- private void validateFunction(AnomalyFunctionDTO functionDTO) throws ValidationException {
- if (functionDTO == null) {
- throw new ValidationException("Couldn't find anomaly function.");
- }
-
- List<DatasetConfigDTO> datasetConfigDTOs = this.datasetConfigDAO.findByPredicate(Predicate.EQ("dataset", functionDTO.getCollection()));
- if (datasetConfigDTOs.isEmpty()) {
- throw new ValidationException("Dataset cannot be found for anomaly function " + functionDTO.getId());
- }
-
- List<MetricConfigDTO> metricConfigDTOs = this.metricConfigDAO.findByPredicate(Predicate.AND(
- Predicate.EQ("name", functionDTO.getMetric()),
- Predicate.EQ("dataset", functionDTO.getCollection())));
- if (metricConfigDTOs.isEmpty()) {
- throw new ValidationException("Metric cannot be found for anomaly function " + functionDTO.getId());
- }
- }
-
- Map<String, Object> translateAlertToYaml(AlertConfigDTO alertConfigDTO) {
- Map<String, Object> yamlConfigs = new LinkedHashMap<>();
-
- yamlConfigs.put(PROP_SUBS_GROUP_NAME, alertConfigDTO.getName());
- yamlConfigs.put(PROP_CRON, alertConfigDTO.getCronExpression());
- yamlConfigs.put(PROP_ACTIVE, alertConfigDTO.isActive());
- yamlConfigs.put(PROP_APPLICATION, alertConfigDTO.getApplication());
- yamlConfigs.put(PROP_EMAIL_SUBJECT_TYPE, alertConfigDTO.getSubjectType().name());
- yamlConfigs.put(PROP_FROM, alertConfigDTO.getFromAddress());
-
- yamlConfigs.put(PROP_TYPE, "DEFAULT_ALERTER_PIPELINE");
-
- Map<String, Object> recipients = new LinkedHashMap<>();
- recipients.put("to", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getTo()));
- recipients.put("cc", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getCc()));
- recipients.put("bcc", ConfigUtils.getList(alertConfigDTO.getReceiverAddresses().getBcc()));
- yamlConfigs.put(PROP_RECIPIENTS, recipients);
-
- List<Map<String, Object>> schemes = new ArrayList<>();
- Map<String, Object> emailScheme = new LinkedHashMap<>();
- emailScheme.put(PROP_TYPE, "EMAIL");
- schemes.add(emailScheme);
- yamlConfigs.put(PROP_ALERT_SCHEMES, schemes);
-
- List<String> detectionNames = new ArrayList<>();
- List<Long> detectionIds = alertConfigDTO.getEmailConfig().getFunctionIds();
- for (Long id : detectionIds) {
- List<AnomalyFunctionDTO> functionDTOS = this.anomalyFunctionDAO.findByPredicate(Predicate.EQ("baseId", id));
- if (functionDTOS.isEmpty()) {
- LOGGER.warn("[MIG] Anomaly function " + id + " cannot be found while generating notification YAML from legacy notification.");
- // Ignore missing anomaly functions
- continue;
- }
-
- String functionName = functionDTOS.get(0).getFunctionName();
- if (functionName.contains(MIGRATED_TAG)) {
- functionName = functionName.substring(0, functionName.lastIndexOf(MIGRATED_TAG));
- }
- detectionNames.add(functionName);
- }
- yamlConfigs.put(PROP_DETECTION_NAMES, detectionNames);
-
- return yamlConfigs;
- }
-
- @GET
- @Produces(MediaType.TEXT_PLAIN)
- @Consumes(MediaType.APPLICATION_JSON)
- @ApiOperation("migrate a function")
- @Path("/legacy-anomaly-function-to-yaml/{id}")
- public Response getYamlFromLegacyAnomalyFunction(@PathParam("id") long anomalyFunctionID) {
- AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionID);
- if (anomalyFunctionDTO == null) {
- return Response.status(Response.Status.BAD_REQUEST)
- .entity(ImmutableMap.of("message", "Legacy Anomaly function cannot be found for id "+ anomalyFunctionID))
- .build();
- }
- return Response.ok(this.yaml.dump(translateAnomalyFunctionToYaml(anomalyFunctionDTO))).build();
- }
-
- @GET
- @Produces(MediaType.TEXT_PLAIN)
- @Consumes(MediaType.APPLICATION_JSON)
- @Path("/legacy-alert-to-yaml/{id}")
- public Response getYamlFromLegacyAlert(@PathParam("id") long alertId) {
- AlertConfigDTO alertConfigDTO = this.alertConfigDAO.findById(alertId);
- if (alertConfigDTO == null) {
- return Response.status(Response.Status.BAD_REQUEST)
- .entity(ImmutableMap.of("message", "Legacy alert cannot be found for ID "+ alertId))
- .build();
- }
- return Response.ok(this.yaml.dump(translateAlertToYaml(alertConfigDTO))).build();
- }
-
- @POST
- @ApiOperation("migrate an application")
- @Path("/application/{name}")
- public Response migrateApplication(@PathParam("name") String application) {
- List<AlertConfigDTO> alertConfigDTOList = alertConfigDAO.findByPredicate(Predicate.EQ("application", application));
- Map<String, String> responseMessage = new HashMap<>();
-
- for (AlertConfigDTO alertConfigDTO : alertConfigDTOList) {
- try {
- migrateLegacyNotification(alertConfigDTO);
- } catch (Exception e) {
- // Skip migrating this alert and move on to the next
- LOGGER.error("[MIG] Failed to migrate alert ID {} name {}. Exception {}", alertConfigDTO.getId(), alertConfigDTO.getName(), e);
- responseMessage.put("Status of alert " + alertConfigDTO.getId(),
- String.format("Failed to migrate alert ID %d with name %s due to %s", alertConfigDTO.getId(),
- alertConfigDTO.getName(), e.getMessage()));
- }
- }
-
- if (responseMessage.isEmpty()) {
- LOGGER.info("[MIG] Application " + application + " has been successfully migrated");
- return Response.ok("Application " + application + " has been successfully migrated").build();
- } else {
- LOGGER.error("[MIG] Found errors while migrating application " + application);
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- }
-
- @POST
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
- @Path("/anomaly-function/{id}")
- public Response migrateAnomalyFunction(@PathParam("id") long anomalyFunctionId) throws Exception {
- return Response.ok(migrateLegacyAnomalyFunction(anomalyFunctionId)).build();
- }
-
- @POST
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
- @Path("/unsubscribed-anomaly-functions")
- public Response migrateAnomalyFunction() throws Exception {
- List<AnomalyFunctionDTO> anomalyFunctionDTOs = anomalyFunctionDAO.findAll();
- Map<String, String> responseMessage = new HashMap<>();
-
- for (AnomalyFunctionDTO func : anomalyFunctionDTOs) {
- if (func.getFunctionName().contains(MIGRATED_TAG)) {
- LOGGER.info("[MIG] Function already migrated. Name " + func.getFunctionName());
- continue;
- }
-
- try {
- validateFunction(func);
- } catch (ValidationException e) {
- LOGGER.info("[MIG] Function failed validation. Name " + func.getFunctionName() + " Error : " + e.getMessage());
- responseMessage.put("Failed to migrate " + func.getId(), String.format("Validation Error : %s", e.getMessage()));
- continue;
- }
-
- try {
- migrateLegacyAnomalyFunction(func);
- } catch (Exception e) {
- // Skip migrating this function and move on to the next
- responseMessage.put("Failed to migrate " + func.getId(), String.format("Error : %s", e.getMessage()));
- }
- }
-
- return Response.ok(responseMessage).build();
- }
-
- @POST
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
- @ApiOperation("migrate all applications")
- @Path("/applications")
- public Response migrateApplication() {
- List<ApplicationDTO> applicationDTOS = this.appDAO.findAll();
- Map<String, String> responseMessage = new HashMap<>();
-
- for (ApplicationDTO app : applicationDTOS) {
- try {
- Response response = migrateApplication(app.getApplication());
- if (response.getStatusInfo() != Response.Status.OK) {
- throw new RuntimeException("Found " + ConfigUtils.getMap(response.getEntity()).size() + " issues while migrating alerts.");
- }
- } catch (Exception e) {
- // Skip migrating this application
- LOGGER.error("[MIG] Failed to migrate application {}. Exception {}", app.getApplication(), e);
- responseMessage.put("Status of application " + app.getApplication(),
- String.format("Failed to migrate application %s due to %s", app.getApplication(), e.getMessage()));
- }
- }
-
- if (responseMessage.isEmpty()) {
- LOGGER.info("[MIG] Successfully migrated all the applications");
- return Response.ok("All applications have been successfully migrated").build();
- } else {
- LOGGER.error("[MIG] Errors found while migrating application. Errors:\n" + responseMessage);
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- }
-
- @POST
- @Produces(MediaType.APPLICATION_JSON)
- @Consumes(MediaType.APPLICATION_JSON)
- @ApiOperation("migrate the partially migrated alerts")
- @Path("/partial-alerts")
- public Response migratePartialAlerts() {
- List<DetectionAlertConfigDTO> subsGroups = this.detectionAlertConfigDAO.findAll();
- Map<String, String> responseMessage = new HashMap<>();
-
- for (DetectionAlertConfigDTO subsGroup : subsGroups) {
- try {
- if (subsGroup.isOnlyFetchLegacyAnomalies()) {
- Set<Long> detectionIds = new HashSet<>();
- // Update and point these to the new detection ids
- if (subsGroup.getVectorClocks() != null) {
- Map<Long, Long> migratedVectorClock = new HashMap<>();
- Map<Long, Long> legacyVectorClock = subsGroup.getVectorClocks();
- for (long id : legacyVectorClock.keySet()) {
- long migratedId = migrateLegacyAnomalyFunction(id);
- detectionIds.add(migratedId);
- migratedVectorClock.put(migratedId, legacyVectorClock.get(id));
- }
- subsGroup.setVectorClocks(migratedVectorClock);
- }
- subsGroup.getProperties().put(PROP_DETECTION_CONFIG_IDS, detectionIds);
-
- // Remove Alert Filters! These are migrated and part of the detection yaml
- subsGroup.getProperties().remove("legacyAlertFilterConfigs");
- subsGroup.getProperties().remove("legacyAlertFilterClassName");
-
- subsGroup.getProperties().put("className", ToAllRecipientsDetectionAlertFilter.class.getName());
-
- int detectionAlertConfigId = this.detectionAlertConfigDAO.update(subsGroup);
- if (detectionAlertConfigId <= 0) {
- throw new RuntimeException("Failed to update the detection alert config.");
- }
- }
- } catch (Exception e) {
- // Skip migrating this partial migrated alert
- LOGGER.error("[MIG] Failed to migrate partial subscription group id {} name {}. Exception {}", subsGroup.getId(), subsGroup.getName(), e);
- responseMessage.put("Status of subscription group " + subsGroup.getName(),
- String.format("Failed to migrate subscription group %s due to %s", subsGroup.getName(), e.getMessage()));
- }
- }
-
- if (responseMessage.isEmpty()) {
- LOGGER.info("[MIG] Successfully migrated all the partially migrated alerts");
- return Response.ok("All partial alerts have been successfully migrated").build();
- } else {
- LOGGER.error("[MIG] Errors found while migrating partially migrated alerts. Errors:\n" + responseMessage);
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
index 21bea17..7226e75 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/StatefulDetectionAlertFilter.java
@@ -57,11 +57,7 @@ public abstract class StatefulDetectionAlertFilter extends DetectionAlertFilter
AnomalySlice slice = new AnomalySlice().withStart(startTime).withEnd(this.endTime);
Collection<MergedAnomalyResultDTO> candidates;
- if (this.config.isOnlyFetchLegacyAnomalies()) {
- candidates = this.provider.fetchLegacyAnomalies(Collections.singletonList(slice), functionId).get(slice);
- } else {
- candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice);
- }
+ candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice);
Collection<MergedAnomalyResultDTO> anomalies =
Collections2.filter(candidates, new Predicate<MergedAnomalyResultDTO>() {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java
deleted file mode 100644
index e96ab88..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilter.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection.alert.filter;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Collections2;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import org.apache.pinot.thirdeye.detection.annotation.AlertFilter;
-import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
-import org.apache.pinot.thirdeye.detection.ConfigUtils;
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilter;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterResult;
-import org.apache.pinot.thirdeye.detector.email.filter.BaseAlertFilter;
-import org.apache.pinot.thirdeye.detector.email.filter.DummyAlertFilter;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.annotation.Nullable;
-import org.apache.commons.collections.MapUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-@AlertFilter(type = "LEGACY_ALERTER_PIPELINE")
-public class LegacyAlertFilter extends DetectionAlertFilter {
- private final static Logger LOG = LoggerFactory.getLogger(LegacyAlertFilter.class);
-
- private static final String PROP_LEGACY_ALERT_FILTER_CONFIGS = "legacyAlertFilterConfigs";
- private static final String PROP_LEGACY_ALERT_FILTER_CLASS_NAME = "legacyAlertFilterClassName";
- private static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds";
- private static final String PROP_RECIPIENTS = "recipients";
- private static final String PROP_TO = "to";
- private static final String PROP_CC = "cc";
- private static final String PROP_BCC = "bcc";
- private static final String PROP_SEND_ONCE = "sendOnce";
-
- private final List<Long> detectionConfigIds;
- private final Map<Long, Long> vectorClocks;
- private final boolean sendOnce;
-
- public LegacyAlertFilter(DataProvider provider, DetectionAlertConfigDTO config, long endTime) throws Exception {
- super(provider, config, endTime);
-
- this.detectionConfigIds = ConfigUtils.getLongs(this.config.getProperties().get(PROP_DETECTION_CONFIG_IDS));
- this.vectorClocks = this.config.getVectorClocks();
- this.sendOnce = MapUtils.getBoolean(this.config.getProperties(), PROP_SEND_ONCE, true);
- }
-
- @Override
- public DetectionAlertFilterResult run() throws Exception {
- DetectionAlertFilterResult result = new DetectionAlertFilterResult();
-
- Map<String, Set<String>> recipientsMap = ConfigUtils.getMap(this.config.getProperties().get(PROP_RECIPIENTS));
- Set<String> to = (recipientsMap.get(PROP_TO) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_TO));
- Set<String> cc = (recipientsMap.get(PROP_CC) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_CC));
- Set<String> bcc = (recipientsMap.get(PROP_BCC) == null) ? Collections.emptySet() : new HashSet<>(recipientsMap.get(PROP_BCC));
- DetectionAlertFilterRecipients recipients = new DetectionAlertFilterRecipients(to, cc, bcc);
-
- Map<String, Object> alertFilterConfig = MapUtils.getMap(config.getProperties(), PROP_LEGACY_ALERT_FILTER_CONFIGS);
- if (alertFilterConfig == null || alertFilterConfig.size() == 0) {
- LOG.warn("alertFilterConfig is null or empty in notification group {}", this.config.getId());
- }
-
- for (Long functionId : this.detectionConfigIds) {
- long startTime = MapUtils.getLong(this.vectorClocks, functionId, 0L);
-
- AnomalySlice slice = new AnomalySlice()
- .withStart(startTime)
- .withEnd(this.endTime);
-
- Collection<MergedAnomalyResultDTO> candidates;
- if (this.config.isOnlyFetchLegacyAnomalies()) {
- candidates = this.provider.fetchLegacyAnomalies(Collections.singletonList(slice), functionId).get(slice);
- } else {
- candidates = this.provider.fetchAnomalies(Collections.singletonList(slice), functionId).get(slice);
- }
-
- BaseAlertFilter alertFilter = new DummyAlertFilter();
- if (config.getProperties().containsKey(PROP_LEGACY_ALERT_FILTER_CLASS_NAME)) {
- String className = MapUtils.getString(config.getProperties(), PROP_LEGACY_ALERT_FILTER_CLASS_NAME);
- alertFilter = (BaseAlertFilter) Class.forName(className).newInstance();
- Map<String, String> params = MapUtils.getMap(alertFilterConfig, functionId.toString());
- if (params == null) {
- LOG.warn("AlertFilter cannot be found for function {} in notification group {}", functionId, this.config.getId());
- }
-
- alertFilter.setParameters(params);
- }
-
- BaseAlertFilter finalAlertFilter = alertFilter;
- final long minId = getMinId(this.config.getHighWaterMark());
- Collection<MergedAnomalyResultDTO> anomalies =
- Collections2.filter(candidates, new Predicate<MergedAnomalyResultDTO>() {
- @Override
- public boolean apply(@Nullable MergedAnomalyResultDTO mergedAnomaly) {
- return mergedAnomaly != null
- && !mergedAnomaly.isChild()
- && finalAlertFilter.isQualified(mergedAnomaly)
- && (mergedAnomaly.getId() == null || mergedAnomaly.getId() >= minId);
- }
- });
-
- if (result.getResult().isEmpty()) {
- result.addMapping(recipients, new HashSet<>(anomalies));
- } else {
- result.getResult().get(recipients).addAll(anomalies);
- }
- }
-
- return result;
- }
-
- private long getMinId(long highWaterMark) {
- if (this.sendOnce) {
- return highWaterMark + 1;
- } else {
- return 0;
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
index 9d36de9..9f6d8d9 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
@@ -171,9 +171,6 @@ public class YamlDetectionAlertConfigTranslator {
alertConfigDTO.setCronExpression(MapUtils.getString(yamlAlertConfig, PROP_CRON, CRON_SCHEDULE_DEFAULT));
alertConfigDTO.setActive(MapUtils.getBooleanValue(yamlAlertConfig, PROP_ACTIVE, true));
- // TODO: Remove all references to onlyFetchLegacyAnomalies after migration
- alertConfigDTO.setOnlyFetchLegacyAnomalies(MapUtils.getBooleanValue(yamlAlertConfig, PROP_ONLY_FETCH_LEGACY_ANOMALIES, false));
-
alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.valueOf(
(String) MapUtils.getObject(yamlAlertConfig, PROP_EMAIL_SUBJECT_TYPE, AlertConfigBean.SubjectType.METRICS.name())));
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 98945a5..c3a1a5a 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -471,7 +471,6 @@ public class YamlResource {
oldAlertConfig.setActive(newAlertConfig.isActive());
oldAlertConfig.setAlertSchemes(newAlertConfig.getAlertSchemes());
oldAlertConfig.setAlertSuppressors(newAlertConfig.getAlertSuppressors());
- oldAlertConfig.setOnlyFetchLegacyAnomalies(newAlertConfig.isOnlyFetchLegacyAnomalies());
oldAlertConfig.setProperties(newAlertConfig.getProperties());
return oldAlertConfig;
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java
deleted file mode 100644
index 4b796ed..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import javax.ws.rs.core.Response;
-import org.apache.commons.collections.MapUtils;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry;
-import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
-import org.apache.pinot.thirdeye.detection.components.PercentageChangeRuleDetector;
-import org.apache.pinot.thirdeye.detection.components.RuleBaselineProvider;
-import org.apache.pinot.thirdeye.detection.components.ThresholdRuleAnomalyFilter;
-import org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
-import org.apache.pinot.thirdeye.detection.yaml.CompositePipelineConfigTranslator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.apache.pinot.thirdeye.detection.DetectionMigrationResource.*;
-import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
-
-
-public class DetectionMigrationResourceTest {
- private static final Logger LOGGER = LoggerFactory.getLogger(DetectionMigrationResourceTest.class);
-
- private DAOTestBase testDAOProvider;
- private MetricConfigManager metricDAO;
- private DatasetConfigManager datasetDAO;
- private MergedAnomalyResultManager anomalyDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private AlertConfigManager alertConfigDAO;
- private ApplicationManager applicationDAO;
- private DetectionConfigManager detectionConfigDAO;
- private DetectionAlertConfigManager detectionAlertConfigDAO;
-
- private DetectionMigrationResource migrationResource;
-
- private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- @BeforeMethod(alwaysRun = true)
- public void setup() {
- this.testDAOProvider = DAOTestBase.getInstance();
- this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
- this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
- this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
- this.anomalyFunctionDAO = DAORegistry.getInstance().getAnomalyFunctionDAO();
- this.alertConfigDAO = DAORegistry.getInstance().getAlertConfigDAO();
- this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager();
- this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager();
- this.applicationDAO = DAORegistry.getInstance().getApplicationDAO();
-
- migrationResource = new DetectionMigrationResource(
- anomalyFunctionDAO, alertConfigDAO, applicationDAO, metricDAO, detectionConfigDAO, detectionAlertConfigDAO, datasetDAO, anomalyDAO);
-
- DetectionRegistry.registerYamlConvertor(CompositePipelineConfigTranslator.class.getName(), "COMPOSITE");
- DetectionRegistry.registerComponent(PercentageChangeRuleDetector.class.getName(), "PERCENTAGE_RULE");
- DetectionRegistry.registerComponent(RuleBaselineProvider.class.getName(), "RULE_BASELINE");
- DetectionRegistry.registerComponent(ThresholdRuleDetector.class.getName(), "THRESHOLD");
- DetectionRegistry.registerComponent(ThresholdRuleAnomalyFilter.class.getName(), "THRESHOLD_RULE_FILTER");
-
- DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL", "EmailClass");
- DetectionAlertRegistry.getInstance().registerAlertFilter("DEFAULT_ALERTER_PIPELINE", "RECIPIENTClass");
-
- MetricConfigDTO metricConfigDTO = new MetricConfigDTO();
- metricConfigDTO.setName("test_metric");
- metricConfigDTO.setDataset("test_collection");
- metricConfigDTO.setActive(true);
- metricConfigDTO.setAlias("test_collection::test_metric");
- metricDAO.save(metricConfigDTO);
-
- DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
- datasetConfigDTO.setDataset("test_collection");
- datasetConfigDTO.setNonAdditiveBucketSize(1);
- datasetConfigDTO.setNonAdditiveBucketUnit(TimeUnit.DAYS);
- datasetDAO.save(datasetConfigDTO);
- }
-
- @AfterMethod(alwaysRun = true)
- public void cleanup() {
- this.testDAOProvider.cleanup();
- }
-
- @Test
- public void testTranslateAlertToYaml() throws Exception {
- AnomalyFunctionDTO anomalyFunction1 = new AnomalyFunctionDTO();
- anomalyFunction1.setFunctionName("function1");
- long id1 = anomalyFunctionDAO.save(anomalyFunction1);
-
- AnomalyFunctionDTO anomalyFunction2 = new AnomalyFunctionDTO();
- anomalyFunction2.setFunctionName("function2");
- long id2 = anomalyFunctionDAO.save(anomalyFunction2);
-
- AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
- alertConfigDTO.setActive(true);
- alertConfigDTO.setName("test_notification");
- alertConfigDTO.setApplication("test_application");
- alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.ALERT);
- alertConfigDTO.setCronExpression("0 0 14 * * ? *");
- alertConfigDTO.setFromAddress("test@thirdeye.com");
- alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients(
- Collections.singleton("to@test"),
- Collections.singleton("cc@test"),
- Collections.singleton("bcc@test")));
-
- AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig();
- emailConfig.setFunctionIds(Arrays.asList(id1, id2));
- emailConfig.setAnomalyWatermark(9999);
- alertConfigDTO.setEmailConfig(emailConfig);
-
- Map<String, Object> yamlMap = migrationResource.translateAlertToYaml(alertConfigDTO);
-
- Assert.assertTrue(MapUtils.getBoolean(yamlMap, "active"));
- Assert.assertEquals(yamlMap.get(PROP_SUBS_GROUP_NAME), "test_notification");
- Assert.assertEquals(yamlMap.get(PROP_CRON), "0 0 14 * * ? *");
- Assert.assertEquals(yamlMap.get(PROP_APPLICATION), "test_application");
- Assert.assertEquals(yamlMap.get(PROP_EMAIL_SUBJECT_TYPE), AlertConfigBean.SubjectType.ALERT.name());
- Assert.assertEquals(yamlMap.get(PROP_FROM), "test@thirdeye.com");
- Assert.assertEquals(yamlMap.get(PROP_TYPE), "DEFAULT_ALERTER_PIPELINE");
- Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_DETECTION_NAMES)).size(), 2);
- Assert.assertNotNull(yamlMap.get(PROP_ALERT_SCHEMES));
- Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).size(), 1);
- Assert.assertEquals(ConfigUtils.getMap(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).get(0)).get(PROP_TYPE), "EMAIL");
- Assert.assertNotNull(yamlMap.get(PROP_RECIPIENTS));
- Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("to")), Collections.singletonList("to@test"));
- Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("cc")), Collections.singletonList("cc@test"));
- Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("bcc")), Collections.singletonList("bcc@test"));
- }
-
- @Test
- public void testMigrateWoWAnomalyFunction() throws Exception {
- AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-1.json"), AnomalyFunctionDTO.class);
- long oldID = anomalyFunctionDAO.save(actual);
-
- AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID);
- legacyAnomalyFunction.setCreatedBy("test_user");
- anomalyFunctionDAO.update(legacyAnomalyFunction);
-
- MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO();
- mergedAnomalyResultDTO.setFunction(actual);
- mergedAnomalyResultDTO.setMetric("test_metric");
- mergedAnomalyResultDTO.setCollection("test_collection");
- anomalyDAO.save(mergedAnomalyResultDTO);
-
- Response responseId = migrationResource.migrateAnomalyFunction(oldID);
- long newID = (long) responseId.getEntity();
-
- DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID);
-
- legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID);
-
- // Verify if the migration status is updated correctly and the old detection is disabled.
- Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID);
- Assert.assertFalse(legacyAnomalyFunction.getIsActive());
-
- // Verify if the anomaly is pointing to the migrated anomaly function
- List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll();
- Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID);
-
- // Assert the migrated object
- DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-1.json"), DetectionConfigDTO.class);
- expected.setId(migratedAnomalyFunction.getId());
- expected.setVersion(migratedAnomalyFunction.getVersion());
- expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy());
- expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy());
- expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp());
-
- Assert.assertEquals(migratedAnomalyFunction, expected);
- }
-
- @Test
- public void testMinMaxAnomalyFunction() throws Exception {
- AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class);
- long oldID = anomalyFunctionDAO.save(actual);
-
- MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO();
- mergedAnomalyResultDTO.setFunction(actual);
- mergedAnomalyResultDTO.setMetric("test_metric");
- mergedAnomalyResultDTO.setCollection("test_collection");
- anomalyDAO.save(mergedAnomalyResultDTO);
-
- Response responseId = migrationResource.migrateAnomalyFunction(oldID);
- long newID = (long) responseId.getEntity();
-
- AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID);
- DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID);
-
- // Verify if the migration status is updated correctly and the old detection is disabled.
- Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID);
- Assert.assertFalse(legacyAnomalyFunction.getIsActive());
-
- // Verify if the anomaly is pointing to the migrated anomaly function
- List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll();
- Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID);
-
- // Assert the migrated object
- DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-2.json"), DetectionConfigDTO.class);
- expected.setId(migratedAnomalyFunction.getId());
- expected.setVersion(migratedAnomalyFunction.getVersion());
- expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy());
- expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy());
- expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp());
- Assert.assertEquals(migratedAnomalyFunction, expected);
- }
-
- @Test
- public void testMigrateApplication() throws Exception {
- AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class);
- long id1 = anomalyFunctionDAO.save(actual);
-
- AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
- alertConfigDTO.setActive(true);
- alertConfigDTO.setName("test_notification");
- alertConfigDTO.setApplication("test_application");
- alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.ALERT);
- alertConfigDTO.setCronExpression("0 0 14 * * ? *");
- alertConfigDTO.setFromAddress("test@thirdeye.com");
- alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients(
- Collections.singleton("to@test"),
- Collections.singleton("cc@test"),
- Collections.singleton("bcc@test")));
- AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig();
- emailConfig.setFunctionIds(Arrays.asList(id1));
- emailConfig.setAnomalyWatermark(9999);
- alertConfigDTO.setEmailConfig(emailConfig);
- alertConfigDAO.save(alertConfigDTO);
-
- AlertConfigDTO alertConfigDTOMigrated = new AlertConfigDTO();
- alertConfigDTOMigrated.setActive(false);
- alertConfigDTOMigrated.setName("test_notification" + MIGRATED_TAG);
- alertConfigDTOMigrated.setApplication("test_application");
- alertConfigDTOMigrated.setSubjectType(AlertConfigBean.SubjectType.ALERT);
- alertConfigDTOMigrated.setCronExpression("0 0 14 * * ? *");
- alertConfigDTOMigrated.setFromAddress("test@thirdeye.com");
- alertConfigDTOMigrated.setReceiverAddresses(new DetectionAlertFilterRecipients(
- Collections.singleton("to@test"),
- Collections.singleton("cc@test"),
- Collections.singleton("bcc@test")));
- alertConfigDTO.setEmailConfig(emailConfig);
- alertConfigDAO.save(alertConfigDTOMigrated);
-
- ApplicationDTO applicationDTO = new ApplicationDTO();
- applicationDTO.setApplication("test_application");
- applicationDTO.setRecipients("test@thirdeye.com");
- applicationDAO.save(applicationDTO);
-
- Response response = migrationResource.migrateApplication("test_application");
-
- Assert.assertEquals(response.getEntity(), "Application test_application has been successfully migrated");
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java
index 17497b7..27483b3 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/MockDataProvider.java
@@ -149,12 +149,6 @@ public class MockDataProvider implements DataProvider {
}
@Override
- public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchLegacyAnomalies(Collection<AnomalySlice> slices,
- long configId) {
- return fetchAnomalies(slices, configId, true);
- }
-
- @Override
public Multimap<AnomalySlice, MergedAnomalyResultDTO> fetchAnomalies(Collection<AnomalySlice> slices,
long configId) {
return fetchAnomalies(slices, configId, false);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java
deleted file mode 100644
index c2c9b01..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/LegacyAlertFilterTest.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.detection.alert.filter;
-
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import org.apache.pinot.thirdeye.detection.MockDataProvider;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterResult;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.apache.pinot.thirdeye.detection.DetectionTestUtils.*;
-
-
-public class LegacyAlertFilterTest {
- private static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds";
- private static final List<Long> PROP_ID_VALUE = Arrays.asList(1001L, 1002L);
- private static final String PROP_LEGACY_ALERT_FILTER_CONFIG = "legacyAlertFilterConfig";
- private static final String PROP_LEGACY_ALERT_FILTER_CLASS_NAME = "legacyAlertFilterClassName";
- private static final Set<String> TO_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("test@example.com", "mytest@example.org"));
- private static final Set<String> CC_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("iamcc@host.domain", "iamcc2@host.domain"));
- private static final Set<String> BCC_RECIPIENTS_VALUES = new HashSet<>(Arrays.asList("iambcc@host.domain"));
- private static final String PROP_RECIPIENTS = "recipients";
-
- private List<MergedAnomalyResultDTO> detectedAnomalies;
- private LegacyAlertFilter legacyAlertFilter;
- private LegacyAlertFilter legacyAlertFilterOnLegacyAnomalies;
- private Map<String, Set<String>> recipientsMap;
-
- @BeforeMethod
- public void beforeMethod() throws Exception {
- this.detectedAnomalies = new ArrayList<>();
- this.detectedAnomalies.add(makeAnomaly(1001L, 1500, 2000));
- this.detectedAnomalies.add(makeAnomaly(1001L, 0, 1000));
- this.detectedAnomalies.add(makeAnomaly(1002L, 0, 1000));
- this.detectedAnomalies.add(makeAnomaly(1002L, 1100, 1500));
- this.detectedAnomalies.add(makeAnomaly(1002L, 3333, 9999));
- this.detectedAnomalies.add(makeAnomaly(1003L, 1100, 1500));
-
- // Anomalies generated by legacy pipeline
- this.detectedAnomalies.add(makeAnomaly(null, 1000L, 1100, 1500));
- this.detectedAnomalies.add(makeAnomaly(null, 1002L, 0, 1000));
- this.detectedAnomalies.add(makeAnomaly(null, 1002L, 1100, 2000));
-
-
- DataProvider mockDataProvider = new MockDataProvider().setAnomalies(this.detectedAnomalies);
-
- DetectionAlertConfigDTO detectionAlertConfig = createDetectionAlertConfig();
- this.legacyAlertFilter = new LegacyAlertFilter(mockDataProvider, detectionAlertConfig, 2500L);
-
- DetectionAlertConfigDTO detectionAlertConfigLegacyAnomalies = createDetectionAlertConfig();
- detectionAlertConfigLegacyAnomalies.setOnlyFetchLegacyAnomalies(true);
- this.legacyAlertFilterOnLegacyAnomalies = new LegacyAlertFilter(mockDataProvider, detectionAlertConfigLegacyAnomalies, 2500L);
-
- this.recipientsMap = new HashMap<>();
- recipientsMap.put("to", TO_RECIPIENTS_VALUES);
- recipientsMap.put("cc", CC_RECIPIENTS_VALUES);
- recipientsMap.put("bcc", BCC_RECIPIENTS_VALUES);
- }
-
- private DetectionAlertConfigDTO createDetectionAlertConfig() {
- DetectionAlertConfigDTO detectionAlertConfig = new DetectionAlertConfigDTO();
- Map<String, Object> properties = new HashMap<>();
- properties.put(PROP_DETECTION_CONFIG_IDS, PROP_ID_VALUE);
- properties.put(PROP_LEGACY_ALERT_FILTER_CLASS_NAME, "org.apache.pinot.thirdeye.detector.email.filter.DummyAlertFilter");
- properties.put(PROP_LEGACY_ALERT_FILTER_CONFIG, "");
- properties.put(PROP_RECIPIENTS, recipientsMap);
- detectionAlertConfig.setHighWaterMark(0L);
- detectionAlertConfig.setProperties(properties);
- detectionAlertConfig.setVectorClocks(new HashMap<Long, Long>());
-
- return detectionAlertConfig;
- }
-
- @Test
- public void testRun() throws Exception {
- DetectionAlertFilterResult result = this.legacyAlertFilter.run();
- Assert.assertEquals(result.getAllAnomalies(), new HashSet<>(this.detectedAnomalies.subList(0, 4)));
- }
-
- @Test
- public void testFetchingLegacyAnomalies() throws Exception {
- DetectionAlertFilterResult result = this.legacyAlertFilterOnLegacyAnomalies.run();
- Assert.assertEquals(result.getAllAnomalies().size(), 2);
- Assert.assertEquals(result.getAllAnomalies(), new HashSet<>(this.detectedAnomalies.subList(7, 9)));
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java
index 6896820..138d054 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/alert/filter/ToAllRecipientsDetectionAlertFilterTest.java
@@ -82,7 +82,6 @@ public class ToAllRecipientsDetectionAlertFilterTest {
this.alertConfig = createDetectionAlertConfig();
this.alertConfigForLegacyAnomalies = createDetectionAlertConfig();
- this.alertConfigForLegacyAnomalies.setOnlyFetchLegacyAnomalies(true);
}
private DetectionAlertConfigDTO createDetectionAlertConfig() {
@@ -113,15 +112,6 @@ public class ToAllRecipientsDetectionAlertFilterTest {
}
@Test
- public void testGetAlertFilterResultForLegacyAnomalies() throws Exception {
- this.alertFilter = new ToAllRecipientsDetectionAlertFilter(this.provider, this.alertConfigForLegacyAnomalies,2500L);
-
- DetectionAlertFilterResult result = this.alertFilter.run();
- Assert.assertEquals(result.getResult().get(RECIPIENTS).size(), 2);
- Assert.assertEquals(result.getResult().get(RECIPIENTS), new HashSet<>(this.detectedAnomalies.subList(7, 9)));
- }
-
- @Test
public void testAlertFilterFeedback() throws Exception {
this.alertConfig.getProperties().put(PROP_DETECTION_CONFIG_IDS, Collections.singletonList(1003L));
this.alertFilter = new ToAllRecipientsDetectionAlertFilter(this.provider, this.alertConfig,2500L);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java
index e9eb0b0..cb57d1c 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslatorTest.java
@@ -68,7 +68,6 @@ public class YamlDetectionAlertConfigTranslatorTest {
DetectionAlertConfigDTO alertConfig = new YamlDetectionAlertConfigTranslator(this.detectionConfigManager).translate(alertYamlConfigs);
Assert.assertTrue(alertConfig.isActive());
- Assert.assertFalse(alertConfig.isOnlyFetchLegacyAnomalies());
Assert.assertEquals(alertConfig.getName(), "test_group_name");
Assert.assertEquals(alertConfig.getApplication(), "test_application");
Assert.assertEquals(alertConfig.getFrom(), "thirdeye@thirdeye");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org