You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/01/22 21:31:29 UTC
[incubator-pinot] branch master updated: [TE] Migration endpoints
for anomaly function and application (#3724)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 8daad94 [TE] Migration endpoints for anomaly function and application (#3724)
8daad94 is described below
commit 8daad94056a3de1ac841d6b303f46bdfd6a01b41
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Tue Jan 22 13:31:19 2019 -0800
[TE] Migration endpoints for anomaly function and application (#3724)
---
.../anomaly/ThirdEyeAnomalyApplication.java | 4 -
.../dashboard/ThirdEyeDashboardApplication.java | 10 +-
.../detection/DetectionMigrationResource.java | 235 ++++++++++++++++++---
.../registry/DetectionAlertRegistry.java | 12 +-
.../annotation/registry/DetectionRegistry.java | 28 ++-
.../validators/DetectionAlertConfigValidator.java | 6 +-
.../yaml/YamlDetectionAlertConfigTranslator.java | 23 +-
.../yaml/YamlDetectionTranslatorLoader.java | 1 +
.../thirdeye/detection/yaml/YamlResource.java | 4 +
.../detection/DetectionMigrationResourceTest.java | 196 ++++++++++++++++-
.../detection/legacy-anomaly-function-1.json | 27 +++
.../detection/legacy-anomaly-function-2.json | 26 +++
.../detection/migrated-detection-config-1.json | 47 +++++
.../detection/migrated-detection-config-2.json | 43 ++++
14 files changed, 595 insertions(+), 67 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
index ddfead8..7b5751d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
@@ -117,10 +117,6 @@ public class ThirdEyeAnomalyApplication
LOG.error("Exception while loading caches", e);
}
- // instantiate registry
- DetectionRegistry.init();
- DetectionAlertRegistry.init();
-
environment.getObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
environment.getObjectMapper().registerModule(makeMapperModule());
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index 141bd01..b38e7df 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -155,10 +155,6 @@ public class ThirdEyeDashboardApplication
LOG.error("Exception while loading caches", e);
}
- // instantiate registry
- DetectionRegistry.init();
- DetectionAlertRegistry.init();
-
AnomalyFunctionFactory anomalyFunctionFactory = new AnomalyFunctionFactory(config.getFunctionConfigPath());
AlertFilterFactory alertFilterFactory = new AlertFilterFactory(config.getAlertFilterConfigPath());
AlertFilterAutotuneFactory alertFilterAutotuneFactory = new AlertFilterAutotuneFactory(config.getFilterAutotuneConfigPath());
@@ -177,8 +173,10 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new ThirdEyeResource());
env.jersey().register(new DataResource(anomalyFunctionFactory, alertFilterFactory));
env.jersey().register(new AnomaliesResource(anomalyFunctionFactory, alertFilterFactory));
- env.jersey().register(new DetectionMigrationResource(DAO_REGISTRY.getAnomalyFunctionDAO(),
- DAO_REGISTRY.getDetectionConfigManager(), DAO_REGISTRY.getDatasetConfigDAO()));
+ env.jersey().register(new DetectionMigrationResource(
+ DAO_REGISTRY.getAnomalyFunctionDAO(), DAO_REGISTRY.getAlertConfigDAO(),
+ DAO_REGISTRY.getDetectionConfigManager(), DAO_REGISTRY.getDetectionAlertConfigManager(),
+ DAO_REGISTRY.getDatasetConfigDAO(), DAO_REGISTRY.getMergedAnomalyResultDAO()));
env.jersey().register(new OnboardResource(config));
env.jersey().register(new EntityMappingResource());
env.jersey().register(new OnboardDatasetMetricResource());
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
index 400de25..7c2c329 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
@@ -19,30 +19,45 @@
package org.apache.pinot.thirdeye.detection;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import org.apache.pinot.thirdeye.anomaly.detection.AnomalyDetectionInputContextBuilder;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import javax.ws.rs.GET;
+import javax.ws.rs.POST;
import javax.ws.rs.Path;
import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.thirdeye.anomaly.detection.AnomalyDetectionInputContextBuilder;
+import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
+import org.apache.pinot.thirdeye.detection.yaml.CompositePipelineConfigTranslator;
+import org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator;
+import org.apache.pinot.thirdeye.detection.yaml.YamlResource;
import org.joda.time.Period;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.DumperOptions;
@@ -62,37 +77,41 @@ public class DetectionMigrationResource {
private static final String PROP_WINDOW_DELAY_UNIT = "windowDelayUnit";
private static final String PROP_WINDOW_SIZE = "windowSize";
private static final String PROP_WINDOW_UNIT = "windowUnit";
+ private static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds";
+
+ static final String MIGRATED_TAG = "_thirdeye_migrated";
private final AnomalyFunctionManager anomalyFunctionDAO;
private final DetectionConfigManager detectionConfigDAO;
+ private final DetectionAlertConfigManager detectionAlertConfigDAO;
private final DatasetConfigManager datasetConfigDAO;
+ private final MergedAnomalyResultManager mergedAnomalyResultDAO;
+ private final AlertConfigManager alertConfigDAO;
private final Yaml yaml;
/**
* Instantiates a new Detection migration resource.
*/
- public DetectionMigrationResource(AnomalyFunctionManager anomalyFunctionDAO,
+ public DetectionMigrationResource(
+ AnomalyFunctionManager anomalyFunctionDAO,
+ AlertConfigManager alertConfigDAO,
DetectionConfigManager detectionConfigDAO,
- DatasetConfigManager datasetConfigDAO) {
+ DetectionAlertConfigManager detectionAlertConfigDAO,
+ DatasetConfigManager datasetConfigDAO,
+ MergedAnomalyResultManager mergedAnomalyResultDAO) {
this.anomalyFunctionDAO = anomalyFunctionDAO;
this.detectionConfigDAO = detectionConfigDAO;
+ this.detectionAlertConfigDAO = detectionAlertConfigDAO;
+ this.alertConfigDAO = alertConfigDAO;
this.datasetConfigDAO = datasetConfigDAO;
+ this.mergedAnomalyResultDAO = mergedAnomalyResultDAO;
DumperOptions options = new DumperOptions();
options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
options.setPrettyFlow(true);
this.yaml = new Yaml(options);
}
- /**
- * Endpoint to convert a existing anomaly function to a composite pipeline yaml
- *
- * @param anomalyFunctionId the anomaly function id
- * @return the yaml config as string
- */
- @GET
- public String migrateToYaml(@QueryParam("id") long anomalyFunctionId) throws Exception {
- AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionId);
- Preconditions.checkArgument(anomalyFunctionDTO.getIsActive(), "try to migrate inactive anomaly function");
+ private Map<String, Object> translateAnomalyFunctionToYaml(AnomalyFunctionDTO anomalyFunctionDTO) throws Exception {
Map<String, Object> yamlConfigs = new LinkedHashMap<>();
yamlConfigs.put("detectionName", anomalyFunctionDTO.getFunctionName());
yamlConfigs.put("metric", anomalyFunctionDTO.getMetric());
@@ -165,7 +184,7 @@ public class DetectionMigrationResource {
yamlConfigs.put("merger", mergerYaml);
}
- return this.yaml.dump(yamlConfigs);
+ return yamlConfigs;
}
private Map<String, Object> getDimensionExplorationParams(AnomalyFunctionDTO functionDTO) throws IOException {
@@ -270,6 +289,75 @@ public class DetectionMigrationResource {
return detectorYaml;
}
+ long migrateLegacyAnomalyFunction(long anomalyFunctionId) {
+ AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionId);
+ if (anomalyFunctionDTO == null) {
+ throw new RuntimeException(String.format("Couldn't find anomaly function with id %d", anomalyFunctionId));
+ }
+
+ return migrateLegacyAnomalyFunction(anomalyFunctionDTO);
+ }
+
+ private long migrateLegacyAnomalyFunction(AnomalyFunctionDTO anomalyFunctionDTO) {
+ DetectionConfigDTO detectionConfig;
+ try {
+ LOGGER.info(String.format("[MIG] Migrating anomaly function %d_%s", anomalyFunctionDTO.getId(),
+ anomalyFunctionDTO.getFunctionName()));
+
+ // Check if this anomaly function is already migrated
+ if (anomalyFunctionDTO.getFunctionName().contains(MIGRATED_TAG)) {
+ LOGGER.info(String.format("[MIG] Anomaly function %d is already migrated.", anomalyFunctionDTO.getId()));
+
+ // Fetch the migrated config id and return
+ String funcName = anomalyFunctionDTO.getFunctionName();
+ return Long.parseLong(funcName.substring(funcName.lastIndexOf("_") + 1, funcName.length()));
+ }
+
+ // Migrate anomaly function config to the detection config by converting to YAML and then to Detection Config
+ Map<String, String> responseMessage = new HashMap<>();
+ Map<String, Object> detectionYAMLMap = translateAnomalyFunctionToYaml(anomalyFunctionDTO);
+ detectionConfig = new YamlResource().translateToDetectionConfig(detectionYAMLMap, responseMessage);
+
+ if (detectionConfig == null) {
+ throw new RuntimeException("Couldn't translate yaml to detection config. Message = " + responseMessage.get("message"));
+ }
+
+ // Save the migrated anomaly function
+ DAORegistry.getInstance().getDetectionConfigManager().save(detectionConfig);
+ if (detectionConfig.getId() == null) {
+ throw new RuntimeException("Error saving the new detection config.");
+ }
+
+ // Point all the associated anomalies to the migrated anomaly function.
+ List<MergedAnomalyResultDTO> mergedAnomalyResultDTOS = mergedAnomalyResultDAO.findByPredicate(Predicate.EQ("functionId", anomalyFunctionDTO.getId()));
+ for (MergedAnomalyResultDTO anomaly : mergedAnomalyResultDTOS) {
+ anomaly.setDetectionConfigId(detectionConfig.getId());
+ int affectedRows = mergedAnomalyResultDAO.update(anomaly);
+ if (affectedRows == 0) {
+ throw new RuntimeException("Failed to update the anomaly " + anomaly.getId() + " with the new detection id"
+ + " for anomaly function " + detectionConfig.getId());
+ }
+ }
+
+ // Mark the old anomaly function as migrated
+ anomalyFunctionDTO.setActive(false);
+ anomalyFunctionDTO.setFunctionName(anomalyFunctionDTO.getFunctionName() + MIGRATED_TAG + "_" + detectionConfig.getId());
+ int affectedRows = this.anomalyFunctionDAO.update(anomalyFunctionDTO);
+ if (affectedRows == 0) {
+ throw new RuntimeException("Anomaly function migrated successfully but failed to disable and update the"
+ + " migration status of the old anomaly function. Recommend doing it manually. Migrated detection id "
+ + detectionConfig.getId());
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Error migrating anomaly function %d with name %s. Message = %s",
+ anomalyFunctionDTO.getId(), anomalyFunctionDTO.getFunctionName(), e.getMessage()), e);
+ }
+
+ LOGGER.info(String.format("[MIG] Successfully migrated anomaly function %d_%s", anomalyFunctionDTO.getId(),
+ anomalyFunctionDTO.getFunctionName()));
+ return detectionConfig.getId();
+ }
+
Map<String, Object> translateAlertToYaml(AlertConfigDTO alertConfigDTO) {
Map<String, Object> yamlConfigs = new LinkedHashMap<>();
@@ -288,12 +376,103 @@ public class DetectionMigrationResource {
recipients.put("bcc", alertConfigDTO.getReceiverAddresses().getBcc());
yamlConfigs.put(PROP_RECIPIENTS, recipients);
- Map<String, Object> alertSchemes = new LinkedHashMap<>();
- alertSchemes.put(PROP_TYPE, "EMAIL");
- yamlConfigs.put(PROP_ALERT_SCHEMES, alertSchemes);
+ List<Map<String, Object>> schemes = new ArrayList<>();
+ Map<String, Object> emailScheme = new LinkedHashMap<>();
+ emailScheme.put(PROP_TYPE, "EMAIL");
+ schemes.add(emailScheme);
+ yamlConfigs.put(PROP_ALERT_SCHEMES, schemes);
- yamlConfigs.put(PROP_DETECTION_CONFIG_IDS, alertConfigDTO.getEmailConfig().getFunctionIds());
+ List<String> detectionNames = new ArrayList<>();
+ List<Long> detectionIds = alertConfigDTO.getEmailConfig().getFunctionIds();
+ try {
+ detectionNames.addAll(detectionIds.stream().map(detectionId -> this.anomalyFunctionDAO.findByPredicate(
+ Predicate.EQ("baseId", detectionId)).get(0).getFunctionName()).collect(Collectors.toList()));
+ } catch (Exception e){
+ throw new IllegalArgumentException("cannot find subscribed detection id, please check the function ids");
+ }
+ yamlConfigs.put(PROP_DETECTION_NAMES, detectionNames);
return yamlConfigs;
}
+
+ @GET
+ public Response getYamlFromLegacyAnomalyFunction(long anomalyFunctionID) throws Exception {
+ AnomalyFunctionDTO anomalyFunctionDTO = this.anomalyFunctionDAO.findById(anomalyFunctionID);
+ if (anomalyFunctionDTO == null) {
+ return Response.status(Response.Status.BAD_REQUEST)
+ .entity(ImmutableMap.of("message", "Legacy Anomaly function cannot be found for id "+ anomalyFunctionID))
+ .build();
+ }
+ return Response.ok(this.yaml.dump(translateAnomalyFunctionToYaml(anomalyFunctionDTO))).build();
+ }
+
+ @GET
+ public Response getYamlFromLegacyAlert(long alertId) throws Exception {
+ AlertConfigDTO alertConfigDTO = this.alertConfigDAO.findById(alertId);
+ if (alertConfigDTO == null) {
+ return Response.status(Response.Status.BAD_REQUEST)
+ .entity(ImmutableMap.of("message", "Legacy alert cannot be found for ID "+ alertId))
+ .build();
+ }
+ return Response.ok(this.yaml.dump(translateAlertToYaml(alertConfigDTO))).build();
+ }
+
+ @POST
+ public Response migrateApplication(@QueryParam("id") String application) throws Exception {
+ List<AlertConfigDTO> alertConfigDTOList = alertConfigDAO.findByPredicate(Predicate.EQ("application", application));
+ Map<String, String> responseMessage = new HashMap<>();
+
+ for (AlertConfigDTO alertConfigDTO : alertConfigDTOList) {
+ LOGGER.info(String.format("[MIG] Migrating alert %d_%s", alertConfigDTO.getId(), alertConfigDTO.getName()));
+ try {
+ // Skip if the alert is already migrated
+ if (alertConfigDTO.getName().contains(MIGRATED_TAG)) {
+ LOGGER.info(String.format("[MIG] Alert %d is already migrated. Skipping!", alertConfigDTO.getId()));
+ continue;
+ }
+
+ // Translate the old alert and capture the state
+ Map<String, Object> detectionAlertYaml = translateAlertToYaml(alertConfigDTO);
+
+ // Migrate all the subscribed anomaly functions. Note that this will update the state of old anomaly functions.
+ List<Long> detectionIds = ConfigUtils.getLongs(alertConfigDTO.getEmailConfig().getFunctionIds());
+ for (long detectionId : detectionIds) {
+ migrateLegacyAnomalyFunction(detectionId);
+ }
+
+ // Migrate the alert/notification group
+ DetectionAlertConfigDTO alertConfig = new YamlDetectionAlertConfigTranslator(detectionConfigDAO).translate(detectionAlertYaml);
+ detectionAlertConfigDAO.save(alertConfig);
+ if (alertConfig.getId() == null) {
+ throw new RuntimeException("Error while saving the migrated alert config for " + alertConfigDTO.getName());
+ }
+
+ // Update migration status and disable the old alert
+ alertConfigDTO.setName(alertConfigDTO.getName() + MIGRATED_TAG + "_" + alertConfig.getId());
+ alertConfigDTO.setActive(false);
+ int affectedRows = alertConfigDAO.update(alertConfigDTO);
+ if (affectedRows == 0) {
+ throw new RuntimeException("Alert migrated successfully but failed to disable and update the migration status"
+ + " of the old alert. Recommend doing it manually." + " Migrated alert id " + alertConfig.getId());
+ }
+ LOGGER.info(String.format("[MIG] Successfully migrated alert %d_%s", alertConfigDTO.getId(), alertConfigDTO.getName()));
+ } catch (Exception e) {
+ LOGGER.error("[MIG] Failed to migrate alert ID {} name {}.", alertConfigDTO.getId(), alertConfigDTO.getName(), e);
+ responseMessage.put("message", String.format("Failed to migrate alert ID %d name %s. Exception %s",
+ alertConfigDTO.getId(), alertConfigDTO.getName(), ExceptionUtils.getStackTrace(e)));
+ // Skip migrating this alert and move on to the next
+ }
+ }
+
+ if (responseMessage.isEmpty()) {
+ return Response.ok("Application " + application + " has been successfully migrated").build();
+ } else {
+ return Response.status(Response.Status.OK).entity(responseMessage).build();
+ }
+ }
+
+ @POST
+ public Response migrateAnomalyFunction(@QueryParam("id") long anomalyFunctionId) throws Exception {
+ return Response.ok(migrateLegacyAnomalyFunction(anomalyFunctionId)).build();
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionAlertRegistry.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionAlertRegistry.java
index 89a9380..8929cde 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionAlertRegistry.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionAlertRegistry.java
@@ -51,16 +51,24 @@ public class DetectionAlertRegistry {
// Alert Filter Type Map
private static final Map<String, String> ALERT_FILTER_MAP = new HashMap<>();
- private static final DetectionAlertRegistry INSTANCE = new DetectionAlertRegistry();
+ private static DetectionAlertRegistry INSTANCE;
public static DetectionAlertRegistry getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE = new DetectionAlertRegistry();
+ }
+
return INSTANCE;
}
+ private DetectionAlertRegistry () {
+ init();
+ }
+
/**
* Read all the alert schemes and suppressors and initialize the registry.
*/
- public static void init() {
+ private static void init() {
try {
Reflections reflections = new Reflections();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
index 44fcb72..705e6cd 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
@@ -33,6 +33,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.collections.MapUtils;
+import org.junit.Assert;
import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,21 +53,24 @@ public class DetectionRegistry {
private static final String KEY_CLASS_NAME = "className";
private static final String KEY_ANNOTATION = "annotation";
- /**
- * Singleton
- */
+ private static DetectionRegistry INSTANCE;
+
public static DetectionRegistry getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE = new DetectionRegistry();
+ }
+
return INSTANCE;
}
- public static void registerComponent(String className, String type) {
- REGISTRY_MAP.put(type, ImmutableMap.of(KEY_CLASS_NAME, className));
+ private DetectionRegistry () {
+ init();
}
/**
* Read all the components, tune, and yaml annotations and initialize the registry.
*/
- public static void init() {
+ private static void init() {
try {
Reflections reflections = new Reflections();
// register components
@@ -100,7 +104,13 @@ public class DetectionRegistry {
}
}
- private static final DetectionRegistry INSTANCE = new DetectionRegistry();
+ public static void registerComponent(String className, String type) {
+ REGISTRY_MAP.put(type, ImmutableMap.of(KEY_CLASS_NAME, className));
+ }
+
+ public static void registerYamlConvertor(String className, String type) {
+ YAML_MAP.put(type, className);
+ }
/**
* Look up the class name for a given component
@@ -148,4 +158,8 @@ public class DetectionRegistry {
}
return annotations;
}
+
+ public String printAnnotations() {
+ return String.join(", ", YAML_MAP.keySet());
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
index 29fcab4..8f2e0de 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/DetectionAlertConfigValidator.java
@@ -23,6 +23,8 @@ import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.junit.Assert;
import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
@@ -31,7 +33,7 @@ public class DetectionAlertConfigValidator extends ConfigValidator {
private static final DetectionAlertConfigValidator INSTANCE = new DetectionAlertConfigValidator();
private static final String PROP_CLASS_NAME = "className";
-
+
public static DetectionAlertConfigValidator getInstance() {
return INSTANCE;
}
@@ -87,7 +89,7 @@ public class DetectionAlertConfigValidator extends ConfigValidator {
return false;
}
// Application name should be valid
- if (super.applicationDAO.findByName(alertConfig.getApplication()).size() == 0) {
+ if (DAORegistry.getInstance().getApplicationDAO().findByName(alertConfig.getApplication()).size() == 0) {
responseMessage.put("message", "Application name doesn't exist in our registry. Please use an existing"
+ " application name. You may search for registered applications from the ThirdEye dashboard or reach out"
+ " to ask_thirdeye if you wish to setup a new application.");
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
index e245b59..0c34b51 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionAlertConfigTranslator.java
@@ -24,6 +24,7 @@ import com.google.common.base.Preconditions;
import java.util.stream.Collectors;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
@@ -41,6 +42,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.collections.MapUtils;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
/**
@@ -57,20 +60,20 @@ public class YamlDetectionAlertConfigTranslator {
public static final String PROP_FROM = "fromAddress";
public static final String PROP_EMAIL_SUBJECT_TYPE = "emailSubjectStyle";
public static final String PROP_ALERT_SCHEMES = "alertSchemes";
-
+ public static final String PROP_DETECTION_NAMES = "subscribedDetections";
public static final String PROP_TYPE = "type";
public static final String PROP_CLASS_NAME = "className";
+
static final String PROP_PARAM = "params";
static final String PROP_ALERT_SUPPRESSORS = "alertSuppressors";
static final String PROP_REFERENCE_LINKS = "referenceLinks";
- static final String PROP_ONLY_FETCH_LEGACY_ANOMALIES = "onlyFetchLegacyAnomalies";
- static final String PROP_DETECTION_NAMES = "subscribedDetections";
-
- static final String PROP_DIMENSION = "dimension";
- static final String PROP_DIMENSION_RECIPIENTS = "dimensionRecipients";
static final String PROP_TIME_WINDOWS = "timeWindows";
static final String CRON_SCHEDULE_DEFAULT = "0 0/5 * * * ? *"; // Every 5 min
+ private static final String PROP_ONLY_FETCH_LEGACY_ANOMALIES = "onlyFetchLegacyAnomalies";
+ private static final String PROP_DIMENSION = "dimension";
+ private static final String PROP_DIMENSION_RECIPIENTS = "dimensionRecipients";
+
private static final DetectionAlertRegistry DETECTION_ALERT_REGISTRY = DetectionAlertRegistry.getInstance();
private static final Set<String> PROPERTY_KEYS = new HashSet<>(
Arrays.asList(PROP_RECIPIENTS, PROP_DIMENSION, PROP_DIMENSION_RECIPIENTS));
@@ -200,11 +203,17 @@ public class YamlDetectionAlertConfigTranslator {
alertConfigDTO.setProperties(buildAlerterProperties(yamlAlertConfig, detectionConfigIds));
Map<Long, Long> vectorClocks = new HashMap<>();
+ long currentTimestamp = System.currentTimeMillis();
for (long detectionConfigId : detectionConfigIds) {
- vectorClocks.put(detectionConfigId, 0L);
+ vectorClocks.put(detectionConfigId, currentTimestamp);
}
alertConfigDTO.setVectorClocks(vectorClocks);
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+ options.setPrettyFlow(true);
+ alertConfigDTO.setYaml(new Yaml(options).dump(yamlAlertConfig));
+
return alertConfigDTO;
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
index bbb3aa5..6bbf0e3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlDetectionTranslatorLoader.java
@@ -25,6 +25,7 @@ import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry
import java.lang.reflect.Constructor;
import java.util.Map;
+
/**
* Loads the detection config translator fo a pipeline type
*/
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 105ec79..6fda48e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -187,6 +187,10 @@ public class YamlResource {
return Response.ok().entity(ImmutableMap.of("detectionConfigId", detectionConfig.getId(), "detectionAlertConfigId", alertConfig.getId())).build();
}
+ public DetectionConfigDTO translateToDetectionConfig(Map<String, Object> yamlConfig, Map<String, String> responseMessage) {
+ return buildDetectionConfigFromYaml(0, 0, yamlConfig, null, responseMessage);
+ }
+
/*
* Build the detection config from a yaml.
* Returns null if building or validation failed. Error messages stored in responseMessage.
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java
index 9073043..550c997 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResourceTest.java
@@ -19,61 +19,121 @@
package org.apache.pinot.thirdeye.detection;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import javax.ws.rs.core.Response;
import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
+import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
+import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry;
+import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
+import org.apache.pinot.thirdeye.detection.components.PercentageChangeRuleDetector;
+import org.apache.pinot.thirdeye.detection.components.RuleBaselineProvider;
+import org.apache.pinot.thirdeye.detection.components.ThresholdRuleAnomalyFilter;
+import org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
+import org.apache.pinot.thirdeye.detection.yaml.CompositePipelineConfigTranslator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import static org.apache.pinot.thirdeye.detection.DetectionMigrationResource.*;
import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
public class DetectionMigrationResourceTest {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DetectionMigrationResourceTest.class);
private DAOTestBase testDAOProvider;
+ private MetricConfigManager metricDAO;
private DatasetConfigManager datasetDAO;
private MergedAnomalyResultManager anomalyDAO;
private AnomalyFunctionManager anomalyFunctionDAO;
private AlertConfigManager alertConfigDAO;
+ private ApplicationManager applicationDAO;
private DetectionConfigManager detectionConfigDAO;
private DetectionAlertConfigManager detectionAlertConfigDAO;
private DetectionMigrationResource migrationResource;
- @BeforeMethod
- public void beforeMethod() {
+ private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ @BeforeMethod(alwaysRun = true)
+ public void setup() {
this.testDAOProvider = DAOTestBase.getInstance();
+ this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
this.anomalyFunctionDAO = DAORegistry.getInstance().getAnomalyFunctionDAO();
this.alertConfigDAO = DAORegistry.getInstance().getAlertConfigDAO();
this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager();
this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager();
+ this.applicationDAO = DAORegistry.getInstance().getApplicationDAO();
+
+ migrationResource = new DetectionMigrationResource(
+ anomalyFunctionDAO, alertConfigDAO, detectionConfigDAO, detectionAlertConfigDAO, datasetDAO, anomalyDAO);
+
+ DetectionRegistry.registerYamlConvertor(CompositePipelineConfigTranslator.class.getName(), "COMPOSITE");
+ DetectionRegistry.registerComponent(PercentageChangeRuleDetector.class.getName(), "PERCENTAGE_RULE");
+ DetectionRegistry.registerComponent(RuleBaselineProvider.class.getName(), "RULE_BASELINE");
+ DetectionRegistry.registerComponent(ThresholdRuleDetector.class.getName(), "THRESHOLD");
+ DetectionRegistry.registerComponent(ThresholdRuleAnomalyFilter.class.getName(), "THRESHOLD_RULE_FILTER");
+
+ DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL", "EmailClass");
+ DetectionAlertRegistry.getInstance().registerAlertFilter("DEFAULT_ALERTER_PIPELINE", "RECIPIENTClass");
- migrationResource = new DetectionMigrationResource(anomalyFunctionDAO, detectionConfigDAO, datasetDAO);
+ MetricConfigDTO metricConfigDTO = new MetricConfigDTO();
+ metricConfigDTO.setName("test_metric");
+ metricConfigDTO.setDataset("test_collection");
+ metricConfigDTO.setActive(true);
+ metricConfigDTO.setAlias("test_collection::test_metric");
+ metricDAO.save(metricConfigDTO);
+
+ DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
+ datasetConfigDTO.setDataset("test_collection");
+ datasetConfigDTO.setNonAdditiveBucketSize(1);
+ datasetConfigDTO.setNonAdditiveBucketUnit(TimeUnit.DAYS);
+ datasetDAO.save(datasetConfigDTO);
}
@AfterMethod(alwaysRun = true)
- public void afterMethod() {
+ public void cleanup() {
this.testDAOProvider.cleanup();
}
@Test
public void testTranslateAlertToYaml() throws Exception {
+ AnomalyFunctionDTO anomalyFunction1 = new AnomalyFunctionDTO();
+ anomalyFunction1.setFunctionName("function1");
+ long id1 = anomalyFunctionDAO.save(anomalyFunction1);
+
+ AnomalyFunctionDTO anomalyFunction2 = new AnomalyFunctionDTO();
+ anomalyFunction2.setFunctionName("function2");
+ long id2 = anomalyFunctionDAO.save(anomalyFunction2);
+
AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
alertConfigDTO.setActive(true);
alertConfigDTO.setName("test_notification");
@@ -87,7 +147,7 @@ public class DetectionMigrationResourceTest {
Collections.singleton("bcc@test")));
AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig();
- emailConfig.setFunctionIds(Arrays.asList(1l, 2l, 3l, 4l));
+ emailConfig.setFunctionIds(Arrays.asList(id1, id2));
emailConfig.setAnomalyWatermark(9999);
alertConfigDTO.setEmailConfig(emailConfig);
@@ -100,12 +160,126 @@ public class DetectionMigrationResourceTest {
Assert.assertEquals(yamlMap.get(PROP_EMAIL_SUBJECT_TYPE), AlertConfigBean.SubjectType.ALERT);
Assert.assertEquals(yamlMap.get(PROP_FROM), "test@thirdeye.com");
Assert.assertEquals(yamlMap.get(PROP_TYPE), "DEFAULT_ALERTER_PIPELINE");
- Assert.assertEquals(yamlMap.get(PROP_DETECTION_CONFIG_IDS), Arrays.asList(1l, 2l, 3l, 4l));
+ Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_DETECTION_NAMES)).size(), 2);
Assert.assertNotNull(yamlMap.get(PROP_ALERT_SCHEMES));
- Assert.assertEquals(((Map<String, Object>) yamlMap.get(PROP_ALERT_SCHEMES)).get(PROP_TYPE), "EMAIL");
+ Assert.assertEquals(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).size(), 1);
+ Assert.assertEquals(ConfigUtils.getMap(ConfigUtils.getList(yamlMap.get(PROP_ALERT_SCHEMES)).get(0)).get(PROP_TYPE), "EMAIL");
Assert.assertNotNull(yamlMap.get(PROP_RECIPIENTS));
- Assert.assertEquals(((Map<String, Object>) yamlMap.get(PROP_RECIPIENTS)).get("to"), Collections.singleton("to@test"));
- Assert.assertEquals(((Map<String, Object>) yamlMap.get(PROP_RECIPIENTS)).get("cc"), Collections.singleton("cc@test"));
- Assert.assertEquals(((Map<String, Object>) yamlMap.get(PROP_RECIPIENTS)).get("bcc"), Collections.singleton("bcc@test"));
+ Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("to")), Collections.singleton("to@test"));
+ Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("cc")), Collections.singleton("cc@test"));
+ Assert.assertEquals((ConfigUtils.getMap(yamlMap.get(PROP_RECIPIENTS)).get("bcc")), Collections.singleton("bcc@test"));
+ }
+
+ @Test
+ public void testMigrateWoWAnomalyFunction() throws Exception {
+ AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-1.json"), AnomalyFunctionDTO.class);
+ long oldID = anomalyFunctionDAO.save(actual);
+
+ MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO();
+ mergedAnomalyResultDTO.setFunction(actual);
+ anomalyDAO.save(mergedAnomalyResultDTO);
+
+ Response responseId = migrationResource.migrateAnomalyFunction(oldID);
+ long newID = (long) responseId.getEntity();
+
+ AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID);
+ DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID);
+
+ // Verify if the migration status is updated correctly and the old detection is disabled.
+ Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID);
+ Assert.assertFalse(legacyAnomalyFunction.getIsActive());
+
+ // Verify if the anomaly is pointing to the migrated anomaly function
+ List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll();
+ Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID);
+
+ // Assert the migrated object
+ DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-1.json"), DetectionConfigDTO.class);
+ expected.setId(migratedAnomalyFunction.getId());
+ expected.setVersion(migratedAnomalyFunction.getVersion());
+ expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy());
+ expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy());
+ expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp());
+
+ Assert.assertEquals(migratedAnomalyFunction, expected);
+ }
+
+ @Test
+ public void testMinMaxAnomalyFunction() throws Exception {
+ AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class);
+ long oldID = anomalyFunctionDAO.save(actual);
+
+ MergedAnomalyResultDTO mergedAnomalyResultDTO = new MergedAnomalyResultDTO();
+ mergedAnomalyResultDTO.setFunction(actual);
+ anomalyDAO.save(mergedAnomalyResultDTO);
+
+ Response responseId = migrationResource.migrateAnomalyFunction(oldID);
+ long newID = (long) responseId.getEntity();
+
+ AnomalyFunctionDTO legacyAnomalyFunction = anomalyFunctionDAO.findById(oldID);
+ DetectionConfigDTO migratedAnomalyFunction = detectionConfigDAO.findById(newID);
+
+ // Verify if the migration status is updated correctly and the old detection is disabled.
+ Assert.assertEquals(legacyAnomalyFunction.getFunctionName(), "test_function_thirdeye_migrated_" + newID);
+ Assert.assertFalse(legacyAnomalyFunction.getIsActive());
+
+ // Verify if the anomaly is pointing to the migrated anomaly function
+ List<MergedAnomalyResultDTO> mergedAnomalyResultDTOList = anomalyDAO.findAll();
+ Assert.assertEquals(mergedAnomalyResultDTOList.get(0).getDetectionConfigId().longValue(), newID);
+
+ // Assert the migrated object
+ DetectionConfigDTO expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("migrated-detection-config-2.json"), DetectionConfigDTO.class);
+ expected.setId(migratedAnomalyFunction.getId());
+ expected.setVersion(migratedAnomalyFunction.getVersion());
+ expected.setCreatedBy(migratedAnomalyFunction.getCreatedBy());
+ expected.setUpdatedBy(migratedAnomalyFunction.getUpdatedBy());
+ expected.setLastTimestamp(migratedAnomalyFunction.getLastTimestamp());
+ Assert.assertEquals(migratedAnomalyFunction, expected);
+ }
+
+ @Test
+ public void testMigrateApplication() throws Exception {
+ AnomalyFunctionDTO actual = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("legacy-anomaly-function-2.json"), AnomalyFunctionDTO.class);
+ long id1 = anomalyFunctionDAO.save(actual);
+
+ AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
+ alertConfigDTO.setActive(true);
+ alertConfigDTO.setName("test_notification");
+ alertConfigDTO.setApplication("test_application");
+ alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.ALERT);
+ alertConfigDTO.setCronExpression("0 0 14 * * ? *");
+ alertConfigDTO.setFromAddress("test@thirdeye.com");
+ alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients(
+ Collections.singleton("to@test"),
+ Collections.singleton("cc@test"),
+ Collections.singleton("bcc@test")));
+ AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig();
+ emailConfig.setFunctionIds(Arrays.asList(id1));
+ emailConfig.setAnomalyWatermark(9999);
+ alertConfigDTO.setEmailConfig(emailConfig);
+ alertConfigDAO.save(alertConfigDTO);
+
+ AlertConfigDTO alertConfigDTOMigrated = new AlertConfigDTO();
+ alertConfigDTOMigrated.setActive(false);
+ alertConfigDTOMigrated.setName("test_notification" + MIGRATED_TAG);
+ alertConfigDTOMigrated.setApplication("test_application");
+ alertConfigDTOMigrated.setSubjectType(AlertConfigBean.SubjectType.ALERT);
+ alertConfigDTOMigrated.setCronExpression("0 0 14 * * ? *");
+ alertConfigDTOMigrated.setFromAddress("test@thirdeye.com");
+ alertConfigDTOMigrated.setReceiverAddresses(new DetectionAlertFilterRecipients(
+ Collections.singleton("to@test"),
+ Collections.singleton("cc@test"),
+ Collections.singleton("bcc@test")));
+ alertConfigDTO.setEmailConfig(emailConfig);
+ alertConfigDAO.save(alertConfigDTOMigrated);
+
+ ApplicationDTO applicationDTO = new ApplicationDTO();
+ applicationDTO.setApplication("test_application");
+ applicationDTO.setRecipients("test@thirdeye.com");
+ applicationDAO.save(applicationDTO);
+
+ Response response = migrationResource.migrateApplication("test_application");
+
+ Assert.assertEquals(response.getEntity(), "Application test_application has been successfully migrated");
}
-}
\ No newline at end of file
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-1.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-1.json
new file mode 100644
index 0000000..2a686e4
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-1.json
@@ -0,0 +1,27 @@
+{
+ "collection" : "test_collection",
+ "functionName" : "test_function",
+ "metric" : "test_metric",
+ "metrics" : [ "test_metric" ],
+ "metricFunction" : "SUM",
+ "type" : "WEEK_OVER_WEEK_RULE",
+ "isActive" : true,
+ "properties" : "comments=2PM PDT;baseline=w/w;changeThreshold=-0.05;averageVolumeThreshold=0;",
+ "cron" : "0 30 20 * * ?",
+ "frequency" : {
+ "size" : 1,
+ "unit" : "DAYS"
+ },
+ "bucketSize" : 1,
+ "bucketUnit" : "DAYS",
+ "windowSize" : 7,
+ "windowUnit" : "DAYS",
+ "windowDelay" : 4,
+ "windowDelayUnit" : "HOURS",
+ "exploreDimensions" : "platform",
+ "filters" : "platform=desktop;platform=tablet",
+ "metricId" : 0,
+ "requiresCompletenessCheck" : true,
+ "topicMetric" : "test_metric",
+ "toCalculateGlobalMetric" : false
+}
\ No newline at end of file
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-2.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-2.json
new file mode 100644
index 0000000..90c1f23
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/legacy-anomaly-function-2.json
@@ -0,0 +1,26 @@
+{
+ "collection" : "test_collection",
+ "functionName" : "test_function",
+ "metric" : "test_metric",
+ "metrics" : [ "test_metric" ],
+ "metricFunction" : "SUM",
+ "type" : "MIN_MAX_THRESHOLD",
+ "isActive" : true,
+ "properties" : "min=0.70;",
+ "cron" : "0 0 * * * ?",
+ "frequency" : {
+ "size" : 1,
+ "unit" : "DAYS"
+ },
+ "bucketSize" : 1,
+ "bucketUnit" : "DAYS",
+ "windowSize" : 1,
+ "windowUnit" : "DAYS",
+ "windowDelay" : 10,
+ "windowDelayUnit" : "HOURS",
+ "filters" : "score_challenge_type=Captcha Challenge",
+ "metricId" : 0,
+ "requiresCompletenessCheck" : true,
+ "topicMetric" : "test_metric",
+ "toCalculateGlobalMetric" : false
+}
\ No newline at end of file
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-1.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-1.json
new file mode 100644
index 0000000..bb112e5
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-1.json
@@ -0,0 +1,47 @@
+{
+ "cron":"0 0 14 * * ? *",
+ "name":"test_function",
+ "properties":{
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nested":[
+ {
+ "nestedMetricUrns":[
+ "thirdeye:metric:1:platform%3Ddesktop:platform%3Dtablet"
+ ],
+ "className":"org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "metricUrn":"thirdeye:metric:1:platform%3Ddesktop:platform%3Dtablet",
+ "nested":[
+ {
+ "baselineValueProvider":"$detection_rule1:RULE_BASELINE",
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "nested":[
+ {
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }
+ ],
+ "detector":"$detection_rule1:PERCENTAGE_RULE"
+ }
+ ],
+ "dimensions":[
+ "platform"
+ ]
+ }
+ ]
+ },
+ "active":true,
+ "yaml":"detectionName: test_function\nmetric: test_metric\ndataset: test_collection\npipelineType: Composite\ndimensionExploration:\n dimensions:\n - platform\nfilters:\n platform:\n - desktop\n - tablet\nrules:\n- detection:\n - name: detection_rule1\n type: PERCENTAGE_RULE\n params:\n percentageChange: 0.05\n pattern: DOWN\n",
+ "componentSpecs":{
+ "detection_rule1:PERCENTAGE_RULE":{
+ "percentageChange":0.05,
+ "pattern":"DOWN",
+ "className":"org.apache.pinot.thirdeye.detection.components.PercentageChangeRuleDetector"
+ },
+ "detection_rule1:RULE_BASELINE":{
+ "percentageChange":0.05,
+ "pattern":"DOWN",
+ "className":"org.apache.pinot.thirdeye.detection.components.RuleBaselineProvider"
+ }
+ },
+ "components":{
+ }
+}
\ No newline at end of file
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-2.json b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-2.json
new file mode 100644
index 0000000..e5b0054
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/migrated-detection-config-2.json
@@ -0,0 +1,43 @@
+{
+ "cron":"0 0 14 * * ? *",
+ "name":"test_function",
+ "lastTimestamp":1547772179159,
+ "properties":{
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper",
+ "nested":[
+ {
+ "nestedMetricUrns":[
+ "thirdeye:metric:1:score_challenge_type%3DCaptcha%20Challenge"
+ ],
+ "className":"org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper",
+ "nested":[
+ {
+ "baselineValueProvider":"$detection_rule1:RULE_BASELINE",
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper",
+ "nested":[
+ {
+ "className":"org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper"
+ }
+ ],
+ "detector":"$detection_rule1:THRESHOLD"
+ }
+ ]
+ }
+ ]
+ },
+ "active":true,
+ "yaml":"detectionName: test_function\nmetric: test_metric\ndataset: test_collection\npipelineType: Composite\nfilters:\n score_challenge_type:\n - Captcha Challenge\nrules:\n- detection:\n - name: detection_rule1\n type: THRESHOLD\n params:\n min: '0.70'\n",
+ "componentSpecs":{
+ "detection_rule1:THRESHOLD":{
+ "min":"0.70",
+ "className":"org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector"
+ },
+ "detection_rule1:RULE_BASELINE":{
+ "min":"0.70",
+ "className":"org.apache.pinot.thirdeye.detection.components.RuleBaselineProvider"
+ }
+ },
+ "components":{
+
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org