You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/06/11 21:58:50 UTC
[incubator-pinot] branch master updated: [TE] Translator code
refactor to accommodate entity config translation (#4286)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 38a1804 [TE] Translator code refactor to accommodate entity config translation (#4286)
38a1804 is described below
commit 38a1804efd97a10ac9968e7939207b521bc8db2c
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Tue Jun 11 14:58:43 2019 -0700
[TE] Translator code refactor to accommodate entity config translation (#4286)
Changes:
- Translator should be like a black box - takes raw config and returns detection object.
- Remove pipelineType and Translator Loader in favor of a single Detection Translator.
- Renamed YamlDetectionConfigTranslator to DetectionConfigTranslator
- Renamed YamlDetectionAlertConfigTranslator to SubscriptionConfigTranslator
---
.../api/application/ApplicationResource.java | 2 +-
.../pinot/thirdeye/detection/annotation/Yaml.java | 39 -------
.../annotation/registry/DetectionRegistry.java | 12 ---
.../onboard/YamlOnboardingTaskRunner.java | 6 --
.../validators/SubscriptionConfigValidator.java | 2 +-
.../thirdeye/detection/yaml/YamlResource.java | 84 +++++----------
.../yaml/translator/ConfigTranslator.java | 20 ++--
...nslator.java => DetectionConfigTranslator.java} | 120 ++++++++++++---------
...ator.java => SubscriptionConfigTranslator.java} | 45 ++++----
.../translator/YamlDetectionConfigTranslator.java | 71 ------------
.../translator/YamlDetectionTranslatorLoader.java | 45 --------
.../thirdeye/detection/yaml/YamlResourceTest.java | 4 +-
...est.java => DetectionConfigTranslatorTest.java} | 25 ++---
.../YamlDetectionAlertConfigTranslatorTest.java | 8 +-
14 files changed, 147 insertions(+), 336 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/application/ApplicationResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/application/ApplicationResource.java
index 7a2edf4..2f4cbe8 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/application/ApplicationResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/api/application/ApplicationResource.java
@@ -45,7 +45,7 @@ import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionAlertConfigTranslator.*;
+import static org.apache.pinot.thirdeye.detection.yaml.translator.SubscriptionConfigTranslator.*;
@Path(value = "/application")
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/Yaml.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/Yaml.java
deleted file mode 100644
index f6e4b5e..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/Yaml.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection.annotation;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Inherited;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-
-/**
- * Yaml annotation
- * Registers the yaml translator to the translator factory.
- */
-@Target({ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Inherited
-public @interface Yaml {
- @JsonProperty String pipelineType() default "";
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
index c14ff15..2382ee1 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
@@ -23,10 +23,8 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.apache.pinot.thirdeye.detection.annotation.Components;
import org.apache.pinot.thirdeye.detection.annotation.Tune;
-import org.apache.pinot.thirdeye.detection.annotation.Yaml;
import org.apache.pinot.thirdeye.detection.spi.components.BaseComponent;
import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
-import org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionConfigTranslator;
import java.lang.annotation.Annotation;
import java.util.ArrayList;
import java.util.HashMap;
@@ -91,16 +89,6 @@ public class DetectionRegistry {
}
}
}
- // register yaml translators
- Set<Class<? extends YamlDetectionConfigTranslator>> yamlConverterClasses =
- reflections.getSubTypesOf(YamlDetectionConfigTranslator.class);
- for (Class<? extends YamlDetectionConfigTranslator> clazz : yamlConverterClasses) {
- for (Annotation annotation : clazz.getAnnotations()) {
- if (annotation instanceof Yaml) {
- YAML_MAP.put(((Yaml) annotation).pipelineType(), clazz.getName());
- }
- }
- }
} catch (Exception e) {
LOG.warn("initialize detection registry error", e);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java
index 95456fb..25f91d1 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/onboard/YamlOnboardingTaskRunner.java
@@ -47,10 +47,8 @@ import org.apache.pinot.thirdeye.detection.DetectionPipeline;
import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
import org.apache.pinot.thirdeye.detection.yaml.DetectionConfigTuner;
-import org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionTranslatorLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.Yaml;
/**
@@ -65,8 +63,6 @@ public class YamlOnboardingTaskRunner implements TaskRunner {
private final EvaluationManager evaluationDAO;
private final DetectionPipelineLoader loader;
private final DataProvider provider;
- private final YamlDetectionTranslatorLoader translatorLoader;
- private final Yaml yaml;
public YamlOnboardingTaskRunner() {
@@ -74,8 +70,6 @@ public class YamlOnboardingTaskRunner implements TaskRunner {
this.detectionDAO = DAORegistry.getInstance().getDetectionConfigManager();
this.anomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
this.evaluationDAO = DAORegistry.getInstance().getEvaluationManager();
- this.translatorLoader = new YamlDetectionTranslatorLoader();
- this.yaml = new Yaml();
MetricConfigManager metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
DatasetConfigManager datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
index d0a35cc..92d7b05 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/validators/SubscriptionConfigValidator.java
@@ -28,7 +28,7 @@ import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.quartz.CronExpression;
-import static org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionAlertConfigTranslator.*;
+import static org.apache.pinot.thirdeye.detection.yaml.translator.SubscriptionConfigTranslator.*;
public class SubscriptionConfigValidator implements ConfigValidator<DetectionAlertConfigDTO> {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index e3806ea..019c7e6 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -33,7 +33,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
import javax.validation.constraints.NotNull;
import javax.ws.rs.Consumes;
@@ -80,16 +79,15 @@ import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
import org.apache.pinot.thirdeye.detection.validators.SubscriptionConfigValidator;
-import org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionAlertConfigTranslator;
-import org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionConfigTranslator;
-import org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionTranslatorLoader;
+import org.apache.pinot.thirdeye.detection.yaml.translator.DetectionConfigTranslator;
+import org.apache.pinot.thirdeye.detection.yaml.translator.SubscriptionConfigTranslator;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
-import static org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionAlertConfigTranslator.*;
+import static org.apache.pinot.thirdeye.detection.yaml.translator.SubscriptionConfigTranslator.*;
@Path("/yaml")
@@ -106,7 +104,6 @@ public class YamlResource {
private final DetectionConfigManager detectionConfigDAO;
private final DetectionAlertConfigManager detectionAlertConfigDAO;
- private final YamlDetectionTranslatorLoader translatorLoader;
private final DetectionConfigValidator detectionValidator;
private final SubscriptionConfigValidator subscriptionValidator;
private final DataProvider provider;
@@ -122,7 +119,6 @@ public class YamlResource {
public YamlResource() {
this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager();
this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager();
- this.translatorLoader = new YamlDetectionTranslatorLoader();
this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
this.eventDAO = DAORegistry.getInstance().getEventDAO();
@@ -147,9 +143,9 @@ public class YamlResource {
}
/*
- * Build the detection config from a yaml.
+ * Helper method to build the detection config from a yaml.
*/
- private DetectionConfigDTO buildDetectionConfigFromYaml(long tuningStartTime, long tuningEndTime, Map<String, Object> yamlConfig,
+ private DetectionConfigDTO buildDetectionConfigFromYaml(long tuningStartTime, long tuningEndTime, String yamlConfig,
DetectionConfigDTO existingConfig) {
// Configure the tuning window
@@ -159,15 +155,8 @@ public class YamlResource {
tuningStartTime = tuningEndTime - TimeUnit.DAYS.toMillis(28);
}
- YamlDetectionConfigTranslator detectionConfigTranslator;
- try {
- detectionConfigTranslator = this.translatorLoader.from(yamlConfig, this.provider);
- } catch (Exception e) {
- throw new IllegalArgumentException("Unable to instantiate the detection pipeline.", e);
- }
-
// Translate the raw yaml config to detection config object
- DetectionConfigDTO config = detectionConfigTranslator.translate();
+ DetectionConfigDTO config = new DetectionConfigTranslator(yamlConfig, this.provider).translate();
if (existingConfig != null) {
config.setId(existingConfig.getId());
@@ -293,14 +282,11 @@ public class YamlResource {
return createDetectionPipeline(yamlDetectionConfig, 0, 0);
}
- long createDetectionPipeline(String yamlDetectionConfig, long startTime, long endTime)
+ long createDetectionPipeline(String payload, long startTime, long endTime)
throws IllegalArgumentException {
- Preconditions.checkArgument(StringUtils.isNotBlank(yamlDetectionConfig), "The Yaml Payload in the request is empty.");
- // Translate config from YAML to detection config (JSON)
- Map<String, Object> newDetectionConfigMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newDetectionConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlDetectionConfig)));
- DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(startTime, endTime, newDetectionConfigMap, null);
- detectionConfig.setYaml(yamlDetectionConfig);
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml Payload in the request is empty.");
+
+ DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(startTime, endTime, payload, null);
// Check for duplicates
List<DetectionConfigDTO> detectionConfigDTOS = detectionConfigDAO
@@ -357,18 +343,14 @@ public class YamlResource {
updateDetectionPipeline(detectionID, yamlDetectionConfig, 0, 0);
}
- void updateDetectionPipeline(long detectionID, String yamlDetectionConfig, long startTime, long endTime)
+ void updateDetectionPipeline(long detectionID, String payload, long startTime, long endTime)
throws IllegalArgumentException {
DetectionConfigDTO existingDetectionConfig = this.detectionConfigDAO.findById(detectionID);
DetectionConfigDTO detectionConfig;
Preconditions.checkNotNull(existingDetectionConfig, "Cannot find detection pipeline " + detectionID);
- Preconditions.checkArgument(StringUtils.isNotBlank(yamlDetectionConfig), "The Yaml Payload in the request is empty.");
- // Translate config from YAML to detection config (JSON)
- TreeMap<String, Object> newDetectionConfigMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newDetectionConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlDetectionConfig)));
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml Payload in the request is empty.");
try {
- detectionConfig = buildDetectionConfigFromYaml(startTime, endTime, newDetectionConfigMap, existingDetectionConfig);
- detectionConfig.setYaml(yamlDetectionConfig);
+ detectionConfig = buildDetectionConfigFromYaml(startTime, endTime, payload, existingDetectionConfig);
// Validate updated config before saving it
detectionValidator.validateUpdatedConfig(detectionConfig, existingDetectionConfig);
@@ -378,9 +360,10 @@ public class YamlResource {
} finally {
// If it is to disable the pipeline then no need to do validation and parsing.
// It is possible that the metric or dataset was deleted so the validation will fail.
- if (!MapUtils.getBooleanValue(newDetectionConfigMap, PROP_ACTIVE, true)) {
+ Map<String, Object> detectionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
+ if (!MapUtils.getBooleanValue(detectionConfigMap, PROP_ACTIVE, true)) {
existingDetectionConfig.setActive(false);
- existingDetectionConfig.setYaml(yamlDetectionConfig);
+ existingDetectionConfig.setYaml(payload);
this.detectionConfigDAO.save(existingDetectionConfig);
}
}
@@ -488,16 +471,11 @@ public class YamlResource {
return Response.ok().entity(responseMessage).build();
}
- long createSubscriptionGroup(String yamlAlertConfig) throws IllegalArgumentException {
- Preconditions.checkArgument(StringUtils.isNotBlank(yamlAlertConfig),
+ long createSubscriptionGroup(String yamlConfig) throws IllegalArgumentException {
+ Preconditions.checkArgument(StringUtils.isNotBlank(yamlConfig),
"The Yaml Payload in the request is empty.");
- // Translate config from YAML to detection alert config (JSON)
- TreeMap<String, Object> newAlertConfigMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newAlertConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlAlertConfig)));
- DetectionAlertConfigDTO alertConfig = (DetectionAlertConfigDTO)
- new YamlDetectionAlertConfigTranslator(detectionConfigDAO, newAlertConfigMap).translate();
- alertConfig.setYaml(yamlAlertConfig);
+ DetectionAlertConfigDTO alertConfig = new SubscriptionConfigTranslator(detectionConfigDAO, yamlConfig).translate();
// Check for duplicates
List<DetectionAlertConfigDTO> alertConfigDTOS = detectionAlertConfigDAO
@@ -562,25 +540,20 @@ public class YamlResource {
oldAlertConfig.setAlertSchemes(newAlertConfig.getAlertSchemes());
oldAlertConfig.setAlertSuppressors(newAlertConfig.getAlertSuppressors());
oldAlertConfig.setProperties(newAlertConfig.getProperties());
+ oldAlertConfig.setYaml(newAlertConfig.getYaml());
return oldAlertConfig;
}
- void updateSubscriptionGroup(long oldAlertConfigID, String yamlAlertConfig) {
+ void updateSubscriptionGroup(long oldAlertConfigID, String yamlConfig) {
DetectionAlertConfigDTO oldAlertConfig = this.detectionAlertConfigDAO.findById(oldAlertConfigID);
if (oldAlertConfig == null) {
throw new RuntimeException("Cannot find subscription group " + oldAlertConfigID);
}
- Preconditions.checkArgument(StringUtils.isNotBlank(yamlAlertConfig), "The Yaml Payload in the request is empty.");
+ Preconditions.checkArgument(StringUtils.isNotBlank(yamlConfig), "The Yaml Payload in the request is empty.");
- // Translate payload to detection alert config
- TreeMap<String, Object> newAlertConfigMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
- newAlertConfigMap.putAll(ConfigUtils.getMap(this.yaml.load(yamlAlertConfig)));
- DetectionAlertConfigDTO newAlertConfig = new YamlDetectionAlertConfigTranslator(detectionConfigDAO, newAlertConfigMap).translate();
-
- // Update existing alert config with the newly supplied config.
+ DetectionAlertConfigDTO newAlertConfig = new SubscriptionConfigTranslator(detectionConfigDAO, yamlConfig).translate();
DetectionAlertConfigDTO updatedAlertConfig = updateDetectionAlertConfig(oldAlertConfig, newAlertConfig);
- updatedAlertConfig.setYaml(yamlAlertConfig);
// Update watermarks to reflect changes to detectionName list in subscription config
Map<Long, Long> currentVectorClocks = updatedAlertConfig.getVectorClocks();
@@ -675,15 +648,12 @@ public class YamlResource {
Map<String, String> responseMessage = new HashMap<>();
DetectionPipelineResult result;
try {
- // Translate config from YAML to detection config (JSON)
- Map<String, Object> newDetectionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
DetectionConfigDTO detectionConfig;
-
if (existingConfig == null) {
- detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap, null);
+ detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, payload, null);
detectionConfig.setId(Long.MAX_VALUE);
} else {
- detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap, existingConfig);
+ detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, payload, existingConfig);
}
Preconditions.checkNotNull(detectionConfig);
@@ -736,9 +706,7 @@ public class YamlResource {
try {
Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml Payload in the request is empty.");
- // Translate config from YAML to detection config (JSON)
- Map<String, Object> newDetectionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
- DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap, null);
+ DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, payload, null);
Preconditions.checkNotNull(detectionConfig);
detectionConfig.setId(Long.MAX_VALUE);
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/ConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/ConfigTranslator.java
index e4d7953..b006009 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/ConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/ConfigTranslator.java
@@ -19,10 +19,13 @@
package org.apache.pinot.thirdeye.detection.yaml.translator;
+import java.util.HashMap;
import java.util.Map;
import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
+import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.validators.ConfigValidator;
+import org.yaml.snakeyaml.Yaml;
/**
@@ -30,23 +33,24 @@ import org.apache.pinot.thirdeye.detection.validators.ConfigValidator;
*/
public abstract class ConfigTranslator<T extends AbstractDTO, V extends ConfigValidator> {
- Map<String, Object> yamlConfig;
+ protected final String yamlConfig;
+ protected final ConfigValidator validator;
+ protected final Yaml yaml;
- protected ConfigValidator validator;
- protected DataProvider dataProvider;
-
- ConfigTranslator(Map<String, Object> yamlConfig, V validator) {
+ ConfigTranslator(String yamlConfig, V validator) {
this.yamlConfig = yamlConfig;
this.validator = validator;
+ this.yaml = new Yaml();
}
- abstract T translateConfig() throws IllegalArgumentException;
+ abstract T translateConfig(Map<String, Object> yamlConfigMap) throws IllegalArgumentException;
/**
* Convert raw yaml configuration into config object with pre and post validation
*/
public T translate() throws IllegalArgumentException {
- validator.validateYaml(this.yamlConfig);
- return this.translateConfig();
+ Map<String, Object> yamlConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(yamlConfig)));
+ validator.validateYaml(yamlConfigMap);
+ return this.translateConfig(yamlConfigMap);
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
similarity index 81%
rename from thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslator.java
rename to thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index f925e46..4644c80 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -19,7 +19,6 @@
package org.apache.pinot.thirdeye.detection.yaml.translator;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
import java.util.Collection;
@@ -38,7 +37,6 @@ import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
-import org.apache.pinot.thirdeye.detection.annotation.Yaml;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
@@ -123,10 +121,8 @@ import static org.apache.pinot.thirdeye.detection.yaml.DetectionConfigTuner.*;
* +-----------------------------------------+
*
*/
-@Yaml(pipelineType = "COMPOSITE")
-public class CompositePipelineConfigTranslator extends YamlDetectionConfigTranslator {
+public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigDTO, DetectionConfigValidator> {
private static final String PROP_DIMENSION_EXPLORATION = "dimensionExploration";
-
private static final String PROP_DETECTION = "detection";
private static final String PROP_CRON = "cron";
private static final String PROP_FILTER = "filter";
@@ -157,6 +153,10 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private static final String PROP_BUCKET_PERIOD = "bucketPeriod";
private static final String PROP_CACHE_PERIOD_LOOKBACK = "cachingPeriodLookback";
+ private static final String PROP_DETECTION_NAME = "detectionName";
+ private static final String PROP_DESC_NAME = "description";
+ private static final String PROP_ACTIVE = "active";
+
private static final DetectionRegistry DETECTION_REGISTRY = DetectionRegistry.getInstance();
static {
// do not tune for alerts migrated from legacy anomaly function.
@@ -168,44 +168,37 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES = ImmutableSet.of("ALGORITHM", "MIGRATED_ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
- private final MetricConfigDTO metricConfig;
- private final DatasetConfigDTO datasetConfig;
- private final String metricUrn;
- private final Map<String, Object> mergerProperties;
- // metric dimension filter maps
- private final Map<String, Collection<String>> filterMaps;
- protected final org.yaml.snakeyaml.Yaml yaml;
-
- public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig, DataProvider provider) {
+ private DataProvider dataProvider;
+
+ public DetectionConfigTranslator(String yamlConfig, DataProvider provider) {
this(yamlConfig, provider, new DetectionConfigValidator(provider));
}
- public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig, DataProvider provider, DetectionConfigValidator validator) {
- super(yamlConfig, provider, validator);
- this.yaml = new org.yaml.snakeyaml.Yaml();
- this.metricConfig = this.dataProvider.fetchMetric(MapUtils.getString(yamlConfig, PROP_METRIC),
- MapUtils.getString(yamlConfig, PROP_DATASET));
- Preconditions.checkNotNull(this.metricConfig, "metric not found");
-
- this.datasetConfig = this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
- .get(metricConfig.getDataset());
- Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
- this.mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new HashMap<String, Object>());
- this.filterMaps = MapUtils.getMap(yamlConfig, PROP_FILTERS);
- this.metricUrn = MetricEntity.fromMetric(filterMaps, this.metricConfig.getId()).getUrn();
+ public DetectionConfigTranslator(String yamlConfig, DataProvider provider, DetectionConfigValidator validator) {
+ super(yamlConfig, validator);
+ this.dataProvider = provider;
}
@Override
- DetectionConfigDTO translateConfig() throws IllegalArgumentException {
- String detectionCronInYaml = MapUtils.getString(yamlConfig, PROP_CRON);
- String cron = (detectionCronInYaml == null) ? buildCron() : detectionCronInYaml;
+ DetectionConfigDTO translateConfig(Map<String, Object> yamlConfigMap) throws IllegalArgumentException {
+ Map<String, Collection<String>> dimensionFiltersMap = MapUtils.getMap(yamlConfigMap, PROP_FILTERS);
- List<Map<String, Object>> ruleYamls = getList(yamlConfig.get(PROP_RULES));
+ MetricConfigDTO metricConfig = this.dataProvider.fetchMetric(MapUtils.getString(yamlConfigMap, PROP_METRIC),
+ MapUtils.getString(yamlConfigMap, PROP_DATASET));
+ DatasetConfigDTO datasetConfig = this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
+ .get(metricConfig.getDataset());
+ String metricUrn = MetricEntity.fromMetric(dimensionFiltersMap, metricConfig.getId()).getUrn();
+
+ String detectionCronInYaml = MapUtils.getString(yamlConfigMap, PROP_CRON);
+ String cron = (detectionCronInYaml == null) ? buildCron(datasetConfig.bucketTimeGranularity()) : detectionCronInYaml;
+ Map<String, Object> mergerProperties = MapUtils.getMap(yamlConfigMap, PROP_MERGER, new HashMap<String, Object>());
+
+ List<Map<String, Object>> ruleYamls = getList(yamlConfigMap.get(PROP_RULES));
List<Map<String, Object>> nestedPipelines = new ArrayList<>();
for (Map<String, Object> ruleYaml : ruleYamls) {
List<Map<String, Object>> filterYamls = ConfigUtils.getList(ruleYaml.get(PROP_FILTER));
List<Map<String, Object>> detectionYamls = ConfigUtils.getList(ruleYaml.get(PROP_DETECTION));
- List<Map<String, Object>> detectionProperties = buildListOfMergeWrapperProperties(detectionYamls);
+ List<Map<String, Object>> detectionProperties = buildListOfMergeWrapperProperties(detectionYamls, mergerProperties, datasetConfig.bucketTimeGranularity());
if (filterYamls.isEmpty()) {
nestedPipelines.addAll(detectionProperties);
} else {
@@ -217,53 +210,55 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
nestedPipelines.addAll(filterNestedProperties);
}
}
- Map<String, Object> dimensionWrapperProperties = buildDimensionWrapperProperties();
+ Map<String, Object> dimensionWrapperProperties = buildDimensionWrapperProperties(
+ yamlConfigMap, dimensionFiltersMap, metricUrn, datasetConfig.getDataset());
Map<String, Object> properties = buildWrapperProperties(
ChildKeepingMergeWrapper.class.getName(),
Collections.singletonList(buildWrapperProperties(DimensionWrapper.class.getName(), nestedPipelines, dimensionWrapperProperties)),
- this.mergerProperties);
+ mergerProperties);
- List<Map<String, Object>> grouperYamls = getList(yamlConfig.get(PROP_GROUPER));
+ List<Map<String, Object>> grouperYamls = getList(yamlConfigMap.get(PROP_GROUPER));
if (!grouperYamls.isEmpty()) {
properties = buildGroupWrapperProperties(grouperYamls.get(0), properties);
}
- return super.generateDetectionConfig(properties, this.components, cron);
+ return generateDetectionConfig(yamlConfigMap, properties, this.components, cron);
}
- private Map<String, Object> buildDimensionWrapperProperties() {
+ private Map<String, Object> buildDimensionWrapperProperties(Map<String, Object> yamlConfigMap,
+ Map<String, Collection<String>> dimensionFilters, String metricUrn, String datasetName) {
Map<String, Object> dimensionWrapperProperties = new HashMap<>();
- dimensionWrapperProperties.put(PROP_NESTED_METRIC_URNS, Collections.singletonList(this.metricUrn));
- if (yamlConfig.containsKey(PROP_DIMENSION_EXPLORATION)) {
- Map<String, Object> dimensionExploreYaml = MapUtils.getMap(this.yamlConfig, PROP_DIMENSION_EXPLORATION);
+ dimensionWrapperProperties.put(PROP_NESTED_METRIC_URNS, Collections.singletonList(metricUrn));
+ if (yamlConfigMap.containsKey(PROP_DIMENSION_EXPLORATION)) {
+ Map<String, Object> dimensionExploreYaml = MapUtils.getMap(yamlConfigMap, PROP_DIMENSION_EXPLORATION);
dimensionWrapperProperties.putAll(dimensionExploreYaml);
if (dimensionExploreYaml.containsKey(PROP_DIMENSION_FILTER_METRIC)){
- MetricConfigDTO dimensionExploreMetric = this.dataProvider.fetchMetric(MapUtils.getString(dimensionExploreYaml, PROP_DIMENSION_FILTER_METRIC), this.datasetConfig.getDataset());
- dimensionWrapperProperties.put(PROP_METRIC_URN, MetricEntity.fromMetric(filterMaps, dimensionExploreMetric.getId()).getUrn());
+ MetricConfigDTO dimensionExploreMetric = this.dataProvider.fetchMetric(MapUtils.getString(dimensionExploreYaml, PROP_DIMENSION_FILTER_METRIC), datasetName);
+ dimensionWrapperProperties.put(PROP_METRIC_URN, MetricEntity.fromMetric(dimensionFilters, dimensionExploreMetric.getId()).getUrn());
} else {
- dimensionWrapperProperties.put(PROP_METRIC_URN, this.metricUrn);
+ dimensionWrapperProperties.put(PROP_METRIC_URN, metricUrn);
}
}
return dimensionWrapperProperties;
}
private List<Map<String, Object>> buildListOfMergeWrapperProperties(
- List<Map<String, Object>> yamlConfigs) {
+ List<Map<String, Object>> yamlConfigs, Map<String, Object> mergerProperties, TimeGranularity datasetTimegranularity) {
List<Map<String, Object>> properties = new ArrayList<>();
for (Map<String, Object> yamlConfig : yamlConfigs) {
- properties.add(buildMergeWrapperProperties(yamlConfig));
+ properties.add(buildMergeWrapperProperties(yamlConfig, mergerProperties, datasetTimegranularity));
}
return properties;
}
- private Map<String, Object> buildMergeWrapperProperties(Map<String, Object> yamlConfig) {
+ private Map<String, Object> buildMergeWrapperProperties(Map<String, Object> yamlConfig, Map<String, Object> mergerProperties, TimeGranularity datasetTimegranularity) {
String detectorType = MapUtils.getString(yamlConfig, PROP_TYPE);
String name = MapUtils.getString(yamlConfig, PROP_NAME);
Map<String, Object> nestedProperties = new HashMap<>();
nestedProperties.put(PROP_CLASS_NAME, AnomalyDetectorWrapper.class.getName());
String detectorRefKey = makeComponentRefKey(detectorType, name);
- fillInDetectorWrapperProperties(nestedProperties, yamlConfig, detectorType);
+ fillInDetectorWrapperProperties(nestedProperties, yamlConfig, detectorType, datasetTimegranularity);
buildComponentSpec(yamlConfig, detectorType, detectorRefKey);
@@ -282,7 +277,7 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
buildComponentSpec(yamlConfig, baselineProviderType, baselineProviderKey);
properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
}
- properties.putAll(this.mergerProperties);
+ properties.putAll(mergerProperties);
return properties;
}
@@ -302,10 +297,10 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
}
// fill in window size and unit if detector requires this
- private void fillInDetectorWrapperProperties(Map<String, Object> properties, Map<String, Object> yamlConfig, String detectorType) {
+ private void fillInDetectorWrapperProperties(Map<String, Object> properties, Map<String, Object> yamlConfig, String detectorType, TimeGranularity datasetTimegranularity) {
if (MOVING_WINDOW_DETECTOR_TYPES.contains(detectorType)) {
properties.put(PROP_MOVING_WINDOW_DETECTION, true);
- switch (this.datasetConfig.bucketTimeGranularity().getUnit()) {
+ switch (datasetTimegranularity.getUnit()) {
case MINUTES:
properties.put(PROP_WINDOW_SIZE, 6);
properties.put(PROP_WINDOW_UNIT, TimeUnit.HOURS);
@@ -326,7 +321,7 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
}
}
// set default bucketPeriod
- properties.put(PROP_BUCKET_PERIOD, this.datasetConfig.bucketTimeGranularity().toPeriod().toString());
+ properties.put(PROP_BUCKET_PERIOD, datasetTimegranularity.toPeriod().toString());
// override from yaml
if (yamlConfig.containsKey(PROP_WINDOW_SIZE)) {
properties.put(PROP_MOVING_WINDOW_DETECTION, true);
@@ -399,8 +394,8 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
// hourly: every hour, starts at 0 minute
// daily: every day, starts at 2 pm UTC
// others: every day, start at 12 am UTC
- private String buildCron() {
- switch (this.datasetConfig.bucketTimeGranularity().getUnit()) {
+ private String buildCron(TimeGranularity timegranularity) {
+ switch (timegranularity.getUnit()) {
case MINUTES:
return "0 0/15 * * * ? *";
case HOURS:
@@ -437,4 +432,23 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private String makeComponentRefKey(String type, String name) {
return "$" + name + ":" + type;
}
+
+ /**
+ * Fill in common fields of detection config. Properties of the pipeline is filled by the subclass.
+ */
+ private DetectionConfigDTO generateDetectionConfig(Map<String, Object> yamlConfigMap, Map<String, Object> properties,
+ Map<String, Object> components, String cron) {
+ DetectionConfigDTO config = new DetectionConfigDTO();
+ config.setName(MapUtils.getString(yamlConfigMap, PROP_DETECTION_NAME));
+ config.setDescription(MapUtils.getString(yamlConfigMap, PROP_DESC_NAME));
+ config.setLastTimestamp(System.currentTimeMillis());
+
+ config.setProperties(properties);
+ config.setComponentSpecs(components);
+ config.setCron(cron);
+ config.setActive(MapUtils.getBooleanValue(yamlConfigMap, PROP_ACTIVE, true));
+ config.setYaml(yamlConfig);
+
+ return config;
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
similarity index 80%
rename from thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslator.java
rename to thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
index 5efab02..baf3ba8 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/SubscriptionConfigTranslator.java
@@ -26,7 +26,6 @@ import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
import org.apache.pinot.thirdeye.datalayer.util.Predicate;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
-import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry;
import java.util.ArrayList;
import java.util.Arrays;
@@ -45,7 +44,7 @@ import org.yaml.snakeyaml.Yaml;
/**
* The translator converts the alert yaml config into a detection alert config
*/
-public class YamlDetectionAlertConfigTranslator extends ConfigTranslator<DetectionAlertConfigDTO, SubscriptionConfigValidator> {
+public class SubscriptionConfigTranslator extends ConfigTranslator<DetectionAlertConfigDTO, SubscriptionConfigValidator> {
public static final String PROP_DETECTION_CONFIG_IDS = "detectionConfigIds";
public static final String PROP_RECIPIENTS = "recipients";
@@ -75,12 +74,12 @@ public class YamlDetectionAlertConfigTranslator extends ConfigTranslator<Detecti
Arrays.asList(PROP_RECIPIENTS, PROP_DIMENSION, PROP_DIMENSION_RECIPIENTS));
private final DetectionConfigManager detectionConfigDAO;
- public YamlDetectionAlertConfigTranslator(DetectionConfigManager detectionConfigDAO, Map<String,Object> yamlAlertConfig) {
- this(detectionConfigDAO, yamlAlertConfig, new SubscriptionConfigValidator());
+ public SubscriptionConfigTranslator(DetectionConfigManager detectionConfigDAO, String yamlConfig) {
+ this(detectionConfigDAO, yamlConfig, new SubscriptionConfigValidator());
}
- public YamlDetectionAlertConfigTranslator(DetectionConfigManager detectionConfigDAO, Map<String,Object> yamlAlertConfig, SubscriptionConfigValidator validator) {
- super(yamlAlertConfig, validator);
+ public SubscriptionConfigTranslator(DetectionConfigManager detectionConfigDAO, String yamlConfig, SubscriptionConfigValidator validator) {
+ super(yamlConfig, validator);
this.detectionConfigDAO = detectionConfigDAO;
}
@@ -163,37 +162,38 @@ public class YamlDetectionAlertConfigTranslator extends ConfigTranslator<Detecti
* Generates the {@link DetectionAlertConfigDTO} from the YAML Alert Map
*/
@Override
- DetectionAlertConfigDTO translateConfig() throws IllegalArgumentException {
+ DetectionAlertConfigDTO translateConfig(Map<String, Object> yamlConfigMap) throws IllegalArgumentException {
DetectionAlertConfigDTO alertConfigDTO = new DetectionAlertConfigDTO();
- alertConfigDTO.setName(MapUtils.getString(this.yamlConfig, PROP_SUBS_GROUP_NAME));
- alertConfigDTO.setApplication(MapUtils.getString(this.yamlConfig, PROP_APPLICATION));
- alertConfigDTO.setFrom(MapUtils.getString(this.yamlConfig, PROP_FROM));
+ alertConfigDTO.setName(MapUtils.getString(yamlConfigMap, PROP_SUBS_GROUP_NAME));
+ alertConfigDTO.setApplication(MapUtils.getString(yamlConfigMap, PROP_APPLICATION));
+ alertConfigDTO.setFrom(MapUtils.getString(yamlConfigMap, PROP_FROM));
- alertConfigDTO.setCronExpression(MapUtils.getString(this.yamlConfig, PROP_CRON, CRON_SCHEDULE_DEFAULT));
- alertConfigDTO.setActive(MapUtils.getBooleanValue(this.yamlConfig, PROP_ACTIVE, true));
+ alertConfigDTO.setCronExpression(MapUtils.getString(yamlConfigMap, PROP_CRON, CRON_SCHEDULE_DEFAULT));
+ alertConfigDTO.setActive(MapUtils.getBooleanValue(yamlConfigMap, PROP_ACTIVE, true));
+ alertConfigDTO.setYaml(yamlConfig);
alertConfigDTO.setSubjectType(AlertConfigBean.SubjectType.valueOf(
- (String) MapUtils.getObject(this.yamlConfig, PROP_EMAIL_SUBJECT_TYPE, AlertConfigBean.SubjectType.METRICS.name())));
+ (String) MapUtils.getObject(yamlConfigMap, PROP_EMAIL_SUBJECT_TYPE, AlertConfigBean.SubjectType.METRICS.name())));
- Map<String, String> refLinks = MapUtils.getMap(this.yamlConfig, PROP_REFERENCE_LINKS);
+ Map<String, String> refLinks = MapUtils.getMap(yamlConfigMap, PROP_REFERENCE_LINKS);
if (refLinks == null) {
refLinks = new HashMap<>();
- this.yamlConfig.put(PROP_REFERENCE_LINKS, refLinks);
+ yamlConfigMap.put(PROP_REFERENCE_LINKS, refLinks);
}
if (refLinks.isEmpty()) {
refLinks.put("ThirdEye User Guide", "https://go/thirdeyeuserguide");
refLinks.put("Add Reference Links", "https://go/thirdeyealertreflink");
}
- alertConfigDTO.setReferenceLinks(MapUtils.getMap(this.yamlConfig, PROP_REFERENCE_LINKS));
+ alertConfigDTO.setReferenceLinks(MapUtils.getMap(yamlConfigMap, PROP_REFERENCE_LINKS));
- alertConfigDTO.setAlertSchemes(buildAlertSchemes(this.yamlConfig));
- alertConfigDTO.setAlertSuppressors(buildAlertSuppressors(this.yamlConfig));
+ alertConfigDTO.setAlertSchemes(buildAlertSchemes(yamlConfigMap));
+ alertConfigDTO.setAlertSuppressors(buildAlertSuppressors(yamlConfigMap));
alertConfigDTO.setHighWaterMark(0L);
// NOTE: The below fields will/should be hidden from the YAML/UI. They will only be updated by the backend pipeline.
List<Long> detectionConfigIds = new ArrayList<>();
- List<String> detectionNames = ConfigUtils.getList(this.yamlConfig.get(PROP_DETECTION_NAMES));
+ List<String> detectionNames = ConfigUtils.getList(yamlConfigMap.get(PROP_DETECTION_NAMES));
try {
detectionConfigIds.addAll(detectionNames.stream().map(detectionName -> this.detectionConfigDAO.findByPredicate(
@@ -202,7 +202,7 @@ public class YamlDetectionAlertConfigTranslator extends ConfigTranslator<Detecti
throw new IllegalArgumentException("Cannot find detection pipeline, please check the subscribed detections.");
}
- alertConfigDTO.setProperties(buildAlerterProperties(this.yamlConfig, detectionConfigIds));
+ alertConfigDTO.setProperties(buildAlerterProperties(yamlConfigMap, detectionConfigIds));
Map<Long, Long> vectorClocks = new HashMap<>();
long currentTimestamp = System.currentTimeMillis();
for (long detectionConfigId : detectionConfigIds) {
@@ -210,11 +210,6 @@ public class YamlDetectionAlertConfigTranslator extends ConfigTranslator<Detecti
}
alertConfigDTO.setVectorClocks(vectorClocks);
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
- options.setPrettyFlow(true);
- alertConfigDTO.setYaml(new Yaml(options).dump(this.yamlConfig));
-
return alertConfigDTO;
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionConfigTranslator.java
deleted file mode 100644
index 4b8fa34..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionConfigTranslator.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection.yaml.translator;
-
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import java.util.Map;
-import org.apache.commons.collections.MapUtils;
-import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.yaml.snakeyaml.DumperOptions;
-import org.yaml.snakeyaml.Yaml;
-
-
-/**
- * The YAML config translator converts the yaml config into a detection config.
- * Calls training module for each stage.
- */
-public abstract class YamlDetectionConfigTranslator extends ConfigTranslator<DetectionConfigDTO, DetectionConfigValidator> {
- protected static final Logger LOG = LoggerFactory.getLogger(YamlDetectionConfigTranslator.class);
- private static final String PROP_NAME = "detectionName";
- private static final String PROP_DESC_NAME = "description";
- private static final String PROP_ACTIVE = "active";
-
- protected DataProvider dataProvider;
-
- YamlDetectionConfigTranslator(Map<String, Object> yamlConfig, DataProvider provider, DetectionConfigValidator validator) {
- super(yamlConfig, validator);
- this.dataProvider = provider;
- }
-
- /**
- * Fill in common fields of detection config. Properties of the pipeline is filled by the subclass.
- */
- DetectionConfigDTO generateDetectionConfig(Map<String, Object> properties, Map<String, Object> components, String cron) {
- DetectionConfigDTO config = new DetectionConfigDTO();
- config.setName(MapUtils.getString(yamlConfig, PROP_NAME));
- config.setDescription(MapUtils.getString(yamlConfig, PROP_DESC_NAME));
- config.setLastTimestamp(System.currentTimeMillis());
-
- config.setProperties(properties);
- config.setComponentSpecs(components);
- config.setCron(cron);
- config.setActive(MapUtils.getBooleanValue(yamlConfig, PROP_ACTIVE, true));
-
- DumperOptions options = new DumperOptions();
- options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
- options.setPrettyFlow(true);
- config.setYaml(new Yaml(options).dump(yamlConfig));
-
- return config;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionTranslatorLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionTranslatorLoader.java
deleted file mode 100644
index 1cb0890..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionTranslatorLoader.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.detection.yaml.translator;
-
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
-import java.lang.reflect.Constructor;
-import java.util.Map;
-
-
-/**
- * Loads the detection config translator fo a pipeline type
- */
-public class YamlDetectionTranslatorLoader {
- private static final String PROP_PIPELINE_TYPE= "pipelineType";
- private static final String DEFAULT_PIPELINE_TYPE= "Composite";
- private static DetectionRegistry DETECTION_REGISTRY = DetectionRegistry.getInstance();
-
- public YamlDetectionConfigTranslator from(Map<String, Object> yamlConfig, DataProvider provider) throws Exception {
- if (!yamlConfig.containsKey(PROP_PIPELINE_TYPE)) {
- yamlConfig.put(PROP_PIPELINE_TYPE, DEFAULT_PIPELINE_TYPE);
- }
- String className = DETECTION_REGISTRY.lookupYamlConverter(yamlConfig.get(PROP_PIPELINE_TYPE).toString().toUpperCase());
- Constructor<?> constructor = Class.forName(className).getConstructor(Map.class, DataProvider.class);
- return (YamlDetectionConfigTranslator) constructor.newInstance(yamlConfig, provider);
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
index f72110c..76cfdb3 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
@@ -14,7 +14,7 @@ import java.io.IOException;
import org.apache.commons.io.IOUtils;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.components.ThresholdRuleDetector;
-import org.apache.pinot.thirdeye.detection.yaml.translator.CompositePipelineConfigTranslator;
+import org.apache.pinot.thirdeye.detection.yaml.translator.DetectionConfigTranslator;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -41,7 +41,7 @@ public class YamlResourceTest {
config2.setName("test_detection_2");
alertId2 = detectionDAO.save(config2);
- DetectionRegistry.getInstance().registerYamlConvertor(CompositePipelineConfigTranslator.class.getName(), "COMPOSITE");
+ DetectionRegistry.getInstance().registerYamlConvertor(DetectionConfigTranslator.class.getName(), "COMPOSITE");
DetectionRegistry.registerComponent(ThresholdRuleDetector.class.getName(), "THRESHOLD");
DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL", "EmailClass");
DetectionAlertRegistry.getInstance().registerAlertScheme("IRIS", "IrisClass");
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
similarity index 77%
rename from thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslatorTest.java
rename to thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
index 85b1aec..3cf3576 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/CompositePipelineConfigTranslatorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
@@ -2,6 +2,7 @@ package org.apache.pinot.thirdeye.detection.yaml.translator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.IOUtils;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
@@ -25,7 +26,7 @@ import org.testng.annotations.Test;
import org.yaml.snakeyaml.Yaml;
-public class CompositePipelineConfigTranslatorTest {
+public class DetectionConfigTranslatorTest {
private Long metricId;
private Yaml yaml;
@@ -72,35 +73,35 @@ public class CompositePipelineConfigTranslatorTest {
@Test
public void testBuildPropertiesFull() throws Exception {
- this.yamlConfig = (Map<String, Object>) this.yaml.load(this.getClass().getResourceAsStream("pipeline-config-1.yaml"));
- CompositePipelineConfigTranslator translator = new CompositePipelineConfigTranslator(this.yamlConfig, this.provider);
- DetectionConfigDTO result = (DetectionConfigDTO) translator.translate();
+ String yamlConfig = IOUtils.toString(this.getClass().getResourceAsStream("pipeline-config-1.yaml"), "UTF-8");
+ DetectionConfigTranslator translator = new DetectionConfigTranslator(yamlConfig, this.provider);
+ DetectionConfigDTO result = translator.translate();
YamlTranslationResult expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("compositePipelineTranslatorTestResult-1.json"), YamlTranslationResult.class);
Assert.assertEquals(result.getProperties(), expected.getProperties());
}
@Test
public void testBuildDetectionPropertiesNoFilter() throws Exception {
- this.yamlConfig = (Map<String, Object>) this.yaml.load(this.getClass().getResourceAsStream("pipeline-config-2.yaml"));
- CompositePipelineConfigTranslator translator = new CompositePipelineConfigTranslator(this.yamlConfig, this.provider);
- DetectionConfigDTO result = (DetectionConfigDTO) translator.translate();
+ String yamlConfig = IOUtils.toString(this.getClass().getResourceAsStream("pipeline-config-2.yaml"), "UTF-8");
+ DetectionConfigTranslator translator = new DetectionConfigTranslator(yamlConfig, this.provider);
+ DetectionConfigDTO result = translator.translate();
YamlTranslationResult expected = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("compositePipelineTranslatorTestResult-2.json"), YamlTranslationResult.class);
Assert.assertEquals(result.getProperties(), expected.getProperties());
}
@Test(expectedExceptions = IllegalArgumentException.class)
- public void testBuildDetectionPipelineMissModuleType() {
+ public void testBuildDetectionPipelineMissModuleType() throws Exception {
this.yamlConfig = (Map<String, Object>) this.yaml.load(this.getClass().getResourceAsStream("pipeline-config-1.yaml"));
this.yamlConfig.put("rules", Collections.singletonList(
ImmutableMap.of("name", "rule2","detection", Collections.singletonList(ImmutableMap.of("change", 0.3)))));
- CompositePipelineConfigTranslator translator = new CompositePipelineConfigTranslator(this.yamlConfig, this.provider);
+ DetectionConfigTranslator translator = new DetectionConfigTranslator(yaml.dump(this.yamlConfig), this.provider);
translator.translate();
}
@Test(expectedExceptions = IllegalArgumentException.class)
- public void testMultipleGrouperLogic() {
- this.yamlConfig = (Map<String, Object>) this.yaml.load(this.getClass().getResourceAsStream("pipeline-config-3.yaml"));
- CompositePipelineConfigTranslator translator = new CompositePipelineConfigTranslator(this.yamlConfig, this.provider);
+ public void testMultipleGrouperLogic() throws Exception {
+ String yamlConfig = IOUtils.toString(this.getClass().getResourceAsStream("pipeline-config-3.yaml"), "UTF-8");
+ DetectionConfigTranslator translator = new DetectionConfigTranslator(yamlConfig, this.provider);
translator.translate();
}
}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslatorTest.java
index 8cf9ede..b8acc01 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslatorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/YamlDetectionAlertConfigTranslatorTest.java
@@ -18,8 +18,9 @@ import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import org.yaml.snakeyaml.Yaml;
-import static org.apache.pinot.thirdeye.detection.yaml.translator.YamlDetectionAlertConfigTranslator.*;
+import static org.apache.pinot.thirdeye.detection.yaml.translator.SubscriptionConfigTranslator.*;
import static org.mockito.Mockito.*;
@@ -71,8 +72,9 @@ public class YamlDetectionAlertConfigTranslatorTest {
SubscriptionConfigValidator validateMocker = mock(SubscriptionConfigValidator.class);
doNothing().when(validateMocker).validateYaml(alertYamlConfigs);
- DetectionAlertConfigDTO alertConfig = (DetectionAlertConfigDTO) new YamlDetectionAlertConfigTranslator(
- this.detectionConfigManager, alertYamlConfigs, validateMocker).translate();
+ String yamlConfig = new Yaml().dump(alertYamlConfigs);
+ DetectionAlertConfigDTO alertConfig = new SubscriptionConfigTranslator(
+ this.detectionConfigManager, yamlConfig, validateMocker).translate();
Assert.assertTrue(alertConfig.isActive());
Assert.assertEquals(alertConfig.getName(), "test_group_name");
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org