You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/01/31 19:34:43 UTC
[incubator-pinot] branch master updated: [TE] detection - yaml
translation/migration changes (#3767)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5f195b6 [TE] detection - yaml translation/migration changes (#3767)
5f195b6 is described below
commit 5f195b6c3594f0e48f37e1a7298b46dd318d4928
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Thu Jan 31 11:34:38 2019 -0800
[TE] detection - yaml translation/migration changes (#3767)
Do not trigger tune for migrated anomaly functions
Default tuning window
---
.../detection/DetectionMigrationResource.java | 4 +-
.../annotation/registry/DetectionRegistry.java | 4 +-
.../yaml/CompositePipelineConfigTranslator.java | 51 +++++++++++++-------
.../thirdeye/detection/yaml/YamlResource.java | 55 ++++++++++++----------
4 files changed, 68 insertions(+), 46 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
index afdfbd8..bc51de7 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionMigrationResource.java
@@ -144,7 +144,7 @@ public class DetectionMigrationResource {
} else{
// algorithm detector
ruleYaml.put("detection", Collections.singletonList(
- ImmutableMap.of("name", "detection_rule1", "type", "ALGORITHM", "params", getAlgorithmDetectorParams(anomalyFunctionDTO),
+ ImmutableMap.of("name", "detection_rule1", "type", "MIGRATED_ALGORITHM", "params", getAlgorithmDetectorParams(anomalyFunctionDTO),
PROP_WINDOW_SIZE, anomalyFunctionDTO.getWindowSize(),
PROP_WINDOW_UNIT, anomalyFunctionDTO.getWindowUnit().toString())));
}
@@ -156,7 +156,7 @@ public class DetectionMigrationResource {
Map<String, Object> filterYaml = new LinkedHashMap<>();
if (!alertFilter.containsKey("thresholdField")) {
// algorithm alert filter
- filterYaml = ImmutableMap.of("name", "filter_rule1", "type", "ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
+ filterYaml = ImmutableMap.of("name", "filter_rule1", "type", "MIGRATED_ALGORITHM_FILTER", "params", getAlertFilterParams(anomalyFunctionDTO));
} else {
// threshold filter migrate to rule filters
// site wide impact filter migrate to rule based swi filter
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
index 8890ddd..0d310ac 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
@@ -117,8 +117,8 @@ public class DetectionRegistry {
* @return component class name
*/
public String lookup(String type) {
- Preconditions.checkArgument(REGISTRY_MAP.containsKey(type.toUpperCase()), type + " not found in registry");
- return MapUtils.getString(REGISTRY_MAP.get(type.toUpperCase()), KEY_CLASS_NAME);
+ Preconditions.checkArgument(REGISTRY_MAP.containsKey(type), type + " not found in registry");
+ return MapUtils.getString(REGISTRY_MAP.get(type), KEY_CLASS_NAME);
}
/**
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 00b3a7c..0151c08 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -24,34 +24,35 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.api.TimeGranularity;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
+import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.InputDataFetcher;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
-import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.annotation.Yaml;
+import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
import org.apache.pinot.thirdeye.detection.spi.components.Tunable;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.BaselineFillingMergeWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.ChildKeepingMergeWrapper;
-import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.collections.MapUtils;
+import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
@@ -158,10 +159,23 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private static final String PROP_MERGER = "merger";
private static final String PROP_TIMEZONE = "timezone";
private static final String PROP_NAME = "name";
+ private static final String DEFAULT_TIMEZONE = "America/Los_Angeles";
private static final String DEFAULT_BASELINE_PROVIDER_YAML_TYPE = "RULE_BASELINE";
private static final DetectionRegistry DETECTION_REGISTRY = DetectionRegistry.getInstance();
- private static final Map<String, String> DETECTOR_TO_BASELINE = ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE");
+ static {
+ // do not tune for alerts migrated from legacy anomaly function.
+ DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibAlertFilter",
+ "MIGRATED_ALGORITHM_FILTER");
+ DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibAnomalyDetector",
+ "MIGRATED_ALGORITHM");
+ DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibBaselineProvider",
+ "MIGRATED_ALGORITHM_BASELINE");
+ }
+ private static final Set<String> TUNING_OFF_COMPONENTS =
+ ImmutableSet.of("MIGRATED_ALGORITHM_FILTER", "MIGRATED_ALGORITHM", "MIGRATED_ALGORITHM_BASELINE");
+ private static final Map<String, String> DETECTOR_TO_BASELINE =
+ ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE", "MIGRATED_ALGORITHM", "MIGRATED_ALGORITHM_BASELINE");
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES = ImmutableSet.of("ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
@@ -386,7 +400,7 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
params = MapUtils.getMap(yamlConfig, PROP_PARAMS);
}
- if (DETECTION_REGISTRY.isTunable(componentClassName)) {
+ if (!TUNING_OFF_COMPONENTS.contains(type) && DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
componentSpecs.putAll(getTunedSpecs(componentName, componentClassName, params));
} catch (Exception e) {
@@ -400,10 +414,15 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private Map<String, Object> getTunedSpecs(String componentName, String componentClassName, Map<String, Object> params)
throws Exception {
- long configId = this.existingConfig == null ? -1 : this.existingConfig.getId();
+ long configId = this.existingConfig == null ? 0 : this.existingConfig.getId();
InputDataFetcher dataFetcher = new DefaultInputDataFetcher(this.dataProvider, configId);
Tunable tunable = getTunable(componentClassName, params, dataFetcher);
- Interval window = new Interval(this.startTime, this.endTime, DateTimeZone.forID(this.datasetConfig.getTimezone()));
+
+ // round to daily boundary
+ DateTimeZone timezone = DateTimeZone.forID(this.datasetConfig.getTimezone() == null ? DEFAULT_TIMEZONE : this.datasetConfig.getTimezone());
+ DateTime start = new DateTime(this.startTime, timezone).withTimeAtStartOfDay();
+ DateTime end = new DateTime(this.endTime, timezone).withTimeAtStartOfDay();
+ Interval window = new Interval(start, end);
Map<String, Object> existingComponentSpec =
this.existingComponentSpecs.containsKey(componentName) ? MapUtils.getMap(this.existingComponentSpecs,
componentName) : Collections.emptyMap();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 57014af..d87386c 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -22,33 +22,8 @@ package org.apache.pinot.thirdeye.detection.yaml;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import javax.ws.rs.DefaultValue;
-import org.apache.pinot.thirdeye.api.Constants;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.util.Predicate;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
-import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
-import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
-import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
-import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
-import org.apache.pinot.thirdeye.detection.DataProvider;
-import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
-import org.apache.pinot.thirdeye.detection.DetectionPipeline;
-import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import com.wordnik.swagger.annotations.Api;
-import com.wordnik.swagger.annotations.ApiImplicitParam;
-import com.wordnik.swagger.annotations.ApiImplicitParams;
import com.wordnik.swagger.annotations.ApiOperation;
-import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
-import org.apache.pinot.thirdeye.detection.validators.DetectionAlertConfigValidator;
import com.wordnik.swagger.annotations.ApiParam;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
@@ -57,6 +32,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@@ -69,6 +45,28 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.pinot.thirdeye.api.Constants;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipeline;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
+import org.apache.pinot.thirdeye.detection.validators.DetectionAlertConfigValidator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
@@ -129,7 +127,6 @@ public class YamlResource {
@ApiParam("tuning window start time for tunable components") @QueryParam("startTime") long startTime,
@ApiParam("tuning window end time for tunable components") @QueryParam("endTime") long endTime) throws Exception{
Map<String, String> yamls = OBJECT_MAPPER.readValue(payload, Map.class);
-
if (StringUtils.isBlank(payload)){
return Response.status(Response.Status.BAD_REQUEST).entity(ImmutableMap.of("message", "Empty payload")).build();
}
@@ -200,6 +197,12 @@ public class YamlResource {
*/
private DetectionConfigDTO buildDetectionConfigFromYaml(long startTime, long endTime, Map<String, Object> yamlConfig,
DetectionConfigDTO existingDetectionConfig, Map<String, String> responseMessage) {
+ if (startTime == 0L && endTime == 0L) {
+ // default tuning window 28 days
+ endTime = System.currentTimeMillis();
+ startTime = endTime - TimeUnit.DAYS.toMillis(28);
+ }
+
try{
YamlDetectionConfigTranslator translator = this.translatorLoader.from(yamlConfig, this.provider);
DetectionConfigDTO detectionConfig = translator.withTrainingWindow(startTime, endTime)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org