You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/04/23 18:06:14 UTC
[incubator-pinot] branch master updated: [TE] avoid tuning the
detector and baseline provider separately if both interfaces are
implemented in the same class (#4154)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 46accf2 [TE] avoid tuning the detector and baseline provider separately if both interfaces are implemented in the same class (#4154)
46accf2 is described below
commit 46accf2464db8b6c5997e3abd80490be2ef7c303
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Tue Apr 23 11:06:09 2019 -0700
[TE] avoid tuning the detector and baseline provider separately if both interfaces are implemented in the same class (#4154)
Previously, the baseline provider and detector are decoupled into separated class and tunning need to be triggered for both detector and baseline provider. This PR makes the following optimization: If both interfaces are implemented by the same detector class, the tunable will be only triggered once, which improves the performance by 50%.
---
.../annotation/registry/DetectionRegistry.java | 37 ++++++++++++++++------
.../yaml/CompositePipelineConfigTranslator.java | 25 +++++++--------
2 files changed, 40 insertions(+), 22 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
index 0d310ac..9df220e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/annotation/registry/DetectionRegistry.java
@@ -24,7 +24,9 @@ import com.google.common.collect.ImmutableMap;
import org.apache.pinot.thirdeye.detection.annotation.Components;
import org.apache.pinot.thirdeye.detection.annotation.Tune;
import org.apache.pinot.thirdeye.detection.annotation.Yaml;
+import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
import org.apache.pinot.thirdeye.detection.spi.components.BaseComponent;
+import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
import org.apache.pinot.thirdeye.detection.yaml.YamlDetectionConfigTranslator;
import java.lang.annotation.Annotation;
import java.util.ArrayList;
@@ -33,6 +35,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.collections.MapUtils;
+import org.apache.pinot.thirdeye.rootcause.timeseries.Baseline;
import org.reflections.Reflections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +54,7 @@ public class DetectionRegistry {
private static final Logger LOG = LoggerFactory.getLogger(DetectionRegistry.class);
private static final String KEY_CLASS_NAME = "className";
private static final String KEY_ANNOTATION = "annotation";
+ private static final String KEY_IS_BASELINE_PROVIDER = "isBaselineProvider";
private static DetectionRegistry INSTANCE;
@@ -74,24 +78,25 @@ public class DetectionRegistry {
Reflections reflections = new Reflections();
// register components
Set<Class<? extends BaseComponent>> classes = reflections.getSubTypesOf(BaseComponent.class);
- for (Class clazz : classes) {
+ for (Class<? extends BaseComponent> clazz : classes) {
String className = clazz.getName();
for (Annotation annotation : clazz.getAnnotations()) {
if (annotation instanceof Components) {
Components componentsAnnotation = (Components) annotation;
REGISTRY_MAP.put(componentsAnnotation.type(),
- ImmutableMap.of(KEY_CLASS_NAME, className, KEY_ANNOTATION, componentsAnnotation));
+ ImmutableMap.of(KEY_CLASS_NAME, className, KEY_ANNOTATION, componentsAnnotation,
+ KEY_IS_BASELINE_PROVIDER, isBaselineProvider(clazz)));
}
if (annotation instanceof Tune) {
- Tune trainingAnnotation = (Tune) annotation;
- TUNE_MAP.put(className, trainingAnnotation);
+ Tune tunableAnnotation = (Tune) annotation;
+ TUNE_MAP.put(className, tunableAnnotation);
}
}
}
// register yaml translators
Set<Class<? extends YamlDetectionConfigTranslator>> yamlConverterClasses =
reflections.getSubTypesOf(YamlDetectionConfigTranslator.class);
- for (Class clazz : yamlConverterClasses) {
+ for (Class<? extends YamlDetectionConfigTranslator> clazz : yamlConverterClasses) {
for (Annotation annotation : clazz.getAnnotations()) {
if (annotation instanceof Yaml) {
YAML_MAP.put(((Yaml) annotation).pipelineType(), clazz.getName());
@@ -104,7 +109,12 @@ public class DetectionRegistry {
}
public static void registerComponent(String className, String type) {
- REGISTRY_MAP.put(type, ImmutableMap.of(KEY_CLASS_NAME, className));
+ try {
+ Class<? extends BaseComponent> clazz = (Class<? extends BaseComponent>) Class.forName(className);
+ REGISTRY_MAP.put(type, ImmutableMap.of(KEY_CLASS_NAME, className, KEY_IS_BASELINE_PROVIDER, isBaselineProvider(clazz)));
+ } catch (Exception e) {
+ LOG.warn("Encountered exception when registering component {}", className, e);
+ }
}
public static void registerYamlConvertor(String className, String type) {
@@ -125,9 +135,9 @@ public class DetectionRegistry {
* Look up the tunable class name for a component class name
* @return tunable class name
*/
- public String lookupTunable(String className) {
- Preconditions.checkArgument(TUNE_MAP.containsKey(className), className + " not found in registry");
- return this.lookup(TUNE_MAP.get(className).tunable());
+ public String lookupTunable(String type) {
+ Preconditions.checkArgument(TUNE_MAP.containsKey(type), type + " not found in registry");
+ return this.lookup(TUNE_MAP.get(type).tunable());
}
/**
@@ -143,6 +153,11 @@ public class DetectionRegistry {
return TUNE_MAP.containsKey(className);
}
+ public boolean isBaselineProvider(String type) {
+ Preconditions.checkArgument(REGISTRY_MAP.containsKey(type), type + " not found in registry");
+ return MapUtils.getBooleanValue(REGISTRY_MAP.get(type), KEY_IS_BASELINE_PROVIDER);
+ }
+
/**
* Return all component implementation annotations
* @return List of component annotation
@@ -161,4 +176,8 @@ public class DetectionRegistry {
public String printAnnotations() {
return String.join(", ", YAML_MAP.keySet());
}
+
+ private static boolean isBaselineProvider(Class<? extends BaseComponent> clazz) {
+ return BaselineProvider.class.isAssignableFrom(clazz);
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 18c9f1a..9f14ded 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -173,14 +173,9 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
"MIGRATED_ALGORITHM_FILTER");
DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibAnomalyDetector",
"MIGRATED_ALGORITHM");
- DetectionRegistry.registerComponent("com.linkedin.thirdeye.detection.components.AdLibBaselineProvider",
- "MIGRATED_ALGORITHM_BASELINE");
}
private static final Set<String> TUNING_OFF_COMPONENTS =
ImmutableSet.of("MIGRATED_ALGORITHM_FILTER", "MIGRATED_ALGORITHM", "MIGRATED_ALGORITHM_BASELINE");
- private static final Map<String, String> DETECTOR_TO_BASELINE =
- ImmutableMap.of("ALGORITHM", "ALGORITHM_BASELINE", "MIGRATED_ALGORITHM", "MIGRATED_ALGORITHM_BASELINE",
- "HOLT_WINTERS_RULE", "HOLT_WINTERS_RULE");
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES = ImmutableSet.of("ALGORITHM", "MIGRATED_ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
@@ -202,7 +197,7 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
this.datasetConfig = this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
.get(metricConfig.getDataset());
Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
- this.mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new HashMap());
+ this.mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new HashMap<String, Object>());
this.filterMaps = MapUtils.getMap(yamlConfig, PROP_FILTERS);
this.metricUrn = buildMetricUrn(filterMaps, this.metricConfig.getId());
}
@@ -275,14 +270,18 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
Map<String, Object> properties = new HashMap<>();
properties.put(PROP_CLASS_NAME, BaselineFillingMergeWrapper.class.getName());
properties.put(PROP_NESTED, Collections.singletonList(nestedProperties));
- String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
- if (DETECTOR_TO_BASELINE.containsKey(detectorType)) {
- baselineProviderType = DETECTOR_TO_BASELINE.get(detectorType);
- }
- String baselineProviderKey = makeComponentKey(baselineProviderType, name);
- properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
properties.put(PROP_DETECTOR, detectorKey);
- buildComponentSpec(yamlConfig, baselineProviderType, baselineProviderKey);
+
+ // fill in baseline provider properties
+ if (DETECTION_REGISTRY.isBaselineProvider(detectorType)) {
+ // if the detector implements the baseline provider interface, use it to generate baseline
+ properties.put(PROP_BASELINE_PROVIDER, detectorKey);
+ } else {
+ String baselineProviderType = DEFAULT_BASELINE_PROVIDER_YAML_TYPE;
+ String baselineProviderKey = makeComponentKey(baselineProviderType, name);
+ buildComponentSpec(yamlConfig, baselineProviderType, baselineProviderKey);
+ properties.put(PROP_BASELINE_PROVIDER, baselineProviderKey);
+ }
properties.putAll(this.mergerProperties);
return properties;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org