You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/06/25 15:29:08 UTC
[incubator-pinot] branch master updated: [TE] Tuning of Composite
alert - Store metric refs in component spec (#4359)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 859c68c [TE] Tuning of Composite alert - Store metric refs in component spec (#4359)
859c68c is described below
commit 859c68c7f92280292bb6f570b6f58708bc53ef20
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Tue Jun 25 08:29:02 2019 -0700
[TE] Tuning of Composite alert - Store metric refs in component spec (#4359)
---
.../thirdeye/detection/DetectionPipeline.java | 2 +-
.../detection/yaml/DetectionConfigTuner.java | 32 ++++++----------------
.../yaml/translator/DetectionConfigTranslator.java | 6 +++-
3 files changed, 15 insertions(+), 25 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
index 51406b8..425a246 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipeline.java
@@ -100,7 +100,7 @@ public abstract class DetectionPipeline {
for (String componentKey : componentSpecs.keySet()) {
Map<String, Object> componentSpec = ConfigUtils.getMap(componentSpecs.get(componentKey));
for (Map.Entry<String, Object> entry : componentSpec.entrySet()){
- if (DetectionUtils.isReferenceName(entry.getValue().toString())) {
+ if (entry.getValue() != null && DetectionUtils.isReferenceName(entry.getValue().toString())) {
componentSpec.put(entry.getKey(), instancesMap.get(DetectionUtils.getComponentKey(entry.getValue().toString())));
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
index c05f2af..9a96527 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/DetectionConfigTuner.java
@@ -77,25 +77,11 @@ public class DetectionConfigTuner {
private final DetectionConfigDTO detectionConfig;
private final DataProvider dataProvider;
- private final DatasetConfigDTO datasetConfig;
- private final String metricUrn;
public DetectionConfigTuner(DetectionConfigDTO config, DataProvider dataProvider) {
Preconditions.checkNotNull(config);
this.detectionConfig = config;
this.dataProvider = dataProvider;
-
- Map<String, Object> yamlConfig = ConfigUtils.getMap(new org.yaml.snakeyaml.Yaml().load(config.getYaml()));
-
- MetricConfigDTO metricConfig = dataProvider.fetchMetric(
- MapUtils.getString(yamlConfig, PROP_METRIC),
- MapUtils.getString(yamlConfig, PROP_DATASET));
- Preconditions.checkNotNull(metricConfig, "metric not found");
- this.datasetConfig = dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
- .get(metricConfig.getDataset());
- Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
-
- this.metricUrn = MetricEntity.fromMetric(ConfigUtils.getMap(yamlConfig.get(PROP_FILTERS)), metricConfig.getId()).getUrn();
}
/**
@@ -117,17 +103,19 @@ public class DetectionConfigTuner {
Tunable tunable = instantiateTunable(componentClassName, yamlParams, dataFetcher);
// round to daily boundary
- DateTimeZone timezone = DateTimeZone.forID(this.datasetConfig.getTimezone() == null ? DEFAULT_TIMEZONE : this.datasetConfig.getTimezone());
+ String metricName = componentProps.get(PROP_METRIC).toString();
+ String datasetName = componentProps.get(PROP_DATASET).toString();
+ MetricConfigDTO metricConfig = dataProvider.fetchMetric(metricName, datasetName);
+ DatasetConfigDTO datasetConfig = dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
+ .get(metricConfig.getDataset());
+ String metricUrn = MetricEntity.fromMetric(ConfigUtils.getMap(componentProps.get(PROP_FILTERS)), metricConfig.getId()).getUrn();
+ DateTimeZone timezone = DateTimeZone.forID(datasetConfig.getTimezone() == null ? DEFAULT_TIMEZONE : datasetConfig.getTimezone());
DateTime start = new DateTime(startTime, timezone).withTimeAtStartOfDay();
DateTime end = new DateTime(endTime, timezone).withTimeAtStartOfDay();
Interval window = new Interval(start, end);
// TODO: if dimension drill down applied, pass in the metric urn of top dimension
- tunedSpec.putAll(tunable.tune(componentProps, window, this.metricUrn));
-
- // Hack to retain the raw yaml parameters.
- // The tunable requires raw yaml params and previously tuned params to generate fresh params
- tunedSpec.put(PROP_YAML_PARAMS, yamlParams);
+ tunedSpec.putAll(tunable.tune(componentProps, window, metricUrn));
return tunedSpec;
}
@@ -162,15 +150,13 @@ public class DetectionConfigTuner {
String type = DetectionUtils.getComponentType(componentKey);
if (!TURNOFF_TUNING_COMPONENTS.contains(type) && DETECTION_REGISTRY.isTunable(componentClassName)) {
try {
- tunedComponentProps.put(PROP_CLASS_NAME, componentClassName);
tunedComponentProps.putAll(getTunedSpecs(existingComponentProps, tuningWindowStart, tuningWindowEnd));
} catch (Exception e) {
LOG.error("Tuning failed for component " + type, e);
}
- } else {
- tunedComponentProps.putAll(existingComponentProps);
}
+ tunedComponentProps.putAll(existingComponentProps);
tunedComponentSpecs.put(componentKey, tunedComponentProps);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 9a5c5cb..ab29b4a 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -40,7 +40,6 @@ import org.apache.pinot.thirdeye.detection.DataProvider;
import org.apache.pinot.thirdeye.detection.DetectionUtils;
import org.apache.pinot.thirdeye.detection.algorithm.DimensionWrapper;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
-import org.apache.pinot.thirdeye.detection.components.MockGrouper;
import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyDetectorWrapper;
import org.apache.pinot.thirdeye.detection.wrapper.AnomalyFilterWrapper;
@@ -133,6 +132,8 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
private static final String PROP_TYPE = "type";
private static final String PROP_CLASS_NAME = "className";
private static final String PROP_PARAMS = "params";
+ private static final String PROP_METRIC = "metric";
+ private static final String PROP_DATASET = "dataset";
private static final String PROP_METRIC_URN = "metricUrn";
private static final String PROP_DIMENSION_FILTER_METRIC = "dimensionFilterMetric";
private static final String PROP_NESTED_METRIC_URNS = "nestedMetricUrns";
@@ -470,6 +471,9 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
String componentClassName = DETECTION_REGISTRY.lookup(type);
componentSpecs.put(PROP_CLASS_NAME, componentClassName);
+ componentSpecs.put(PROP_METRIC, MapUtils.getString(yamlConfig, PROP_METRIC));
+ componentSpecs.put(PROP_DATASET, MapUtils.getString(yamlConfig, PROP_DATASET));
+ componentSpecs.put(PROP_FILTERS, MapUtils.getString(yamlConfig, PROP_FILTERS));
Map<String, Object> params = ConfigUtils.getMap(yamlConfig.get(PROP_PARAMS));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org