You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/03/18 19:28:33 UTC
[incubator-pinot] branch master updated: [TE] yaml - more
validation on max duration (#3982)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 67b729d [TE] yaml - more validation on max duration (#3982)
67b729d is described below
commit 67b729d4b0935e652be31d5a3fff025b9f361204
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Mon Mar 18 12:28:26 2019 -0700
[TE] yaml - more validation on max duration (#3982)
---
.../yaml/CompositePipelineConfigTranslator.java | 29 +++++++++++-----------
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
index 13ffbdf..657efa6 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/CompositePipelineConfigTranslator.java
@@ -182,19 +182,17 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
private static final Set<String> MOVING_WINDOW_DETECTOR_TYPES = ImmutableSet.of("ALGORITHM", "MIGRATED_ALGORITHM");
private final Map<String, Object> components = new HashMap<>();
- private MetricConfigDTO metricConfig;
- private DatasetConfigDTO datasetConfig;
- private String metricUrn;
- private Map<String, Object> mergerProperties = new HashMap<>();
+ private final MetricConfigDTO metricConfig;
+ private final DatasetConfigDTO datasetConfig;
+ private final String metricUrn;
+ private final Map<String, Object> mergerProperties;
+ // metric dimension filter maps
+ private final Map<String, Collection<String>> filterMaps;
protected final org.yaml.snakeyaml.Yaml yaml;
public CompositePipelineConfigTranslator(Map<String, Object> yamlConfig, DataProvider provider) {
super(yamlConfig, provider);
this.yaml = new org.yaml.snakeyaml.Yaml();
- }
-
- @Override
- YamlTranslationResult translateYaml() {
this.metricConfig = this.dataProvider.fetchMetric(MapUtils.getString(yamlConfig, PROP_METRIC),
MapUtils.getString(yamlConfig, PROP_DATASET));
Preconditions.checkNotNull(this.metricConfig, "Metric not found");
@@ -202,12 +200,13 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
this.datasetConfig = this.dataProvider.fetchDatasets(Collections.singletonList(metricConfig.getDataset()))
.get(metricConfig.getDataset());
Preconditions.checkNotNull(this.datasetConfig, "dataset not found");
-
- // if user set merger properties
this.mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new HashMap());
-
- Map<String, Collection<String>> filterMaps = MapUtils.getMap(yamlConfig, PROP_FILTERS);
+ this.filterMaps = MapUtils.getMap(yamlConfig, PROP_FILTERS);
this.metricUrn = buildMetricUrn(filterMaps, this.metricConfig.getId());
+ }
+
+ @Override
+ YamlTranslationResult translateYaml() {
String detectionCronInYaml = MapUtils.getString(yamlConfig, PROP_CRON);
String cron = (detectionCronInYaml == null) ? buildCron() : detectionCronInYaml;
@@ -228,14 +227,14 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
nestedPipelines.addAll(filterNestedProperties);
}
}
- Map<String, Object> dimensionWrapperProperties = buildDimensionWrapperProperties(filterMaps);
+ Map<String, Object> dimensionWrapperProperties = buildDimensionWrapperProperties();
Map<String, Object> properties = buildWrapperProperties(ChildKeepingMergeWrapper.class.getName(),
Collections.singletonList(
buildWrapperProperties(DimensionWrapper.class.getName(), nestedPipelines, dimensionWrapperProperties)), this.mergerProperties);
return new YamlTranslationResult().withProperties(properties).withComponents(this.components).withCron(cron);
}
- private Map<String, Object> buildDimensionWrapperProperties(Map<String, Collection<String>> filterMaps) {
+ private Map<String, Object> buildDimensionWrapperProperties() {
Map<String, Object> dimensionWrapperProperties = new HashMap<>();
dimensionWrapperProperties.put(PROP_NESTED_METRIC_URNS, Collections.singletonList(this.metricUrn));
if (yamlConfig.containsKey(PROP_DIMENSION_EXPLORATION)) {
@@ -470,7 +469,7 @@ public class CompositePipelineConfigTranslator extends YamlDetectionConfigTransl
// Safety condition: Validate if maxDuration is greater than 15 minutes
Map<String, Object> mergerProperties = MapUtils.getMap(yamlConfig, PROP_MERGER, new HashMap());
if (mergerProperties.get(PROP_MAX_DURATION) != null) {
- Preconditions.checkArgument(MapUtils.getLong(mergerProperties, PROP_MAX_DURATION) >= TimeUnit.MINUTES.toMillis(15),
+ Preconditions.checkArgument(MapUtils.getLong(mergerProperties, PROP_MAX_DURATION) >= datasetConfig.bucketTimeGranularity().toMillis(),
"The maxDuration field set is not acceptable. Please check the the document and set it correctly.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org