You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xh...@apache.org on 2019/04/24 20:28:24 UTC
[incubator-pinot] branch master updated: [TE] Endpoint for
upper/lower bounds (#4160)
This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 440a3a3 [TE] Endpoint for upper/lower bounds (#4160)
440a3a3 is described below
commit 440a3a30b66c7b0a2bcc07fd1935a4201fbcabd7
Author: Xiaohui Sun <xh...@linkedin.com>
AuthorDate: Wed Apr 24 13:28:18 2019 -0700
[TE] Endpoint for upper/lower bounds (#4160)
---
.../thirdeye/dataframe/util/DataFrameUtils.java | 1 +
.../thirdeye/detection/spi/model/TimeSeries.java | 4 +
.../thirdeye/detection/yaml/YamlResource.java | 93 ++++++++++++++++++++++
3 files changed, 98 insertions(+)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java
index 3a66a79..15b810d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dataframe/util/DataFrameUtils.java
@@ -61,6 +61,7 @@ import org.joda.time.PeriodType;
*/
public class DataFrameUtils {
public static final String COL_TIME = "timestamp";
+ // baseline value
public static final String COL_VALUE = "value";
public static final String COL_CURRENT = "current";
public static final String COL_UPPER_BOUND = "upper_bound";
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java
index 8758c19..a513037 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/model/TimeSeries.java
@@ -81,6 +81,10 @@ public class TimeSeries {
return this.df.getDoubles(DataFrameUtils.COL_CURRENT);
}
+ public LongSeries getTime() {
+ return this.df.getLongs(DataFrameUtils.COL_TIME);
+ }
+
public DoubleSeries getPredictedBaseline() {
return this.df.getDoubles(DataFrameUtils.COL_VALUE);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 81451e0..c4e1160 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -31,8 +31,11 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import javax.validation.constraints.NotNull;
import javax.ws.rs.Consumes;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
@@ -48,6 +51,7 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
import org.apache.pinot.thirdeye.api.Constants;
+import org.apache.pinot.thirdeye.dataframe.util.MetricSlice;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
@@ -72,12 +76,16 @@ import org.apache.pinot.thirdeye.detection.DetectionPipeline;
import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.apache.pinot.thirdeye.detection.DetectionPipelineResult;
import org.apache.pinot.thirdeye.detection.onboard.YamlOnboardingTaskInfo;
+import org.apache.pinot.thirdeye.detection.spi.components.BaselineProvider;
+import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
import org.apache.pinot.thirdeye.detection.validators.DetectionConfigValidator;
import org.apache.pinot.thirdeye.detection.validators.SubscriptionConfigValidator;
+import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
@@ -630,6 +638,91 @@ public class YamlResource {
return Response.ok(result).build();
}
+ @POST
+ @Path("/preview/baseline")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.TEXT_PLAIN)
+ @ApiOperation("Get baseline from YAML configuration")
+ /* TODO: Will return baseline from yamlPreviewApi together with detection in the future. */
+ public Response yamlPreviewBaselineApi(
+ @QueryParam("start") long start,
+ @QueryParam("end") long end,
+ @QueryParam("urn") @NotNull String urn,
+ @QueryParam("tuningStart") long tuningStart,
+ @QueryParam("tuningEnd") long tuningEnd,
+ @ApiParam("jsonPayload") String payload,
+ @QueryParam("ruleName") String ruleName) {
+ try {
+ Preconditions.checkArgument(StringUtils.isNotBlank(payload), "The Yaml Payload in the request is empty.");
+
+ // Translate config from YAML to detection config (JSON)
+ Map<String, Object> newDetectionConfigMap = new HashMap<>(ConfigUtils.getMap(this.yaml.load(payload)));
+ DetectionConfigDTO detectionConfig = buildDetectionConfigFromYaml(tuningStart, tuningEnd, newDetectionConfigMap, null);
+ Preconditions.checkNotNull(detectionConfig);
+ detectionConfig.setId(Long.MAX_VALUE);
+
+ // There is a side effect to update detectionConfig when loading the pipeline.
+ this.loader.from(this.provider, detectionConfig, start, end);
+ TimeSeries baseline = getBaseline(detectionConfig, start, end, urn, ruleName);
+
+ return Response.ok(makeTimeSeriesMap(baseline)).build();
+ } catch (Exception e) {
+ LOG.error("Error getting baseline with payload " + payload, e);
+ }
+ return Response.ok().build();
+ }
+
+ /**
+ * Returns a map of time/baseline/current/upper/lower time series derived from the TimeSeries.
+ *
+ * @param baseline Baseline values.
+ * @return map of time/baseline/current/upper/lower time series.
+ */
+ private static Map<String, List<? extends Number>> makeTimeSeriesMap(TimeSeries baseline) {
+ Map<String, List<? extends Number>> output = new HashMap<>();
+ // time and baseline are mandatory
+ output.put(COL_TIME, baseline.getTime().toList());
+ output.put(COL_VALUE, baseline.getPredictedBaseline().toList());
+ if (baseline.getDataFrame().contains(COL_CURRENT)) {
+ output.put(COL_CURRENT, baseline.getCurrent().toList());
+ }
+ if (baseline.getDataFrame().contains(COL_UPPER_BOUND)) {
+ output.put(COL_UPPER_BOUND, baseline.getPredictedUpperBound().toList());
+ }
+ if (baseline.getDataFrame().contains(COL_LOWER_BOUND)) {
+ output.put(COL_LOWER_BOUND, baseline.getPredictedLowerBound().toList());
+ }
+ return output;
+ }
+
+ /**
+ * Get the baselines for metric urn.
+ * If there are multiple rules return the first rule's baseline.
+ * TODO: The baseline should be calculated together with detection in the future.
+ *
+ * @param detectionConfig The detection configuration.
+ * @param start Start time for baseline calculation.
+ * @param end End time for baseline calculation.
+ * @param urn The metric urn.
+ * @param rule The rule name. If not provided then find the first rule.
+ * @return The baseline for the urn.
+ */
+ private TimeSeries getBaseline(DetectionConfigDTO detectionConfig, long start, long end, String urn, String rule) {
+ MetricEntity metric = MetricEntity.fromURN(urn);
+ MetricSlice slice = MetricSlice.from(metric.getId(), start, end, metric.getFilters(), MetricSlice.NATIVE_GRANULARITY);
+
+ Optional<BaselineProvider> provider = detectionConfig.getComponents().entrySet().stream()
+ .filter(x -> x.getValue() instanceof BaselineProvider && (rule.isEmpty() || x.getKey().startsWith(rule)))
+ .map(x -> (BaselineProvider) x.getValue())
+ .findFirst();
+
+ if (provider.isPresent()) {
+ return provider.get().computePredictedTimeSeries(slice);
+ }
+
+ return new TimeSeries();
+ }
+
/**
* List all yaml configurations as JSON enhanced with detection config id, isActive and createBy information.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org