You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/06/19 21:56:12 UTC
[incubator-pinot] branch master updated: [TE] detection health
status (#4322)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0c339d9 [TE] detection health status (#4322)
0c339d9 is described below
commit 0c339d978b055736a674b8276fa48da3b7da3c19
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Wed Jun 19 14:56:08 2019 -0700
[TE] detection health status (#4322)
- The endpoint to return the detection health status in a certain time range.
- The detection health status builder
---
.../thirdeye/detection/DetectionResource.java | 29 ++-
.../detection/health/AnomalyCoverageStatus.java | 73 ++++++
.../thirdeye/detection/health/DetectionHealth.java | 261 +++++++++++++++++++++
.../detection/health/DetectionTaskStatus.java | 98 ++++++++
.../thirdeye/detection/health/HealthStatus.java | 28 +++
.../detection/health/RegressionStatus.java | 101 ++++++++
.../detection/health/DetectionHealthTest.java | 127 ++++++++++
7 files changed, 713 insertions(+), 4 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
index 53609df..8d23058 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
@@ -20,7 +20,6 @@
package org.apache.pinot.thirdeye.detection;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Multimaps;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
@@ -46,7 +45,6 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.api.Constants;
import org.apache.pinot.thirdeye.constant.AnomalyFeedbackType;
import org.apache.pinot.thirdeye.constant.AnomalyResultSource;
@@ -61,12 +59,12 @@ import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager;
import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datalayer.util.Predicate;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
@@ -76,8 +74,8 @@ import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
import org.apache.pinot.thirdeye.detection.finetune.GridSearchTuningAlgorithm;
import org.apache.pinot.thirdeye.detection.finetune.TuningAlgorithm;
+import org.apache.pinot.thirdeye.detection.health.DetectionHealth;
import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
-import org.apache.pinot.thirdeye.detection.spi.model.TimeSeries;
import org.apache.pinot.thirdeye.detector.function.BaseAnomalyFunction;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.apache.pinot.thirdeye.util.AnomalyOffset;
@@ -109,6 +107,8 @@ public class DetectionResource {
private final DataProvider provider;
private final DetectionConfigManager configDAO;
private final EvaluationManager evaluationDAO;
+ private final TaskManager taskDAO;
+
private final DetectionAlertConfigManager detectionAlertConfigDAO;
public DetectionResource() {
@@ -119,6 +119,7 @@ public class DetectionResource {
this.configDAO = DAORegistry.getInstance().getDetectionConfigManager();
this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager();
this.evaluationDAO = DAORegistry.getInstance().getEvaluationManager();
+ this.taskDAO = DAORegistry.getInstance().getTaskDAO();
TimeSeriesLoader timeseriesLoader =
new DefaultTimeSeriesLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache());
@@ -570,4 +571,24 @@ public class DetectionResource {
return Response.ok(baselineTimeseries).build();
}
+
+ @GET
+ @Path(value = "/health/{id}")
+ @ApiOperation("Get the detection health metrics and statuses for a detection config")
+ public Response getDetectionHealth(@PathParam("id") @ApiParam("detection config id") long id,
+ @ApiParam("Start time for the the health metric") @QueryParam("start") long start,
+ @ApiParam("End time for the the health metric") @QueryParam("end") long end,
+ @ApiParam("Max number of detection tasks returned") @QueryParam("limit") @DefaultValue("500") long limit) {
+ DetectionHealth health;
+ try {
+ health = new DetectionHealth.Builder(id, start, end).addRegressionStatus(this.evaluationDAO)
+ .addAnomalyCoverageStatus(this.anomalyDAO)
+ .addDetectionTaskStatus(this.taskDAO, limit)
+ .addOverallHealth()
+ .build();
+ } catch (Exception e) {
+ return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build();
+ }
+ return Response.ok(health).build();
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/AnomalyCoverageStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/AnomalyCoverageStatus.java
new file mode 100644
index 0000000..9a4d893
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/AnomalyCoverageStatus.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+
+/**
+ * The anomaly coverage status for a detection config
+ */
+public class AnomalyCoverageStatus {
+ // the anomaly coverage ratio. the percentage of anomalous duration in the duration of the whole window
+ @JsonProperty
+ private final double anomalyCoverageRatio;
+
+ // the health status of the anomaly coverage ratio
+ @JsonProperty
+ private final HealthStatus healthStatus;
+
+ private static final double COVERAGE_RATIO_BAD_UPPER_LIMIT = 0.85;
+ private static final double COVERAGE_RATIO_BAD_LOWER_LIMIT = 0.01;
+ private static final double COVERAGE_RATIO_MODERATE_LIMIT = 0.5;
+
+
+ public AnomalyCoverageStatus(double anomalyCoverageRatio, HealthStatus healthStatus) {
+ this.anomalyCoverageRatio = anomalyCoverageRatio;
+ this.healthStatus = healthStatus;
+ }
+
+ public static AnomalyCoverageStatus fromCoverageRatio(double anomalyCoverageRatio) {
+ return new AnomalyCoverageStatus(anomalyCoverageRatio, classifyCoverageStatus(anomalyCoverageRatio));
+ }
+
+ private static HealthStatus classifyCoverageStatus(double anomalyCoverageRatio) {
+ if (Double.isNaN(anomalyCoverageRatio)) {
+ return HealthStatus.UNKNOWN;
+ }
+ if (anomalyCoverageRatio > COVERAGE_RATIO_BAD_UPPER_LIMIT
+ || anomalyCoverageRatio < COVERAGE_RATIO_BAD_LOWER_LIMIT) {
+ return HealthStatus.BAD;
+ }
+ if (anomalyCoverageRatio > COVERAGE_RATIO_MODERATE_LIMIT) {
+ return HealthStatus.MODERATE;
+ }
+ return HealthStatus.GOOD;
+ }
+
+ public double getAnomalyCoverageRatio() {
+ return anomalyCoverageRatio;
+ }
+
+ public HealthStatus getHealthStatus() {
+ return healthStatus;
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java
new file mode 100644
index 0000000..a2496c0
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
+import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
+import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.TaskDTO;
+import org.apache.pinot.thirdeye.datalayer.pojo.EvaluationBean;
+import org.apache.pinot.thirdeye.datalayer.pojo.MergedAnomalyResultBean;
+import org.apache.pinot.thirdeye.datalayer.pojo.TaskBean;
+import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+import org.joda.time.Interval;
+
+
+/**
+ * The detection health metric and status
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class DetectionHealth {
+ // overall health for a detection config
+ @JsonProperty
+ private HealthStatus overallHealth;
+
+ // the regression metrics and status for a detection config
+ @JsonProperty
+ private RegressionStatus regressionStatus;
+
+ // the anomaly coverage status for a detection config
+ @JsonProperty
+ private AnomalyCoverageStatus anomalyCoverageStatus;
+
+ // the detection task status for a detection config
+ @JsonProperty
+ private DetectionTaskStatus detectionTaskStatus;
+
+ public HealthStatus getOverallHealth() {
+ return overallHealth;
+ }
+
+ public RegressionStatus getRegressionStatus() {
+ return regressionStatus;
+ }
+
+ public AnomalyCoverageStatus getAnomalyCoverageStatus() {
+ return anomalyCoverageStatus;
+ }
+
+ public DetectionTaskStatus getDetectionTaskStatus() {
+ return detectionTaskStatus;
+ }
+
+ /**
+ * Builder for the detection health
+ */
+ public static class Builder {
+ private final long startTime;
+ private final long endTime;
+ private final long detectionConfigId;
+ private EvaluationManager evaluationDAO;
+ private MergedAnomalyResultManager anomalyDAO;
+ private TaskManager taskDAO;
+ private long taskLimit;
+ private boolean provideOverallHealth;
+
+ // database column name constants
+ private static String COL_NAME_START_TIME = "startTime";
+ private static String COL_NAME_END_TIME = "endTime";
+ private static String COL_NAME_DETECTION_CONFIG_ID = "detectionConfigId";
+ private static String COL_NAME_TASK_NAME = "name";
+ private static String COL_NAME_TASK_STATUS = "status";
+ private static String COL_NAME_TASK_TYPE = "type";
+
+ public Builder(long detectionConfigId, long startTime, long endTime) {
+ Preconditions.checkArgument(endTime >= startTime, "end time must be after start time");
+ this.startTime = startTime;
+ this.endTime = endTime;
+ this.detectionConfigId = detectionConfigId;
+ }
+
+ /**
+ * Add the regression health status in the health report built by the builder
+ * @param evaluationDAO the evaluation dao
+ * @return the builder
+ */
+ public Builder addRegressionStatus(EvaluationManager evaluationDAO) {
+ this.evaluationDAO = evaluationDAO;
+ return this;
+ }
+
+ /**
+ * Add the anomaly coverage health status in the health report built by the builder
+ * @param anomalyDAO the anomaly dao
+ * @return the builder
+ */
+ public Builder addAnomalyCoverageStatus(MergedAnomalyResultManager anomalyDAO) {
+ this.anomalyDAO = anomalyDAO;
+ return this;
+ }
+
+ /**
+ * Add the detection task health status in the health report built by the builder
+ * @param taskDAO the task dao
+ * @param limit the maximum number of tasks returned in the health report (ordered by task start time, latest task first)
+ * @return the builder
+ */
+ public Builder addDetectionTaskStatus(TaskManager taskDAO, long limit) {
+ this.taskDAO = taskDAO;
+ this.taskLimit = limit;
+ return this;
+ }
+
+ /**
+ * Add the global health status in the report built by the builder, consider regression health, coverage ratio and task health
+ * @return the builder
+ */
+ public Builder addOverallHealth() {
+ this.provideOverallHealth = true;
+ return this;
+ }
+
+ /**
+ * Build the health status object
+ * @return the health status object
+ */
+ public DetectionHealth build() {
+ DetectionHealth health = new DetectionHealth();
+ if (this.evaluationDAO != null) {
+ health.regressionStatus = buildRegressionStatus();
+ }
+ if (this.anomalyDAO != null) {
+ health.anomalyCoverageStatus = buildAnomalyCoverageStatus();
+ }
+ if (this.taskDAO != null) {
+ health.detectionTaskStatus = buildTaskStatus();
+ }
+ if (this.provideOverallHealth) {
+ health.overallHealth = classifyOverallHealth(health);
+ }
+ return health;
+ }
+
+ private RegressionStatus buildRegressionStatus() {
+ // fetch evaluations
+ List<EvaluationDTO> evaluations = this.evaluationDAO.findByPredicate(
+ Predicate.AND(Predicate.LT(COL_NAME_START_TIME, endTime), Predicate.GT(COL_NAME_END_TIME, startTime),
+ Predicate.EQ(COL_NAME_DETECTION_CONFIG_ID, detectionConfigId)));
+
+ // calculate average mapes for each detector
+ Map<String, Double> detectorMapes = evaluations.stream()
+ .filter(eval -> Objects.nonNull(eval.getMape()))
+ .collect(Collectors.groupingBy(EvaluationBean::getDetectorName,
+ Collectors.averagingDouble(EvaluationBean::getMape)));
+
+ // construct regression status
+ return RegressionStatus.fromDetectorMapes(detectorMapes);
+ }
+
+ private AnomalyCoverageStatus buildAnomalyCoverageStatus() {
+ // fetch anomalies
+ List<MergedAnomalyResultDTO> anomalies = this.anomalyDAO.findByPredicate(
+ Predicate.AND(Predicate.LT(COL_NAME_START_TIME, this.endTime),
+ Predicate.GT(COL_NAME_END_TIME, this.startTime),
+ Predicate.EQ(COL_NAME_DETECTION_CONFIG_ID, detectionConfigId)));
+ anomalies = anomalies.stream().filter(anomaly -> !anomaly.isChild()).collect(Collectors.toList());
+
+ // the anomalies can come from different sub-dimensions, merge the anomaly range if possible
+ List<Interval> intervals = new ArrayList<>();
+ if (!anomalies.isEmpty()) {
+ anomalies.sort(Comparator.comparingLong(MergedAnomalyResultBean::getStartTime));
+ long start = anomalies.stream().findFirst().get().getStartTime();
+ long end = anomalies.stream().findFirst().get().getEndTime();
+ for (MergedAnomalyResultDTO anomaly : anomalies) {
+ if (anomaly.getStartTime() <= end) {
+ end = Math.max(end, anomaly.getEndTime());
+ } else {
+ intervals.add(new Interval(start, end));
+ start = anomaly.getStartTime();
+ end = anomaly.getEndTime();
+ }
+ }
+ intervals.add(new Interval(start, end));
+ }
+
+ // compute coverage
+ long totalAnomalyCoverage =
+ intervals.stream().map(interval -> interval.getEndMillis() - interval.getStartMillis()).reduce(0L, Long::sum);
+ double coverageRatio = (double) totalAnomalyCoverage / (this.endTime - this.startTime);
+ return AnomalyCoverageStatus.fromCoverageRatio(coverageRatio);
+ }
+
+ private DetectionTaskStatus buildTaskStatus() {
+ // fetch tasks
+ List<TaskDTO> tasks = this.taskDAO.findByPredicate(
+ Predicate.AND(Predicate.EQ(COL_NAME_TASK_NAME, "DETECTION_" + this.detectionConfigId),
+ Predicate.LT(COL_NAME_START_TIME, endTime), Predicate.GT(COL_NAME_END_TIME, startTime),
+ Predicate.EQ(COL_NAME_TASK_TYPE, TaskConstants.TaskType.DETECTION.toString()),
+ Predicate.IN(COL_NAME_TASK_STATUS, new String[]{TaskConstants.TaskStatus.COMPLETED.toString(),
+ TaskConstants.TaskStatus.FAILED.toString(), TaskConstants.TaskStatus.TIMEOUT.toString()})));
+ tasks.sort(Comparator.comparingLong(TaskBean::getStartTime).reversed());
+ // limit the task size
+ tasks = tasks.stream().limit(this.taskLimit).collect(Collectors.toList());
+
+ return DetectionTaskStatus.fromTasks(tasks);
+ }
+
+ private static HealthStatus classifyOverallHealth(DetectionHealth health) {
+ HealthStatus taskHealth = health.detectionTaskStatus.getHealthStatus();
+ HealthStatus regressionHealth = health.regressionStatus.getHealthStatus();
+ HealthStatus coverageHealth = health.anomalyCoverageStatus.getHealthStatus();
+
+ Preconditions.checkNotNull(taskHealth);
+ Preconditions.checkNotNull(regressionHealth);
+ Preconditions.checkNotNull(coverageHealth);
+
+ // if task fail ratio is high or both regression and coverage are bad, we say the overall status is bad
+ if (taskHealth.equals(HealthStatus.BAD) || (regressionHealth.equals(HealthStatus.BAD) && coverageHealth.equals(
+ HealthStatus.BAD))) {
+ return HealthStatus.BAD;
+ }
+
+ Set<HealthStatus> statusSet = ImmutableSet.of(taskHealth, regressionHealth, coverageHealth);
+ if (statusSet.contains(HealthStatus.MODERATE) || statusSet.contains(HealthStatus.BAD)) {
+ return HealthStatus.MODERATE;
+ }
+ return HealthStatus.GOOD;
+ }
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java
new file mode 100644
index 0000000..1133f0b
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
+import org.apache.pinot.thirdeye.datalayer.dto.TaskDTO;
+import org.apache.pinot.thirdeye.datalayer.pojo.TaskBean;
+
+
+/**
+ * The detection task status for a detection config
+ */
+public class DetectionTaskStatus {
+ // the task success rate for the detection config
+ @JsonProperty
+ private final double taskSuccessRate;
+
+ // the health status for the detection tasks
+ @JsonProperty
+ private final HealthStatus healthStatus;
+
+ // the list of tasks for the detection config
+ @JsonProperty
+ private final List<TaskDTO> tasks;
+
+ private static final double TASK_SUCCESS_RATE_BAD_THRESHOLD = 0.2;
+ private static final double TASK_SUCCESS_RATE_MODERATE_THRESHOLD = 0.8;
+
+ public DetectionTaskStatus(double taskSuccessRate, HealthStatus healthStatus, List<TaskDTO> tasks) {
+ this.taskSuccessRate = taskSuccessRate;
+ this.healthStatus = healthStatus;
+ this.tasks = tasks;
+ }
+
+ public double getTaskSuccessRate() {
+ return taskSuccessRate;
+ }
+
+ public HealthStatus getHealthStatus() {
+ return healthStatus;
+ }
+
+ public List<TaskDTO> getTasks() {
+ return tasks;
+ }
+
+ public static DetectionTaskStatus fromTasks(List<TaskDTO> tasks) {
+
+ double taskSuccessRate = Double.NaN;
+ // count the number of tasks by task status
+ Map<TaskConstants.TaskStatus, Long> count =
+ tasks.stream().collect(Collectors.groupingBy(TaskBean::getStatus, Collectors.counting()));
+ if (count.size() != 0) {
+ long completedTasks = count.getOrDefault(TaskConstants.TaskStatus.COMPLETED, 0L);
+ long failedTasks = count.getOrDefault(
+ TaskConstants.TaskStatus.FAILED, 0L);
+ long timeoutTasks = count.getOrDefault(TaskConstants.TaskStatus.TIMEOUT, 0L);
+ taskSuccessRate = (double) completedTasks / (failedTasks + timeoutTasks + completedTasks);
+ }
+ return new DetectionTaskStatus(taskSuccessRate, classifyTaskStatus(taskSuccessRate), tasks);
+ }
+
+ private static HealthStatus classifyTaskStatus(double taskSuccessRate) {
+ if (Double.isNaN(taskSuccessRate)) {
+ return HealthStatus.UNKNOWN;
+ }
+ if (taskSuccessRate < TASK_SUCCESS_RATE_BAD_THRESHOLD) {
+ return HealthStatus.BAD;
+ }
+ if (taskSuccessRate < TASK_SUCCESS_RATE_MODERATE_THRESHOLD) {
+ return HealthStatus.MODERATE;
+ }
+ return HealthStatus.GOOD;
+ }
+
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/HealthStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/HealthStatus.java
new file mode 100644
index 0000000..83f3e30
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/HealthStatus.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+/**
+ * The detection health status
+ */
+public enum HealthStatus {
+ GOOD, MODERATE, BAD, UNKNOWN
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/RegressionStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/RegressionStatus.java
new file mode 100644
index 0000000..3ccaee8
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/RegressionStatus.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+
+/**
+ * The regression status for a detection config
+ */
+public class RegressionStatus {
+ // the average mape for each detector
+ @JsonProperty
+ private final Map<String, Double> detectorMapes;
+
+ // the health status for each detector
+ @JsonProperty
+ private final Map<String, HealthStatus> detectorHealthStatus;
+
+ // the overall regression health for the detection config
+ @JsonProperty
+ private final HealthStatus healthStatus;
+
+ public RegressionStatus(Map<String, Double> detectorMapes, Map<String, HealthStatus> detectorHealthStatus,
+ HealthStatus healthStatus) {
+ this.detectorMapes = detectorMapes;
+ this.detectorHealthStatus = detectorHealthStatus;
+ this.healthStatus = healthStatus;
+ }
+
+ public static RegressionStatus fromDetectorMapes(Map<String, Double> detectorMapes) {
+ Map<String, HealthStatus> detectorHealthStatus = detectorMapes.entrySet()
+ .stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> classifyMapeHealth(e.getValue())));
+ return new RegressionStatus(detectorMapes, detectorHealthStatus,
+ classifyOverallRegressionStatus(detectorHealthStatus));
+ }
+
+ public Map<String, Double> getDetectorMapes() {
+ return detectorMapes;
+ }
+
+ public Map<String, HealthStatus> getDetectorHealthStatus() {
+ return detectorHealthStatus;
+ }
+
+ public HealthStatus getHealthStatus() {
+ return healthStatus;
+ }
+
+ private static HealthStatus classifyMapeHealth(double mape) {
+ if (Double.isNaN(mape)) {
+ return HealthStatus.UNKNOWN;
+ }
+ if (mape < 0.2) {
+ return HealthStatus.GOOD;
+ }
+ if (mape < 0.5) {
+ return HealthStatus.MODERATE;
+ }
+ return HealthStatus.BAD;
+ }
+
+ /**
+ * Classify the regression status of the detection config based on the health status for each detector
+ * @param detectorHealthStatus the health status for each detector
+ * @return the overall regression status
+ */
+ private static HealthStatus classifyOverallRegressionStatus(Map<String, HealthStatus> detectorHealthStatus) {
+ if (detectorHealthStatus.isEmpty()) {
+ return HealthStatus.UNKNOWN;
+ }
+ if (detectorHealthStatus.values().contains(HealthStatus.GOOD)) {
+ return HealthStatus.GOOD;
+ }
+ if (detectorHealthStatus.values().contains(HealthStatus.MODERATE)) {
+ return HealthStatus.MODERATE;
+ }
+ return HealthStatus.BAD;
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java
new file mode 100644
index 0000000..2fa5ba1
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.detection.health;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Arrays;
+import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
+import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
+import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
+import org.apache.pinot.thirdeye.datalayer.dto.EvaluationDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.TaskDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+public class DetectionHealthTest {
+ private DAOTestBase testDAOProvider;
+ private MergedAnomalyResultManager anomalyDAO;
+ private TaskManager taskDAO;
+ private EvaluationManager evaluationDAO;
+ private long configId;
+
+ @BeforeMethod
+ public void setUp() {
+ this.testDAOProvider = DAOTestBase.getInstance();
+ DAORegistry daoRegistry = DAORegistry.getInstance();
+ this.anomalyDAO = daoRegistry.getMergedAnomalyResultDAO();
+ this.taskDAO = daoRegistry.getTaskDAO();
+ this.evaluationDAO = daoRegistry.getEvaluationManager();
+ this.configId = 1;
+ }
+
+ @Test
+ public void testBuildRegressionStatus() {
+ long startTime = 100;
+ long endTime = 200;
+ EvaluationDTO evaluation = new EvaluationDTO();
+ evaluation.setDetectionConfigId(this.configId);
+ evaluation.setStartTime(110);
+ evaluation.setEndTime(120);
+ evaluation.setMape(0.1);
+ evaluation.setDetectorName("detection_rule_1");
+ this.evaluationDAO.save(evaluation);
+
+ DetectionHealth
+ health = new DetectionHealth.Builder(configId, startTime, endTime).addRegressionStatus(this.evaluationDAO).build();
+ Assert.assertEquals(health.getRegressionStatus().getDetectorMapes(), ImmutableMap.of(evaluation.getDetectorName(), evaluation.getMape()));
+ Assert.assertEquals(health.getRegressionStatus().getDetectorHealthStatus(), ImmutableMap.of(evaluation.getDetectorName(),
+ HealthStatus.GOOD));
+ Assert.assertEquals(health.getRegressionStatus().getHealthStatus(), HealthStatus.GOOD);
+ }
+
+ @Test
+ public void testBuildTaskStatus() {
+ long startTime = 100;
+ long endTime = 200;
+ TaskDTO task1 = new TaskDTO();
+ task1.setJobName("DETECTION_" + this.configId);
+ task1.setStatus(TaskConstants.TaskStatus.COMPLETED);
+ task1.setStartTime(110);
+ task1.setEndTime(120);
+ task1.setTaskType(TaskConstants.TaskType.DETECTION);
+ this.taskDAO.save(task1);
+
+ TaskDTO task2 = new TaskDTO();
+ task2.setJobName("DETECTION_" + this.configId);
+ task2.setStatus(TaskConstants.TaskStatus.FAILED);
+ task2.setStartTime(130);
+ task2.setEndTime(140);
+ task2.setTaskType(TaskConstants.TaskType.DETECTION);
+ this.taskDAO.save(task2);
+ DetectionHealth health = new DetectionHealth.Builder(configId, startTime, endTime).addDetectionTaskStatus(this.taskDAO, 2).build();
+ Assert.assertEquals(health.getDetectionTaskStatus().getHealthStatus(), HealthStatus.MODERATE);
+ Assert.assertEquals(health.getDetectionTaskStatus().getTaskSuccessRate(), 0.5);
+ Assert.assertEquals(health.getDetectionTaskStatus().getTasks(), Arrays.asList(task2, task1));
+ }
+
+ @Test
+ public void testAnomalyCoverageStatus() {
+ long startTime = 100;
+ long endTime = 200;
+ MergedAnomalyResultDTO anomaly = new MergedAnomalyResultDTO();
+ anomaly.setDetectionConfigId(this.configId);
+ anomaly.setStartTime(110);
+ anomaly.setEndTime(120);
+ anomaly.setMetricUrn("thirdeye:metric:1:country%3Dus");
+ this.anomalyDAO.save(anomaly);
+ MergedAnomalyResultDTO anomaly2 = new MergedAnomalyResultDTO();
+ anomaly2.setDetectionConfigId(this.configId);
+ anomaly2.setStartTime(115);
+ anomaly2.setEndTime(125);
+ anomaly2.setMetricUrn("thirdeye:metric:1:country%3Dcn");
+ this.anomalyDAO.save(anomaly2);
+ DetectionHealth health = new DetectionHealth.Builder(configId, startTime, endTime).addAnomalyCoverageStatus(this.anomalyDAO).build();
+ Assert.assertEquals(health.getAnomalyCoverageStatus().getAnomalyCoverageRatio(), 0.15);
+ Assert.assertEquals(health.getAnomalyCoverageStatus().getHealthStatus(), HealthStatus.GOOD);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void tearDown() {
+ this.testDAOProvider.cleanup();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org