You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/06/25 23:46:42 UTC
[incubator-pinot] branch master updated: [TE] update detection
health (#4364)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 4ce688e [TE] update detection health (#4364)
4ce688e is described below
commit 4ce688e7b9a70bb99966c54ae558c02710b74259
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Tue Jun 25 16:46:38 2019 -0700
[TE] update detection health (#4364)
Update the detection health to include task counts for different task status. Including, COMPLETED, TIMEOUT, FAILED, WAITING tasks.
---
.../thirdeye/detection/health/DetectionHealth.java | 6 +++--
.../detection/health/DetectionTaskStatus.java | 31 ++++++++++++++++------
.../detection/health/DetectionHealthTest.java | 3 +++
3 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java
index a2496c0..347d6c5 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionHealth.java
@@ -143,7 +143,8 @@ public class DetectionHealth {
}
/**
- * Add the global health status in the report built by the builder, consider regression health, coverage ratio and task health
+ * Add the global health status in the report built by the builder, consider both regression health, coverage ratio and task health.
+ * The overall health can be generated only if regression health, coverage ratio and task health are available.
* @return the builder
*/
public Builder addOverallHealth() {
@@ -228,7 +229,8 @@ public class DetectionHealth {
Predicate.LT(COL_NAME_START_TIME, endTime), Predicate.GT(COL_NAME_END_TIME, startTime),
Predicate.EQ(COL_NAME_TASK_TYPE, TaskConstants.TaskType.DETECTION.toString()),
Predicate.IN(COL_NAME_TASK_STATUS, new String[]{TaskConstants.TaskStatus.COMPLETED.toString(),
- TaskConstants.TaskStatus.FAILED.toString(), TaskConstants.TaskStatus.TIMEOUT.toString()})));
+ TaskConstants.TaskStatus.FAILED.toString(), TaskConstants.TaskStatus.TIMEOUT.toString(),
+ TaskConstants.TaskStatus.WAITING.toString()})));
tasks.sort(Comparator.comparingLong(TaskBean::getStartTime).reversed());
// limit the task size
tasks = tasks.stream().limit(this.taskLimit).collect(Collectors.toList());
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java
index 1133f0b..84acc6b 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/health/DetectionTaskStatus.java
@@ -21,6 +21,7 @@
package org.apache.pinot.thirdeye.detection.health;
import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -41,6 +42,15 @@ public class DetectionTaskStatus {
@JsonProperty
private final HealthStatus healthStatus;
+ // the counting for detection task status
+ @JsonProperty
+ private final Map<TaskConstants.TaskStatus, Long> taskCounts = new HashMap<TaskConstants.TaskStatus, Long>() {{
+ put(TaskConstants.TaskStatus.COMPLETED, 0L);
+ put(TaskConstants.TaskStatus.FAILED, 0L);
+ put(TaskConstants.TaskStatus.WAITING, 0L);
+ put(TaskConstants.TaskStatus.TIMEOUT, 0L);
+ }};
+
// the list of tasks for the detection config
@JsonProperty
private final List<TaskDTO> tasks;
@@ -48,10 +58,11 @@ public class DetectionTaskStatus {
private static final double TASK_SUCCESS_RATE_BAD_THRESHOLD = 0.2;
private static final double TASK_SUCCESS_RATE_MODERATE_THRESHOLD = 0.8;
- public DetectionTaskStatus(double taskSuccessRate, HealthStatus healthStatus, List<TaskDTO> tasks) {
+ public DetectionTaskStatus(double taskSuccessRate, HealthStatus healthStatus, Map<TaskConstants.TaskStatus, Long> counts, List<TaskDTO> tasks) {
this.taskSuccessRate = taskSuccessRate;
this.healthStatus = healthStatus;
this.tasks = tasks;
+ this.taskCounts.putAll(counts);
}
public double getTaskSuccessRate() {
@@ -66,20 +77,24 @@ public class DetectionTaskStatus {
return tasks;
}
- public static DetectionTaskStatus fromTasks(List<TaskDTO> tasks) {
+ public Map<TaskConstants.TaskStatus, Long> getTaskCounts() {
+ return taskCounts;
+ }
+ public static DetectionTaskStatus fromTasks(List<TaskDTO> tasks) {
double taskSuccessRate = Double.NaN;
// count the number of tasks by task status
- Map<TaskConstants.TaskStatus, Long> count =
+ Map<TaskConstants.TaskStatus, Long> counts =
tasks.stream().collect(Collectors.groupingBy(TaskBean::getStatus, Collectors.counting()));
- if (count.size() != 0) {
- long completedTasks = count.getOrDefault(TaskConstants.TaskStatus.COMPLETED, 0L);
- long failedTasks = count.getOrDefault(
+
+ if (counts.size() != 0) {
+ long completedTasks = counts.getOrDefault(TaskConstants.TaskStatus.COMPLETED, 0L);
+ long failedTasks = counts.getOrDefault(
TaskConstants.TaskStatus.FAILED, 0L);
- long timeoutTasks = count.getOrDefault(TaskConstants.TaskStatus.TIMEOUT, 0L);
+ long timeoutTasks = counts.getOrDefault(TaskConstants.TaskStatus.TIMEOUT, 0L);
taskSuccessRate = (double) completedTasks / (failedTasks + timeoutTasks + completedTasks);
}
- return new DetectionTaskStatus(taskSuccessRate, classifyTaskStatus(taskSuccessRate), tasks);
+ return new DetectionTaskStatus(taskSuccessRate, classifyTaskStatus(taskSuccessRate), counts, tasks);
}
private static HealthStatus classifyTaskStatus(double taskSuccessRate) {
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java
index 2fa5ba1..a88d7e7 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/health/DetectionHealthTest.java
@@ -95,6 +95,9 @@ public class DetectionHealthTest {
this.taskDAO.save(task2);
DetectionHealth health = new DetectionHealth.Builder(configId, startTime, endTime).addDetectionTaskStatus(this.taskDAO, 2).build();
Assert.assertEquals(health.getDetectionTaskStatus().getHealthStatus(), HealthStatus.MODERATE);
+ Assert.assertEquals(health.getDetectionTaskStatus().getTaskCounts(),
+ ImmutableMap.of(TaskConstants.TaskStatus.COMPLETED, 1L, TaskConstants.TaskStatus.TIMEOUT, 0L,
+ TaskConstants.TaskStatus.WAITING, 0L, TaskConstants.TaskStatus.FAILED, 1L));
Assert.assertEquals(health.getDetectionTaskStatus().getTaskSuccessRate(), 0.5);
Assert.assertEquals(health.getDetectionTaskStatus().getTasks(), Arrays.asList(task2, task1));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org