You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2019/07/24 17:53:48 UTC
[incubator-pinot] branch master updated: [TE] logs and precondition
checks for pipeline re-tuning (#4465)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 1642f00 [TE] logs and precondition checks for pipeline re-tuning (#4465)
1642f00 is described below
commit 1642f00482d53ce3d724117ec7221f430e38cf42
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Wed Jul 24 10:53:42 2019 -0700
[TE] logs and precondition checks for pipeline re-tuning (#4465)
- Add logs and precondition checks for pipeline re-tuning flow.
- Remove redundant saves for detection config.
---
.../pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java | 3 +++
.../thirdeye/detection/DetectionPipelineTaskRunner.java | 11 +++++++----
.../apache/pinot/thirdeye/detection/ModelRetuneFlow.java | 13 ++++++++++++-
3 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
index e2b440b..9fcd8a2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/utils/ThirdeyeMetricsUtil.java
@@ -146,6 +146,9 @@ public class ThirdeyeMetricsUtil {
public static final Counter cubeExceptionCounter =
metricsRegistry.newCounter(ThirdeyeMetricsUtil.class, "cubeExceptionCounter");
+ public static final Counter detectionRetuneCounter =
+ metricsRegistry.newCounter(ThirdeyeMetricsUtil.class, "detectionRetuneCounter");
+
public static MetricsRegistry getMetricsRegistry() {
return metricsRegistry;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java
index 5a52a1b..5e5373c 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionPipelineTaskRunner.java
@@ -95,7 +95,7 @@ public class DetectionPipelineTaskRunner implements TaskRunner {
* @param loader pipeline loader
* @param provider pipeline data provider
*/
- DetectionPipelineTaskRunner(DetectionConfigManager detectionDAO, MergedAnomalyResultManager anomalyDAO,
+ public DetectionPipelineTaskRunner(DetectionConfigManager detectionDAO, MergedAnomalyResultManager anomalyDAO,
EvaluationManager evaluationDAO, DetectionPipelineLoader loader, DataProvider provider) {
this.detectionDAO = detectionDAO;
this.anomalyDAO = anomalyDAO;
@@ -125,7 +125,6 @@ public class DetectionPipelineTaskRunner implements TaskRunner {
}
config.setLastTimestamp(result.getLastTimestamp());
- this.detectionDAO.update(config);
for (MergedAnomalyResultDTO mergedAnomalyResultDTO : result.getAnomalies()) {
this.anomalyDAO.save(mergedAnomalyResultDTO);
@@ -138,8 +137,12 @@ public class DetectionPipelineTaskRunner implements TaskRunner {
this.evaluationDAO.save(evaluationDTO);
}
- // run maintenance flow to update model
- config = maintenanceFlow.maintain(config, Instant.now());
+ try {
+ // run maintenance flow to update model
+ config = maintenanceFlow.maintain(config, Instant.now());
+ } catch (Exception e) {
+ LOG.warn("Re-tune pipeline {} failed", config.getId(), e);
+ }
this.detectionDAO.update(config);
return Collections.emptyList();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/ModelRetuneFlow.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/ModelRetuneFlow.java
index d3d4a96..987d8d0 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/ModelRetuneFlow.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/ModelRetuneFlow.java
@@ -22,8 +22,11 @@
package org.apache.pinot.thirdeye.detection;
+import com.google.common.base.Preconditions;
+import com.sun.org.apache.xpath.internal.operations.Mod;
import java.util.Collection;
import java.util.Collections;
+import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionRegistry;
@@ -34,6 +37,10 @@ import org.apache.pinot.thirdeye.detection.spi.components.ModelEvaluator;
import org.apache.pinot.thirdeye.detection.spi.model.ModelStatus;
import org.apache.pinot.thirdeye.detection.yaml.DetectionConfigTuner;
import org.joda.time.Instant;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.pinot.thirdeye.anomaly.utils.ThirdeyeMetricsUtil.*;
/**
@@ -42,16 +49,18 @@ import org.joda.time.Instant;
*/
public class ModelRetuneFlow implements ModelMaintenanceFlow {
private static final int DEFAULT_TUNING_WINDOW_DAYS = 28;
+ private static final Logger LOG = LoggerFactory.getLogger(ModelRetuneFlow.class);
private final DataProvider provider;
private final DetectionRegistry detectionRegistry;
- ModelRetuneFlow(DataProvider provider, DetectionRegistry detectionRegistry) {
+ public ModelRetuneFlow(DataProvider provider, DetectionRegistry detectionRegistry) {
this.provider = provider;
this.detectionRegistry = detectionRegistry;
}
public DetectionConfigDTO maintain(DetectionConfigDTO config, Instant timestamp) {
+ Preconditions.checkArgument(!Objects.isNull(config.getComponents()) && !config.getComponents().isEmpty(), "Components not initialized");
if (isTunable(config)) {
// if the pipeline is tunable, get the model evaluators
Collection<? extends ModelEvaluator<? extends AbstractSpec>> modelEvaluators = getModelEvaluators(config);
@@ -59,6 +68,8 @@ public class ModelRetuneFlow implements ModelMaintenanceFlow {
for (ModelEvaluator<? extends AbstractSpec> modelEvaluator : modelEvaluators) {
// if returns bad model status, trigger model tuning
if (modelEvaluator.evaluateModel(timestamp).getStatus().equals(ModelStatus.BAD)) {
+ LOG.info("Status for detection pipeline {} is {}, re-tuning", config.getId(), ModelStatus.BAD.toString());
+ detectionRetuneCounter.inc();
DetectionConfigTuner detectionConfigTuner = new DetectionConfigTuner(config, provider);
config = detectionConfigTuner.tune(timestamp.toDateTime().minusDays(DEFAULT_TUNING_WINDOW_DAYS).getMillis(),
timestamp.getMillis());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org