You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2020/09/13 21:55:25 UTC
[incubator-pinot] branch master updated: [TE] entity anomaly
logging for ad-hoc debugging (#6001)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 143f398 [TE] entity anomaly logging for ad-hoc debugging (#6001)
143f398 is described below
commit 143f398fc1a3e23df991c43820049ad3cd1dc730
Author: Akshay Rai <ak...@linkedin.com>
AuthorDate: Sun Sep 13 14:55:08 2020 -0700
[TE] entity anomaly logging for ad-hoc debugging (#6001)
---
.../tools/RunAdhocDatabaseQueriesTool.java | 87 ++++++++++++++++++++--
1 file changed, 80 insertions(+), 7 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
index c2da5df..f8d576e 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
@@ -18,6 +18,7 @@ package org.apache.pinot.thirdeye.tools;
import java.util.HashMap;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.Multimap;
import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
import org.apache.pinot.thirdeye.constant.AnomalyResultSource;
import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
@@ -80,9 +81,22 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.collections4.CollectionUtils;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import org.jfree.util.Log;
+import org.apache.pinot.thirdeye.detection.cache.builder.AnomaliesCacheBuilder;
+import org.apache.pinot.thirdeye.detection.cache.builder.TimeSeriesCacheBuilder;
+import org.apache.pinot.thirdeye.detection.spi.model.AnomalySlice;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,6 +125,11 @@ public class RunAdhocDatabaseQueriesTool {
private ClassificationConfigManager classificationConfigDAO;
private ApplicationManager applicationDAO;
+ private static final DateTimeZone TIMEZONE = DateTimeZone.forID("America/Los_Angeles");
+ private static final DateTimeFormatter DATE_FORMAT = DateTimeFormat.forPattern("YYYY-MM-dd");
+ private static final String ENTITY_STATS_TEMPLATE = "#children = {}, properties = {}";
+ private static final String ENTITY_TIME_TEMPLATE = "[create {}, start {}, end {}]";
+
public RunAdhocDatabaseQueriesTool(File persistenceFile)
throws Exception {
init(persistenceFile);
@@ -692,8 +711,7 @@ public class RunAdhocDatabaseQueriesTool {
// find all recent anomalies (last 14 days) with end time <= watermark;
long lookback = TimeUnit.DAYS.toMillis(14);
- Predicate predicate = Predicate.AND(
- Predicate.LE("createTime", clockEntry.getValue()),
+ Predicate predicate = Predicate.AND(Predicate.LE("createTime", clockEntry.getValue()),
Predicate.GT("createTime", clockEntry.getValue() - lookback),
Predicate.EQ("detectionConfigId", detectionConfigDTO.getId()));
List<MergedAnomalyResultDTO> anomalies = mergedResultDAO.findByPredicate(predicate);
@@ -723,9 +741,65 @@ public class RunAdhocDatabaseQueriesTool {
}
}
- public static void main(String[] args) throws Exception {
+ private void printEntityAnomalyDetails(MergedAnomalyResultDTO anomaly, String indent, int index) {
+ LOG.info("");
+ LOG.info("Exploring Entity Anomaly {} with id {}", index, anomaly.getId());
+ LOG.info(ENTITY_STATS_TEMPLATE, anomaly.getChildren().size(), anomaly.getProperties());
+ LOG.info(ENTITY_TIME_TEMPLATE,
+ new DateTime(anomaly.getCreatedTime(), TIMEZONE),
+ DATE_FORMAT.print(new DateTime(anomaly.getStartTime(), TIMEZONE)),
+ DATE_FORMAT.print(new DateTime(anomaly.getEndTime(), TIMEZONE)));
+ }
- File persistenceFile = new File("/Users/akrai/persistence-linux.yml");
+ /**
+ * Visualizes the entity anomalies by printing them
+ *
+ * Eg: dq.printEntityAnomalyTrees(158750221, 0, System.currentTimeMillis())
+ *
+ * @param detectionId The detection id whose anomalies need to be printed
+ * @param start The start time of the anomaly slice
+ * @param end The end time of the anomaly slice
+ */
+ private void printEntityAnomalyTrees(long detectionId, long start, long end) {
+ TimeSeriesLoader timeseriesLoader =
+ new DefaultTimeSeriesLoader(metricConfigDAO, datasetConfigDAO,
+ ThirdEyeCacheRegistry.getInstance().getQueryCache(),
+ ThirdEyeCacheRegistry.getInstance().getTimeSeriesCache());
+ AggregationLoader aggregationLoader =
+ new DefaultAggregationLoader(metricConfigDAO, datasetConfigDAO,
+ ThirdEyeCacheRegistry.getInstance().getQueryCache(),
+ ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache());
+ DefaultDataProvider provider =
+ new DefaultDataProvider(metricConfigDAO, datasetConfigDAO, eventDAO, mergedResultDAO,
+ DAORegistry.getInstance().getEvaluationManager(), timeseriesLoader, aggregationLoader,
+ new DetectionPipelineLoader(), TimeSeriesCacheBuilder.getInstance(), AnomaliesCacheBuilder.getInstance());
+
+ AnomalySlice anomalySlice = new AnomalySlice();
+ anomalySlice = anomalySlice.withDetectionId(detectionId).withStart(start).withEnd(end);
+ Multimap<AnomalySlice, MergedAnomalyResultDTO>
+ sliceToAnomaliesMap = provider.fetchAnomalies(Collections.singletonList(anomalySlice));
+
+ LOG.info("Total number of entity anomalies = " + sliceToAnomaliesMap.values().size());
+
+ int i = 1;
+ for (MergedAnomalyResultDTO parentAnomaly : sliceToAnomaliesMap.values()) {
+ printEntityAnomalyDetails(parentAnomaly, "", i);
+ int j = 1;
+ for (MergedAnomalyResultDTO child : parentAnomaly.getChildren()) {
+ printEntityAnomalyDetails(parentAnomaly, "\t", j);
+ int k = 1;
+ for (MergedAnomalyResultDTO grandchild : child.getChildren()) {
+ printEntityAnomalyDetails(grandchild, "\t\t", k);
+ k++;
+ }
+ j++;
+ }
+ i++;
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ File persistenceFile = new File("/path/to/persistence.yml");
if (!persistenceFile.exists()) {
System.err.println("Missing file:" + persistenceFile);
System.exit(1);
@@ -734,5 +808,4 @@ public class RunAdhocDatabaseQueriesTool {
dq.disableAllActiveDetections(Collections.singleton(160640739L));
LOG.info("DONE");
}
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org