You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/01/31 20:48:40 UTC

[incubator-pinot] branch master updated: [TE] Queries for fetching detection & notification owner report (#3773)

This is an automated email from the ASF dual-hosted git repository.

akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 26f2aad  [TE] Queries for fetching detection & notification owner report (#3773)
26f2aad is described below

commit 26f2aad92ad17427083de4459e7c53e546c43fcc
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Thu Jan 31 12:48:35 2019 -0800

    [TE] Queries for fetching detection & notification owner report (#3773)
---
 .../tools/RunAdhocDatabaseQueriesTool.java         | 146 ++++++++++++++++++++-
 1 file changed, 145 insertions(+), 1 deletion(-)

diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
index 9b732c9..911f3c5 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/RunAdhocDatabaseQueriesTool.java
@@ -19,9 +19,12 @@ package org.apache.pinot.thirdeye.tools;
 import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
 import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
 import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
+import org.apache.pinot.thirdeye.datalayer.bao.ApplicationManager;
 import org.apache.pinot.thirdeye.datalayer.bao.ClassificationConfigManager;
 import org.apache.pinot.thirdeye.datalayer.bao.DataCompletenessConfigManager;
 import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
 import org.apache.pinot.thirdeye.datalayer.bao.DetectionStatusManager;
 import org.apache.pinot.thirdeye.datalayer.bao.JobManager;
 import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
@@ -31,10 +34,14 @@ import org.apache.pinot.thirdeye.datalayer.bao.RawAnomalyResultManager;
 import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AlertConfigManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.AnomalyFunctionManagerImpl;
+import org.apache.pinot.thirdeye.datalayer.bao.jdbc.ApplicationManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.ClassificationConfigManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.DataCompletenessConfigManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.DatasetConfigManagerImpl;
+import org.apache.pinot.thirdeye.datalayer.bao.jdbc.DetectionAlertConfigManagerImpl;
+import org.apache.pinot.thirdeye.datalayer.bao.jdbc.DetectionConfigManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.DetectionStatusManagerImpl;
+import org.apache.pinot.thirdeye.datalayer.bao.jdbc.EventManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.JobManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.MergedAnomalyResultManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.bao.jdbc.MetricConfigManagerImpl;
@@ -44,9 +51,12 @@ import org.apache.pinot.thirdeye.datalayer.bao.jdbc.TaskManagerImpl;
 import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.ClassificationConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.DataCompletenessConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.DetectionStatusDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.JobDTO;
 import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
@@ -56,16 +66,19 @@ import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
 import org.apache.pinot.thirdeye.datalayer.util.DaoProviderUtil;
 
 import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+
 import java.io.File;
 import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 
 import java.util.Set;
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -76,6 +89,9 @@ public class RunAdhocDatabaseQueriesTool {
 
   private static final Logger LOG = LoggerFactory.getLogger(RunAdhocDatabaseQueriesTool.class);
 
+  private DetectionConfigManager detectionConfigDAO;
+  private DetectionAlertConfigManager detectionAlertConfigDAO;
+  private EventManagerImpl eventDAO;
   private AnomalyFunctionManager anomalyFunctionDAO;
   private RawAnomalyResultManager rawResultDAO;
   private MergedAnomalyResultManager mergedResultDAO;
@@ -88,6 +104,7 @@ public class RunAdhocDatabaseQueriesTool {
   private DetectionStatusManager detectionStatusDAO;
   private AlertConfigManager alertConfigDAO;
   private ClassificationConfigManager classificationConfigDAO;
+  private ApplicationManager applicationDAO;
 
   public RunAdhocDatabaseQueriesTool(File persistenceFile)
       throws Exception {
@@ -96,6 +113,9 @@ public class RunAdhocDatabaseQueriesTool {
 
   public void init(File persistenceFile) throws Exception {
     DaoProviderUtil.init(persistenceFile);
+    detectionConfigDAO = DaoProviderUtil.getInstance(DetectionConfigManagerImpl.class);
+    detectionAlertConfigDAO = DaoProviderUtil.getInstance(DetectionAlertConfigManagerImpl.class);
+    eventDAO = DaoProviderUtil.getInstance(EventManagerImpl.class);
     anomalyFunctionDAO = DaoProviderUtil.getInstance(AnomalyFunctionManagerImpl.class);
     rawResultDAO = DaoProviderUtil.getInstance(RawAnomalyResultManagerImpl.class);
     mergedResultDAO = DaoProviderUtil.getInstance(MergedAnomalyResultManagerImpl.class);
@@ -108,6 +128,7 @@ public class RunAdhocDatabaseQueriesTool {
     detectionStatusDAO = DaoProviderUtil.getInstance(DetectionStatusManagerImpl.class);
     alertConfigDAO = DaoProviderUtil.getInstance(AlertConfigManagerImpl.class);
     classificationConfigDAO = DaoProviderUtil.getInstance(ClassificationConfigManagerImpl.class);
+    applicationDAO = DaoProviderUtil.getInstance(ApplicationManagerImpl.class);
   }
 
   private void toggleAnomalyFunction(Long id) {
@@ -433,6 +454,128 @@ public class RunAdhocDatabaseQueriesTool {
     }
   }
 
+  /**
+   * Generates a report of the status and owner of all the un-subscribed anomaly functions
+   */
+  private void unsubscribedDetections(){
+    List<ApplicationDTO> apps = applicationDAO.findAll();
+
+    List<AnomalyFunctionDTO> allAnomalyFuncs = anomalyFunctionDAO.findAll();
+    LOG.info("Total number of funcs: " + allAnomalyFuncs.size());
+
+    Set<Long> allAnomalyFuncsIds = new HashSet<>();
+    Set<Long> subscribedFuncsIds = new HashSet<>();
+    for (AnomalyFunctionDTO anom : allAnomalyFuncs) {
+      allAnomalyFuncsIds.add(anom.getId());
+    }
+
+    List<AnomalyFunctionDTO> subscribedFuncs = new ArrayList<>();
+    for (ApplicationDTO app : apps) {
+      List<AlertConfigDTO> alertConfigDTOS = alertConfigDAO.findByPredicate(Predicate.EQ("application", app.getApplication()));
+      if (alertConfigDTOS != null) {
+        for (AlertConfigDTO alertDTO : alertConfigDTOS) {
+          if (alertDTO.getEmailConfig() != null) {
+            if (alertDTO.getEmailConfig().getFunctionIds() != null) {
+              for (long id : alertDTO.getEmailConfig().getFunctionIds()) {
+                AnomalyFunctionDTO func = anomalyFunctionDAO.findById(id);
+                if (func != null) {
+                  subscribedFuncs.add(func);
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+
+    for (AnomalyFunctionDTO anom : subscribedFuncs) {
+      subscribedFuncsIds.add(anom.getId());
+    }
+
+    allAnomalyFuncsIds.removeAll(subscribedFuncsIds);
+
+    for (long id : allAnomalyFuncsIds) {
+      AnomalyFunctionDTO func = anomalyFunctionDAO.findById(id);
+      System.out.println(String.format("%s\t%s\t%s\t%s\t%s", func.getId(), func.getFunctionName(), func.getIsActive(), func.getCreatedBy(), func.getUpdatedBy()));
+
+    }
+  }
+
+  /**
+   * Generates a report of the status, owner and recipients of all the subscription groups by application
+   */
+  private void notificationOwners(){
+    List<ApplicationDTO> apps = applicationDAO.findAll();
+    for (ApplicationDTO app : apps) {
+      List<AlertConfigDTO> alertConfigDTOS = alertConfigDAO.findByPredicate(Predicate.EQ("application", app.getApplication()));
+      for (AlertConfigDTO alertDTO : alertConfigDTOS) {
+        String recipients = fetchRecipients(alertDTO.getReceiverAddresses());
+        System.out.println(String.format("%s\t%s\t%s\t%s\t%s", app.getApplication(), alertDTO.getName(), alertDTO.isActive(), alertDTO.getCreatedBy(), recipients));
+      }
+    }
+  }
+
+  /**
+   * Generates a report of the status, owner and recipients of all the subscribed anomaly functions by application
+   */
+  private void detectionOwners(){
+    List<ApplicationDTO> apps = applicationDAO.findAll();
+
+    for (ApplicationDTO app : apps) {
+      List<AlertConfigDTO> alertConfigDTOS = alertConfigDAO.findByPredicate(Predicate.EQ("application", app.getApplication()));
+      if (alertConfigDTOS != null) {
+        for (AlertConfigDTO alertDTO : alertConfigDTOS) {
+          if (alertDTO.getEmailConfig() != null) {
+            if (alertDTO.getEmailConfig().getFunctionIds() != null) {
+              for (long id : alertDTO.getEmailConfig().getFunctionIds()) {
+                AnomalyFunctionDTO func = anomalyFunctionDAO.findById(id);
+                if (func != null) {
+                  String recipients = fetchRecipients(alertDTO.getReceiverAddresses());
+                  System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s", app.getApplication(), func.getId(), func.getFunctionName(),
+                      func.getIsActive(), func.getCreatedBy(), String.join(",", recipients)));
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private String fetchRecipients(DetectionAlertFilterRecipients receivers) {
+    String recipients = String.join(", ", receivers.getTo()).trim();
+    while (recipients.startsWith(",")) {
+      recipients = recipients.substring(1, recipients.length());
+    }
+    while (recipients.endsWith(",")) {
+      recipients = recipients.substring(0, recipients.length() - 1);
+    }
+    return recipients;
+  }
+
+  private void cleanup(){
+    List<DetectionConfigDTO> detectionConfigDTOS = detectionConfigDAO.findAll();
+
+    for (DetectionConfigDTO detFunction : detectionConfigDTOS) {
+      // Delete all the anomalies generated by the functions
+      List<AnomalyFunctionDTO> anomalies = anomalyFunctionDAO.findByPredicate(Predicate.EQ("baseId", detFunction.getId()));
+      for (AnomalyFunctionDTO anomaly : anomalies) {
+        anomalyFunctionDAO.delete(anomaly);
+      }
+      // Delete all the functions
+      detectionConfigDAO.delete(detFunction);
+    }
+
+    List<DetectionAlertConfigDTO> detectionAlertConfigDTOS = detectionAlertConfigDAO.findAll();
+    for (DetectionAlertConfigDTO alert : detectionAlertConfigDTOS) {
+      if (alert.getId() == 89049435 || alert.getId() == 89235978) {
+        continue;
+      }
+
+      detectionAlertConfigDAO.delete(alert);
+    }
+  }
+
   public static void main(String[] args) throws Exception {
 
     File persistenceFile = new File(args[0]);
@@ -441,7 +584,8 @@ public class RunAdhocDatabaseQueriesTool {
       System.exit(1);
     }
     RunAdhocDatabaseQueriesTool dq = new RunAdhocDatabaseQueriesTool(persistenceFile);
-
+    dq.unsubscribedDetections();
+    LOG.info("DONE");
   }
 
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org