You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2020/03/16 18:21:03 UTC
[incubator-pinot] branch model-downloaders created (now 90a031a)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a change to branch model-downloaders
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 90a031a [TE] anomaly detection model downloaders
This branch includes the following new commits:
new 90a031a [TE] anomaly detection model downloaders
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: [TE] anomaly detection model downloaders
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch model-downloaders
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 90a031ae1c7a4a1d3e1f6c732b8d06531fb073a4
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Mon Mar 16 11:20:19 2020 -0700
[TE] anomaly detection model downloaders
---
.../anomaly/ThirdEyeAnomalyApplication.java | 9 +++
.../thirdeye/common/ThirdEyeConfiguration.java | 11 ++++
.../dashboard/ThirdEyeDashboardApplication.java | 10 ++++
.../thirdeye/model/download/ModelDownloader.java | 28 +++++++++
.../download/ModelDownloaderConfiguration.java | 45 ++++++++++++++
.../model/download/ModelDownloaderManager.java | 68 ++++++++++++++++++++++
6 files changed, 171 insertions(+)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
index 240cdc5..08126dd 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/ThirdEyeAnomalyApplication.java
@@ -41,6 +41,7 @@ import org.apache.pinot.thirdeye.dashboard.resources.EmailResource;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
import org.apache.pinot.thirdeye.datasource.pinot.resources.PinotDataSourceResource;
+import org.apache.pinot.thirdeye.model.download.ModelDownloaderManager;
import org.apache.pinot.thirdeye.scheduler.DetectionCronScheduler;
import org.apache.pinot.thirdeye.scheduler.SubscriptionCronScheduler;
import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
@@ -77,6 +78,7 @@ public class ThirdEyeAnomalyApplication
private DataAvailabilityTaskScheduler dataAvailabilityTaskScheduler = null;
private DetectionCronScheduler detectionScheduler = null;
private SubscriptionCronScheduler subscriptionScheduler = null;
+ private ModelDownloaderManager modelDownloaderManager = null;
public static void main(final String[] args) throws Exception {
List<String> argList = new ArrayList<>(Arrays.asList(args));
@@ -199,6 +201,10 @@ public class ThirdEyeAnomalyApplication
config.getDataAvailabilitySchedulingConfiguration().getScheduleDelayInSec());
dataAvailabilityTaskScheduler.start();
}
+ if (config.getModelDownloaderConfig() != null) {
+ modelDownloaderManager = new ModelDownloaderManager(config.getModelDownloaderConfig());
+ modelDownloaderManager.start();
+ }
}
@Override
@@ -236,6 +242,9 @@ public class ThirdEyeAnomalyApplication
if (dataAvailabilityEventListenerDriver != null) {
dataAvailabilityEventListenerDriver.shutdown();
}
+ if (modelDownloaderManager != null) {
+ modelDownloaderManager.shutdown();
+ }
}
});
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java
index bbfa37d..83b0dd8 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/common/ThirdEyeConfiguration.java
@@ -28,6 +28,7 @@ import java.util.List;
import io.dropwizard.Configuration;
import java.util.Map;
+import org.apache.pinot.thirdeye.model.download.ModelDownloaderConfiguration;
public class ThirdEyeConfiguration extends Configuration {
@@ -51,6 +52,8 @@ public class ThirdEyeConfiguration extends Configuration {
private String failureFromAddress;
private String failureToAddress;
+ private List<ModelDownloaderConfiguration> modelDownloaderConfig;
+
/**
* allow cross request for local development
*/
@@ -190,4 +193,12 @@ public class ThirdEyeConfiguration extends Configuration {
public void setAlerterConfiguration(Map<String, Map<String, Object>> alerterConfigurations) {
this.alerterConfigurations = alerterConfigurations;
}
+
+ public List<ModelDownloaderConfiguration> getModelDownloaderConfig() {
+ return modelDownloaderConfig;
+ }
+
+ public void setModelDownloaderConfig(List<ModelDownloaderConfiguration> modelDownloaderConfig) {
+ this.modelDownloaderConfig = modelDownloaderConfig;
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index bf6a6c7..427173e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -78,6 +78,7 @@ import org.apache.pinot.thirdeye.detection.annotation.DetectionConfigurationReso
import org.apache.pinot.thirdeye.detection.yaml.YamlResource;
import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
+import org.apache.pinot.thirdeye.model.download.ModelDownloaderManager;
import org.apache.pinot.thirdeye.rootcause.RCAFramework;
import org.apache.pinot.thirdeye.rootcause.impl.RCAFrameworkLoader;
import org.apache.pinot.thirdeye.tracking.RequestStatisticsLogger;
@@ -113,6 +114,7 @@ public class ThirdEyeDashboardApplication
private static final Logger LOG = LoggerFactory.getLogger(ThirdEyeDashboardApplication.class);
private RequestStatisticsLogger requestStatisticsLogger;
+ private ModelDownloaderManager modelDownloaderManager;
@Override
public String getName() {
@@ -253,6 +255,11 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new AuthValueFactoryProvider.Binder<>(ThirdEyePrincipal.class));
}
+ if (config.getModelDownloaderConfig() != null) {
+ modelDownloaderManager = new ModelDownloaderManager(config.getModelDownloaderConfig());
+ modelDownloaderManager.start();
+ }
+
env.lifecycle().manage(new Managed() {
@Override
public void start() throws Exception {
@@ -265,6 +272,9 @@ public class ThirdEyeDashboardApplication
if (requestStatisticsLogger != null) {
requestStatisticsLogger.shutdown();
}
+ if (modelDownloaderManager != null) {
+ modelDownloaderManager.shutdown();
+ }
}
});
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java
new file mode 100644
index 0000000..f76db13
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloader.java
@@ -0,0 +1,28 @@
+package org.apache.pinot.thirdeye.model.download;
+
+import java.nio.file.Path;
+import java.util.Map;
+
+
+/**
+ * The model downloader interface. It downloads model files (e.x., trained tensorflow models), into a local path.
+ * The implementation of this class can be configured to run at a certain frequency in ThirdEye server, so that the
+ * models can be kept up-to-date.
+ */
+public abstract class ModelDownloader {
+ private final Map<String, Object> properties;
+
+ /**
+ * Create a model downloader.
+ * @param properties the properties
+ */
+ public ModelDownloader(Map<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ /**
+ * fetch the models into the local path.
+ * @param destination the destination path
+ */
+ public abstract void fetchModel(Path destination);
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java
new file mode 100644
index 0000000..20b4dde
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderConfiguration.java
@@ -0,0 +1,45 @@
+package org.apache.pinot.thirdeye.model.download;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+
+
+public class ModelDownloaderConfiguration {
+ private TimeGranularity runFrequency;
+ private String className;
+ private String destinationPath;
+ private Map<String, Object> properties = new HashMap<>();
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setClassName(String className) {
+ this.className = className;
+ }
+
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Map<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ public TimeGranularity getRunFrequency() {
+ return runFrequency;
+ }
+
+ public void setRunFrequency(TimeGranularity runFrequency) {
+ this.runFrequency = runFrequency;
+ }
+
+ public String getDestinationPath() {
+ return destinationPath;
+ }
+
+ public void setDestinationPath(String destinationPath) {
+ this.destinationPath = destinationPath;
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java
new file mode 100644
index 0000000..ed6f205
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/model/download/ModelDownloaderManager.java
@@ -0,0 +1,68 @@
+package org.apache.pinot.thirdeye.model.download;
+
+import java.lang.reflect.Constructor;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardService;
+import org.apache.pinot.thirdeye.common.time.TimeGranularity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * The model downloader manager. This class manages the life cycle of the model downloader.
+ * It constructs the model downloader, and then schedules the model downloader to run periodically and downloads the
+ * models into a local destination path. The class names, the run frequency and the download path can be configured.
+ */
+public class ModelDownloaderManager {
+ private static final Logger LOG = LoggerFactory.getLogger(AutoOnboardService.class);
+
+ private final List<ModelDownloaderConfiguration> configs;
+ private final Map<String, ModelDownloader> modelDownloaders;
+ private ScheduledExecutorService scheduledExecutorService;
+
+ public ModelDownloaderManager(List<ModelDownloaderConfiguration> modelDownloaderConfigs) {
+ this.configs = modelDownloaderConfigs;
+ this.modelDownloaders = new HashMap<>();
+ this.scheduledExecutorService = Executors.newScheduledThreadPool(5);
+
+ constructModelDownloaders();
+ }
+
+ private void constructModelDownloaders() {
+ for (ModelDownloaderConfiguration config : this.configs) {
+ try {
+ Constructor<?> constructor = Class.forName(config.getClassName()).getConstructor(Map.class);
+ ModelDownloader downloader = (ModelDownloader) constructor.newInstance(config.getProperties());
+ this.modelDownloaders.put(config.getClassName(), downloader);
+ } catch (Exception e) {
+ LOG.warn("Failed to initialize model downloader {}", config.getClassName(), e);
+ }
+ }
+ }
+
+ /**
+ * start the model downloader manager
+ */
+ public void start() {
+ for (ModelDownloaderConfiguration config : this.configs) {
+ TimeGranularity runFrequency = config.getRunFrequency();
+ this.scheduledExecutorService.scheduleAtFixedRate(() -> {
+ LOG.info("running the model downloader: {}", config.getClassName());
+ this.modelDownloaders.get(config.getClassName()).fetchModel(Paths.get(config.getDestinationPath()));
+ }, 0L, runFrequency.getSize(), runFrequency.getUnit());
+ }
+ }
+
+ /**
+ * shut down the manager
+ */
+ public void shutdown() {
+ LOG.info("Shutting down model downloader manager");
+ scheduledExecutorService.shutdown();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org