You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xh...@apache.org on 2019/03/11 18:26:08 UTC
[incubator-pinot] branch master updated: Set caching look back for
different time granularities (#3947)
This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new ca0b106 Set caching look back for different time granularities (#3947)
ca0b106 is described below
commit ca0b106174b0fa00ee04d3d0ac891ce9fdd1433f
Author: Xiaohui Sun <xh...@linkedin.com>
AuthorDate: Mon Mar 11 11:26:02 2019 -0700
Set caching look back for different time granularities (#3947)
* [TE] backend - add an API to clear cache for DefaultDataProvider
* [TE] backend - set different lookback period for different time granularities
---
.../detection/wrapper/AnomalyDetectorWrapper.java | 29 +++++++++++++++++++---
1 file changed, 26 insertions(+), 3 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
index 0f1aab7..8450cf6 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
@@ -72,7 +72,10 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
private static final String PROP_DETECTOR_COMPONENT_NAME = "detectorComponentName";
private static final String PROP_TIMEZONE = "timezone";
private static final String PROP_BUCKET_PERIOD = "bucketPeriod";
- private static final long DEFAULT_CACHING_PERIOD_LOOKBACK = TimeUnit.DAYS.toMillis(90);
+ private static final long DEFAULT_CACHING_PERIOD_LOOKBACK = TimeUnit.DAYS.toMillis(30);
+ private static final long CACHING_PERIOD_LOOKBACK_DAILY = TimeUnit.DAYS.toMillis(90);
+ private static final long CACHING_PERIOD_LOOKBACK_HOURLY = TimeUnit.DAYS.toMillis(60);
+ private static final long CACHING_PERIOD_LOOKBACK_MINUTELY = TimeUnit.DAYS.toMillis(28);
private static final Logger LOG = LoggerFactory.getLogger(
AnomalyDetectorWrapper.class);
@@ -138,7 +141,8 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
// 1. get the last time stamp for the time series.
// 2. to calculate current values and baseline values for the anomalies detected
// 3. anomaly detection current and baseline time series value
- MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - DEFAULT_CACHING_PERIOD_LOOKBACK, endTime, this.metricEntity.getFilters());
+ long cachingPeriodLookback = getCachingPeriodLookback(this.dataset.bucketTimeGranularity());
+ MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - cachingPeriodLookback, endTime, this.metricEntity.getFilters());
this.provider.fetchTimeseries(Collections.singleton(cacheSlice));
List<Interval> monitoringWindows = this.getMonitoringWindows();
@@ -208,7 +212,8 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
LOG.info("Will run detection in window {}", window);
}
// pre cache the time series for the whole detection time period instead of fetching for each window
- MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - DEFAULT_CACHING_PERIOD_LOOKBACK, endTime, this.metricEntity.getFilters(), toTimeGranularity(this.bucketPeriod));
+ long cachingPeriodLookback = getCachingPeriodLookback(this.dataset.bucketTimeGranularity());
+ MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - cachingPeriodLookback, endTime, this.metricEntity.getFilters(), toTimeGranularity(this.bucketPeriod));
this.provider.fetchTimeseries(Collections.singleton(cacheSlice));
return monitoringWindows;
} catch (Exception e) {
@@ -218,6 +223,24 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
return Collections.singletonList(new Interval(startTime, endTime));
}
+ private long getCachingPeriodLookback(TimeGranularity granularity) {
+ long period;
+ switch (granularity.getUnit()) {
+ case DAYS:
+ period = CACHING_PERIOD_LOOKBACK_DAILY;
+ break;
+ case HOURS:
+ period = CACHING_PERIOD_LOOKBACK_HOURLY;
+ break;
+ case MINUTES:
+ period = CACHING_PERIOD_LOOKBACK_MINUTELY;
+ break;
+ default:
+ period = DEFAULT_CACHING_PERIOD_LOOKBACK;
+ }
+ return period;
+ }
+
// get the list of monitoring window end times
private List<Long> getMonitoringWindowEndTimes() {
List<Long> endTimes = new ArrayList<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org