You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xh...@apache.org on 2019/03/27 19:01:44 UTC
[incubator-pinot] branch master updated: [TE] add exceptions for
detector API (#4021)
This is an automated email from the ASF dual-hosted git repository.
xhsun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new aefbada [TE] add exceptions for detector API (#4021)
aefbada is described below
commit aefbada67ff5ec56c59ce7605c07863da87fa3e2
Author: Xiaohui Sun <xh...@linkedin.com>
AuthorDate: Wed Mar 27 12:01:39 2019 -0700
[TE] add exceptions for detector API (#4021)
* [TE] add exceptions for detector API
* [TE] Clean up the email template (#4019)
* [TE] disable cache warmup for minute level metrics
* [TE] add exceptions for detector API
* [TE] disable cache warmup for minute level metrics
* [TE] disable cache warmup for minute level metrics
* [TE] add exceptions for detector API
* [TE] disable cache warmup for minute level metrics
* [TE] disable cache warmup for minute level metrics
* [TE] add exceptions for detector API
* [TE] disable cache warmup for minute level metrics
* [TE] disable cache warmup for minute level metrics
* [TE] revert setting job id in detection task
---
.../thirdeye/detection/DefaultDataProvider.java | 14 +++++++----
.../detection/spi/components/AnomalyDetector.java | 4 ++--
.../DetectorDataInsufficientException.java} | 27 ++++++++++------------
.../DetectorDataWrongFormatException.java} | 24 ++++++++-----------
.../DetectorException.java} | 24 +++++++------------
.../detection/wrapper/AnomalyDetectorWrapper.java | 10 +++++---
.../PercentageChangeRuleDetectorTest.java | 5 ++--
.../components/ThresholdRuleDetectorTest.java | 7 +++---
8 files changed, 54 insertions(+), 61 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
index 935b629..0a2e200 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DefaultDataProvider.java
@@ -125,10 +125,14 @@ public class DefaultDataProvider implements DataProvider {
// if the time series slice is already in cache, return directly
for (MetricSlice slice : slices){
for (Map.Entry<MetricSlice, DataFrame> entry : DETECTION_TIME_SERIES_CACHE.asMap().entrySet()) {
+ // current slice potentially contained in cache
if (entry.getKey().containSlice(slice)){
DataFrame df = entry.getValue().filter(entry.getValue().getLongs(COL_TIME).between(slice.getStart(), slice.getEnd())).dropNull(COL_TIME);
- output.put(slice, df);
- break;
+ // double check if it is cache hit
+ if (df.getLongs(COL_TIME).size() > 0) {
+ output.put(slice, df);
+ break;
+ }
}
}
}
@@ -140,14 +144,14 @@ public class DefaultDataProvider implements DataProvider {
futures.put(slice, this.executor.submit(() -> DefaultDataProvider.this.timeseriesLoader.load(slice)));
}
}
- LOG.info("Fetching {} slices of timeseries, {} cache hit, {} cache miss", slices.size(), output.size(), futures.size());
+ //LOG.info("Fetching {} slices of timeseries, {} cache hit, {} cache miss", slices.size(), output.size(), futures.size());
final long deadline = System.currentTimeMillis() + TIMEOUT;
for (MetricSlice slice : slices) {
if (!output.containsKey(slice)) {
output.put(slice, futures.get(slice).get(makeTimeout(deadline), TimeUnit.MILLISECONDS));
}
}
- LOG.info("Fetching {} slices used {} milliseconds", slices.size(), System.currentTimeMillis() - ts);
+ //LOG.info("Fetching {} slices used {} milliseconds", slices.size(), System.currentTimeMillis() - ts);
return output;
} catch (Exception e) {
@@ -236,7 +240,7 @@ public class DefaultDataProvider implements DataProvider {
// filter all child anomalies. those are kept in the parent anomaly children set.
anomalies = Collections2.filter(anomalies, mergedAnomaly -> mergedAnomaly != null && !mergedAnomaly.isChild());
- LOG.info("Fetched {} anomalies between (startTime = {}, endTime = {}) with confid Id = {}", anomalies.size(), slice.getStart(), slice.getEnd(), configId);
+ //LOG.info("Fetched {} anomalies between (startTime = {}, endTime = {}) with confid Id = {}", anomalies.size(), slice.getStart(), slice.getEnd(), configId);
output.putAll(slice, anomalies);
}
return output;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
index 181f0bf..072f977 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
@@ -21,6 +21,7 @@ package org.apache.pinot.thirdeye.detection.spi.components;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
+import org.apache.pinot.thirdeye.detection.spi.exception.DetectorException;
import org.apache.pinot.thirdeye.detection.spi.model.InputData;
import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
import java.util.List;
@@ -32,6 +33,5 @@ public interface AnomalyDetector<T extends AbstractSpec> extends BaseComponent<T
* Run detection in the specified time range and return a list of anomalies
* @return list of anomalies
*/
- List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn);
-
+ List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn) throws DetectorException;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataInsufficientException.java
similarity index 54%
copy from thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
copy to thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataInsufficientException.java
index 181f0bf..644aece 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataInsufficientException.java
@@ -17,21 +17,18 @@
* under the License.
*/
-package org.apache.pinot.thirdeye.detection.spi.components;
+package org.apache.pinot.thirdeye.detection.spi.exception;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
-import org.apache.pinot.thirdeye.detection.spi.model.InputData;
-import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
-import java.util.List;
-import org.joda.time.Interval;
-
-
-public interface AnomalyDetector<T extends AbstractSpec> extends BaseComponent<T> {
- /**
- * Run detection in the specified time range and return a list of anomalies
- * @return list of anomalies
- */
- List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn);
+/**
+ * Data is insufficient to run detection.
+ */
+public class DetectorDataInsufficientException extends DetectorException {
+ public DetectorDataInsufficientException(Exception ex) {
+ super(ex);
+ }
+ @Override
+ public String toString() {
+ return String.format("Data is insufficient to run detection.");
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataWrongFormatException.java
similarity index 54%
copy from thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
copy to thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataWrongFormatException.java
index 181f0bf..abee239 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorDataWrongFormatException.java
@@ -17,21 +17,15 @@
* under the License.
*/
-package org.apache.pinot.thirdeye.detection.spi.components;
+package org.apache.pinot.thirdeye.detection.spi.exception;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
-import org.apache.pinot.thirdeye.detection.spi.model.InputData;
-import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
-import java.util.List;
-import org.joda.time.Interval;
-
-
-public interface AnomalyDetector<T extends AbstractSpec> extends BaseComponent<T> {
- /**
- * Run detection in the specified time range and return a list of anomalies
- * @return list of anomalies
- */
- List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn);
+public class DetectorDataWrongFormatException extends DetectorException {
+ public DetectorDataWrongFormatException(Exception ex) {
+ super(ex);
+ }
+ @Override
+ public String toString() {
+ return String.format("Data format is wrong.");
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorException.java
similarity index 54%
copy from thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
copy to thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorException.java
index 181f0bf..08de1b0 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/components/AnomalyDetector.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/spi/exception/DetectorException.java
@@ -17,21 +17,13 @@
* under the License.
*/
-package org.apache.pinot.thirdeye.detection.spi.components;
-
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.detection.spec.AbstractSpec;
-import org.apache.pinot.thirdeye.detection.spi.model.InputData;
-import org.apache.pinot.thirdeye.detection.spi.model.InputDataSpec;
-import java.util.List;
-import org.joda.time.Interval;
-
-
-public interface AnomalyDetector<T extends AbstractSpec> extends BaseComponent<T> {
- /**
- * Run detection in the specified time range and return a list of anomalies
- * @return list of anomalies
- */
- List<MergedAnomalyResultDTO> runDetection(Interval window, String metricUrn);
+package org.apache.pinot.thirdeye.detection.spi.exception;
+/**
+ * Base detector exception class.
+ */
+public class DetectorException extends Exception {
+ public DetectorException(Exception ex) {
+ super(ex);
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
index eb7117e..589e987 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/wrapper/AnomalyDetectorWrapper.java
@@ -75,7 +75,8 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
private static final long DEFAULT_CACHING_PERIOD_LOOKBACK = TimeUnit.DAYS.toMillis(30);
private static final long CACHING_PERIOD_LOOKBACK_DAILY = TimeUnit.DAYS.toMillis(90);
private static final long CACHING_PERIOD_LOOKBACK_HOURLY = TimeUnit.DAYS.toMillis(60);
- private static final long CACHING_PERIOD_LOOKBACK_MINUTELY = TimeUnit.DAYS.toMillis(28);
+ // disable minute level cache warm up
+ private static final long CACHING_PERIOD_LOOKBACK_MINUTELY = TimeUnit.DAYS.toMillis(0);
private static final Logger LOG = LoggerFactory.getLogger(
AnomalyDetectorWrapper.class);
@@ -142,8 +143,11 @@ public class AnomalyDetectorWrapper extends DetectionPipeline {
// 2. to calculate current values and baseline values for the anomalies detected
// 3. anomaly detection current and baseline time series value
long cachingPeriodLookback = getCachingPeriodLookback(this.dataset.bucketTimeGranularity());
- MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - cachingPeriodLookback, endTime, this.metricEntity.getFilters());
- this.provider.fetchTimeseries(Collections.singleton(cacheSlice));
+ if (cachingPeriodLookback > 0) {
+ MetricSlice cacheSlice = MetricSlice.from(this.metricEntity.getId(), startTime - cachingPeriodLookback, endTime,
+ this.metricEntity.getFilters());
+ this.provider.fetchTimeseries(Collections.singleton(cacheSlice));
+ }
List<Interval> monitoringWindows = this.getMonitoringWindows();
List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
index 7ad66d9..9ca76a6 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/PercentageChangeRuleDetectorTest.java
@@ -36,6 +36,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.pinot.thirdeye.detection.spec.ThresholdRuleDetectorSpec;
import org.apache.pinot.thirdeye.detection.spi.components.AnomalyDetector;
+import org.apache.pinot.thirdeye.detection.spi.exception.DetectorException;
import org.joda.time.Interval;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
@@ -160,7 +161,7 @@ public class PercentageChangeRuleDetectorTest {
}
@Test
- public void testMonthlyDetectionPercentage() {
+ public void testMonthlyDetectionPercentage() throws DetectorException {
AnomalyDetector percentageRule = new PercentageChangeRuleDetector();
PercentageChangeRuleDetectorSpec spec = new PercentageChangeRuleDetectorSpec();
spec.setOffset("mo1m");
@@ -174,7 +175,7 @@ public class PercentageChangeRuleDetectorTest {
}
@Test
- public void testZeroDivide() {
+ public void testZeroDivide() throws DetectorException {
AnomalyDetector percentageRule = new PercentageChangeRuleDetector();
PercentageChangeRuleDetectorSpec spec = new PercentageChangeRuleDetectorSpec();
spec.setOffset("wo1w");
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetectorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetectorTest.java
index 51c2ad6..92d84c6 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetectorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/components/ThresholdRuleDetectorTest.java
@@ -32,6 +32,7 @@ import org.apache.pinot.thirdeye.detection.DefaultInputDataFetcher;
import org.apache.pinot.thirdeye.detection.MockDataProvider;
import org.apache.pinot.thirdeye.detection.spec.ThresholdRuleDetectorSpec;
import org.apache.pinot.thirdeye.detection.spi.components.AnomalyDetector;
+import org.apache.pinot.thirdeye.detection.spi.exception.DetectorException;
import org.joda.time.Interval;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
@@ -82,7 +83,7 @@ public class ThresholdRuleDetectorTest {
}
@Test
- public void testThresholdAlgorithmRun() {
+ public void testThresholdAlgorithmRun() throws DetectorException {
AnomalyDetector thresholdAlgorithm = new ThresholdRuleDetector();
ThresholdRuleDetectorSpec spec = new ThresholdRuleDetectorSpec();
spec.setMin(100);
@@ -97,7 +98,7 @@ public class ThresholdRuleDetectorTest {
}
@Test
- public void testMonthlyDetectionThreshold() {
+ public void testMonthlyDetectionThreshold() throws DetectorException {
AnomalyDetector thresholdRule = new ThresholdRuleDetector();
ThresholdRuleDetectorSpec spec = new ThresholdRuleDetectorSpec();
spec.setMin(200);
@@ -110,7 +111,7 @@ public class ThresholdRuleDetectorTest {
}
@Test
- public void testMonthlyDetectionThresholdMax() {
+ public void testMonthlyDetectionThresholdMax() throws DetectorException {
AnomalyDetector thresholdRule = new ThresholdRuleDetector();
ThresholdRuleDetectorSpec spec = new ThresholdRuleDetectorSpec();
spec.setMax(200);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org