You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/10/03 05:24:28 UTC
[incubator-pinot] branch master updated: [TE] Clean up legacy
detection related classes; update ADContentFormatterContext (#4657)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 9746e82 [TE] Clean up legacy detection related classes; update ADContentFormatterContext (#4657)
9746e82 is described below
commit 9746e82dc3ea69ace25cb99928f525e0a5a5e442
Author: Akshay Rai <ak...@linkedin.com>
AuthorDate: Wed Oct 2 22:24:23 2019 -0700
[TE] Clean up legacy detection related classes; update ADContentFormatterContext (#4657)
* Updated ADContentFormatterContext with latest detectionConfig and detectionAlertConfig
* Selectively removed a lot of legacy detection related classes
---
.../detection/lib/FunctionReplayRunnable.java | 295 ---------
.../anomaly/onboard/DetectionOnboardResource.java | 185 ------
.../thirdeye/anomaly/onboard/ReplayTaskInfo.java | 46 --
.../thirdeye/anomaly/onboard/ReplayTaskRunner.java | 147 -----
.../onboard/framework/BaseDetectionOnboardJob.java | 43 --
.../framework/BaseDetectionOnboardTask.java | 50 --
.../framework/DetectionOnBoardJobRunner.java | 184 ------
.../DetectionOnboardExecutionContext.java | 54 --
.../onboard/framework/DetectionOnboardJob.java | 50 --
.../framework/DetectionOnboardJobContext.java | 113 ----
.../framework/DetectionOnboardJobStatus.java | 93 ---
.../onboard/framework/DetectionOnboardTask.java | 50 --
.../framework/DetectionOnboardTaskContext.java | 52 --
.../framework/DetectionOnboardTaskRunner.java | 55 --
.../framework/DetectionOnboardTaskStatus.java | 69 ---
.../tasks/AlertFilterAutoTuneOnboardingTask.java | 127 ----
.../tasks/DataPreparationOnboardingTask.java | 79 ---
.../onboard/tasks/DefaultDetectionOnboardJob.java | 319 ----------
.../tasks/FunctionCreationOnboardingTask.java | 366 -----------
.../tasks/FunctionReplayOnboardingTask.java | 129 ----
.../onboard/tasks/NotificationOnboardingTask.java | 160 -----
.../onboard/utils/FunctionCreationUtils.java | 52 --
.../anomaly/onboard/utils/PropertyCheckUtils.java | 50 --
.../thirdeye/anomaly/task/TaskInfoFactory.java | 4 -
.../thirdeye/anomaly/task/TaskRunnerFactory.java | 3 -
.../dashboard/ThirdEyeDashboardApplication.java | 5 -
.../dashboard/resources/AnomalyResource.java | 214 -------
.../dashboard/resources/DetectionJobResource.java | 439 -------------
.../dashboard/resources/OnboardResource.java | 681 ---------------------
.../bao/jdbc/MergedAnomalyResultManagerImpl.java | 2 +
.../datalayer/dto/MergedAnomalyResultDTO.java | 1 +
.../datalayer/pojo/AnomalyFunctionBean.java | 15 +-
.../alert/scheme/DetectionEmailAlerter.java | 14 +-
.../content/BaseNotificationContent.java | 10 +-
.../content/templates/EntityGroupKeyContent.java | 2 +-
.../templates/HierarchicalAnomaliesContent.java | 2 +-
.../content/templates/MetricAnomaliesContent.java | 7 +-
.../templates/OnboardingNotificationContent.java | 114 ----
.../formatter/ADContentFormatterContext.java | 24 +-
.../formatter/channels/EmailContentFormatter.java | 7 +-
.../thirdeye/alert/feed/TestUnionAnomalyFeed.java | 104 ----
.../alert/fetcher/TestContinuumAnomalyFetcher.java | 85 ---
.../fetcher/TestUnnotifiedAnomalyFetcher.java | 81 ---
.../onboard/DetectionOnBoardJobRunnerTest.java | 344 -----------
.../onboard/DetectionOnboardResourceTest.java | 97 ---
.../anomaly/onboard/OnboardingTaskTestUtils.java | 72 ---
.../anomaly/onboard/tasks/TestOnboardingTasks.java | 128 ----
.../dashboard/resource/OnboardResourceTest.java | 100 ---
.../pinot/thirdeye/datalayer/DaoTestUtils.java | 26 +
.../pinot/thirdeye/detection/DataProviderTest.java | 32 +-
.../templates/TestEntityGroupKeyContent.java | 4 +-
.../TestHierarchicalAnomaliesContent.java | 5 +-
.../templates/TestMetricAnomaliesContent.java | 60 +-
.../TestOnboardingNotificationContent.java | 138 -----
.../tools/CleanupAndRegenerateAnomaliesTool.java | 262 --------
.../src/test/resources/sample-detection-config.yml | 4 +-
.../resources/test-metric-anomalies-template.html | 12 +-
57 files changed, 144 insertions(+), 5722 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/detection/lib/FunctionReplayRunnable.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/detection/lib/FunctionReplayRunnable.java
deleted file mode 100644
index 6c49876..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/detection/lib/FunctionReplayRunnable.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.detection.lib;
-
-import org.apache.pinot.thirdeye.anomaly.detection.DetectionJobScheduler;
-import org.apache.pinot.thirdeye.anomalydetection.performanceEvaluation.PerformanceEvaluate;
-import org.apache.pinot.thirdeye.anomalydetection.performanceEvaluation.PerformanceEvaluateHelper;
-import org.apache.pinot.thirdeye.anomalydetection.performanceEvaluation.PerformanceEvaluationMethod;
-import org.apache.pinot.thirdeye.dashboard.resources.OnboardResource;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AutotuneConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AutotuneConfigDTO;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class FunctionReplayRunnable implements Runnable {
- private static final Logger LOG = LoggerFactory.getLogger(FunctionReplayRunnable.class);
- private DetectionJobScheduler detectionJobScheduler;
- private MergedAnomalyResultManager mergedAnomalyResultDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private AutotuneMethodType autotuneMethodType;
- private AutotuneConfigManager autotuneConfigDAO;
- private PerformanceEvaluationMethod performanceEvaluationMethod;
- private long tuningFunctionId;
- private DateTime replayStart;
- private DateTime replayEnd;
- private double goal;
- private boolean isForceBackfill;
- private Map<String, String> tuningParameter;
- private Long functionAutotuneConfigId;
- private boolean speedUp;
- private boolean selfKill;
- private long lastClonedFunctionId;
-
- public FunctionReplayRunnable(DetectionJobScheduler detectionJobScheduler, AnomalyFunctionManager anomalyFunctionDAO,
- MergedAnomalyResultManager mergedAnomalyResultDAO, AutotuneConfigManager autotuneConfigDAO) {
- this.detectionJobScheduler = detectionJobScheduler;
- this.mergedAnomalyResultDAO = mergedAnomalyResultDAO;
- this.anomalyFunctionDAO = anomalyFunctionDAO;
- this.autotuneConfigDAO = autotuneConfigDAO;
- setSpeedUp(true);
- setForceBackfill(true);
- setSelfKill(true);
- }
-
- public FunctionReplayRunnable(DetectionJobScheduler detectionJobScheduler, AnomalyFunctionManager anomalyFunctionDAO,
- MergedAnomalyResultManager mergedAnomalyResultDAO, AutotuneConfigManager autotuneConfigDAO,
- Map<String, String> tuningParameter, long tuningFunctionId, DateTime replayStart, DateTime replayEnd, double goal,
- long functionAutotuneConfigId, boolean isForceBackfill, boolean selfKill) {
- this(detectionJobScheduler, anomalyFunctionDAO, mergedAnomalyResultDAO, autotuneConfigDAO);
- setTuningFunctionId(tuningFunctionId);
- setReplayStart(replayStart);
- setReplayEnd(replayEnd);
- setForceBackfill(isForceBackfill);
- setTuningParameter(tuningParameter);
- setFunctionAutotuneConfigId(functionAutotuneConfigId);
- setSpeedUp(true);
- setGoal(goal);
- setSelfKill(selfKill);
- }
-
- public FunctionReplayRunnable(DetectionJobScheduler detectionJobScheduler, AnomalyFunctionManager anomalyFunctionDAO,
- MergedAnomalyResultManager mergedAnomalyResultDAO, Map<String, String> tuningParameter, long tuningFunctionId,
- DateTime replayStart, DateTime replayEnd, boolean selfKill) {
- this(detectionJobScheduler, anomalyFunctionDAO, mergedAnomalyResultDAO, null);
- setTuningFunctionId(tuningFunctionId);
- setReplayStart(replayStart);
- setReplayEnd(replayEnd);
- setForceBackfill(true);
- setTuningParameter(tuningParameter);
- setSpeedUp(true);
- setSelfKill(selfKill);
- }
-
- public static void speedup(AnomalyFunctionDTO anomalyFunctionDTO) {
- switch (anomalyFunctionDTO.getWindowUnit()) {
- case NANOSECONDS:
- case MICROSECONDS:
- case MILLISECONDS:
- case SECONDS:
- case MINUTES: // These TimeUnits are not currently in use
- case HOURS:
- case DAYS:
- /*
- SignTest takes HOURS data, but changing to 7 days won't affect the final result
- SPLINE takes 1 DAYS data, for heuristic, we extend it to 7 days.
- */
- default:
- anomalyFunctionDTO.setWindowSize(7);
- anomalyFunctionDTO.setWindowUnit(TimeUnit.DAYS);
- anomalyFunctionDTO.setCron("0 0 0 ? * MON *");
- }
- }
-
- @Override
- public void run() {
- long currentTime = System.currentTimeMillis();
- long clonedFunctionId = 0l;
- OnboardResource onboardResource = new OnboardResource(anomalyFunctionDAO, mergedAnomalyResultDAO);
- StringBuilder functionName = new StringBuilder("clone");
- for (Map.Entry<String, String> entry : tuningParameter.entrySet()) {
- functionName.append("_");
- functionName.append(entry.getKey());
- functionName.append("_");
- functionName.append(entry.getValue());
- }
- try {
- clonedFunctionId = onboardResource.cloneAnomalyFunctionById(tuningFunctionId, functionName.toString(), false);
- this.lastClonedFunctionId = clonedFunctionId;
- }
- catch (Exception e) {
- LOG.error("Unable to clone function {} with given name {}", tuningFunctionId, functionName.toString(), e);
- return;
- }
-
- AnomalyFunctionDTO anomalyFunctionDTO = anomalyFunctionDAO.findById(clonedFunctionId);
- // Remove alert filters
- anomalyFunctionDTO.setAlertFilter(null);
-
- int originWindowSize = anomalyFunctionDTO.getWindowSize();
- TimeUnit originWindowUnit = anomalyFunctionDTO.getWindowUnit();
- String originCron = anomalyFunctionDTO.getCron();
-
- // enlarge window size so that we can speed-up the replay speed
- if(speedUp) {
- FunctionReplayRunnable.speedup(anomalyFunctionDTO);
- }
-
- // Set Properties
- anomalyFunctionDTO.updateProperties(tuningParameter);
- anomalyFunctionDTO.setActive(true);
-
- anomalyFunctionDAO.update(anomalyFunctionDTO);
-
- List<Long> functionIdList = new ArrayList<>();
- functionIdList.add(clonedFunctionId);
- detectionJobScheduler.synchronousBackFill(functionIdList, replayStart, replayEnd, isForceBackfill);
-
- if(autotuneConfigDAO != null) { // if no functionAutotuneId, skip update
- PerformanceEvaluate performanceEvaluator =
- PerformanceEvaluateHelper.getPerformanceEvaluator(performanceEvaluationMethod, tuningFunctionId,
- clonedFunctionId, new Interval(replayStart.getMillis(), replayEnd.getMillis()), mergedAnomalyResultDAO);
- double performance = performanceEvaluator.evaluate();
-
- AutotuneConfigDTO targetAutotuneDTO = autotuneConfigDAO.findById(functionAutotuneConfigId);
-
- Map<String, Double> prevPerformance = targetAutotuneDTO.getPerformance();
- // if there is no previous performance, update performance directly
- // Otherwise, compare the performance, and update if betterW
- if (prevPerformance == null || prevPerformance.isEmpty() ||
- Math.abs(prevPerformance.get(performanceEvaluationMethod.name()) - goal) > Math.abs(performance - goal)) {
- targetAutotuneDTO.setConfiguration(tuningParameter);
- Map<String, Double> newPerformance = targetAutotuneDTO.getPerformance();
- newPerformance.put(performanceEvaluationMethod.name(), performance);
- targetAutotuneDTO.setPerformance(newPerformance);
- targetAutotuneDTO.setAvgRunningTime((System.currentTimeMillis() - currentTime) / 1000);
- targetAutotuneDTO.setLastUpdateTimestamp(System.currentTimeMillis());
- }
- String message = (targetAutotuneDTO.getMessage().isEmpty()) ? "" : (targetAutotuneDTO.getMessage() + ";");
-
- targetAutotuneDTO.setMessage(message + tuningParameter.toString() + ":" + performance);
-
- autotuneConfigDAO.update(targetAutotuneDTO);
- }
-
- // clean up and kill itself
- if(selfKill) {
- onboardResource.deleteExistingAnomalies(clonedFunctionId, replayStart.getMillis(),
- replayEnd.getMillis());
- anomalyFunctionDAO.deleteById(clonedFunctionId);
- } else {
- anomalyFunctionDTO.setWindowSize(originWindowSize);
- anomalyFunctionDTO.setWindowUnit(originWindowUnit);
- anomalyFunctionDTO.setCron(originCron);
- anomalyFunctionDAO.update(anomalyFunctionDTO);
- }
- }
-
-
- public long getTuningFunctionId() {
- return tuningFunctionId;
- }
-
- public void setTuningFunctionId(long functionId) {
- this.tuningFunctionId = functionId;
- }
-
- public DateTime getReplayStart() {
- return replayStart;
- }
-
- public void setReplayStart(DateTime replayStart) {
- this.replayStart = replayStart;
- }
-
- public DateTime getReplayEnd() {
- return replayEnd;
- }
-
- public void setReplayEnd(DateTime replayEnd) {
- this.replayEnd = replayEnd;
- }
-
- public boolean isForceBackfill() {
- return isForceBackfill;
- }
-
- public void setForceBackfill(boolean forceBackfill) {
- isForceBackfill = forceBackfill;
- }
-
- public Map<String, String> getTuningParameter() {
- return tuningParameter;
- }
-
- public void setTuningParameter(Map<String, String> tuningParameter) {
- this.tuningParameter = tuningParameter;
- }
-
- public AutotuneMethodType getAutotuneMethodType() {
- return autotuneMethodType;
- }
-
- public void setAutotuneMethodType(AutotuneMethodType autotuneMethodType) {
- this.autotuneMethodType = autotuneMethodType;
- }
-
- public PerformanceEvaluationMethod getPerformanceEvaluationMethod() {
- return performanceEvaluationMethod;
- }
-
- public void setPerformanceEvaluationMethod(PerformanceEvaluationMethod performanceEvaluationMethod) {
- this.performanceEvaluationMethod = performanceEvaluationMethod;
- }
-
- public double getGoal() {
- return goal;
- }
-
- public void setGoal(double goal) {
- this.goal = goal;
- }
-
- public Long getFunctionAutotuneConfigId() {
- return functionAutotuneConfigId;
- }
-
- public void setFunctionAutotuneConfigId(Long functionAutotuneConfigId) {
- this.functionAutotuneConfigId = functionAutotuneConfigId;
- }
-
- public boolean isSpeedUp() {
- return speedUp;
- }
-
- public void setSpeedUp(boolean speedUp) {
- this.speedUp = speedUp;
- }
-
- public boolean isSelfKill() {
- return selfKill;
- }
-
- public void setSelfKill(boolean selfKill) {
- this.selfKill = selfKill;
- }
-
- public long getLastClonedFunctionId(){return lastClonedFunctionId;}
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResource.java
deleted file mode 100644
index e7ae582..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResource.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobStatus;
-import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
-import org.apache.pinot.thirdeye.api.Constants;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.TaskDTO;
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import java.util.Map;
-import javax.validation.constraints.NotNull;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.MediaType;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * NOTE: This resource was re-written for backwards-compatibility with existing UI code.
- */
-@Path("/detection-onboard")
-@Produces(MediaType.APPLICATION_JSON)
-@Api(tags = {Constants.ONBOARD_TAG})
-@Deprecated
-public class DetectionOnboardResource {
- private static final Logger LOG = LoggerFactory.getLogger(DetectionOnboardResource.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private final TaskManager taskDAO;
- private final AnomalyFunctionManager anomalyDAO;
-
- public DetectionOnboardResource(TaskManager taskDAO, AnomalyFunctionManager anomalyDAO) {
- this.taskDAO = taskDAO;
- this.anomalyDAO = anomalyDAO;
- }
-
- /**
- * Create a job with the given name and properties.
- *
- * @param jobName the unique name for the job. Must be equal to anomaly function name.
- * @param jsonPayload a map of properties in JSON string.
- *
- * @return Job name
- */
- @POST
- @Path("/create-job")
- @ApiOperation("POST request to update an existing alert function with properties payload")
- public String createDetectionOnboardingJob(@ApiParam(required = true) @NotNull @QueryParam("jobName") String jobName,
- @ApiParam("jsonPayload") String jsonPayload) {
-
- // Check user input
- if (jsonPayload == null) {
- jsonPayload = "";
- }
-
- long anomalyFunctionId;
- Map<String, String> properties;
-
- try {
- properties = OBJECT_MAPPER.readValue(jsonPayload, Map.class);
-
- // create minimal anomaly function
- AnomalyFunctionDTO function = new AnomalyFunctionDTO();
- function.setFunctionName(jobName);
- function.setMetricId(-1);
- function.setIsActive(false);
- anomalyFunctionId = anomalyDAO.save(function);
-
- } catch (Exception e) {
- LOG.error("Error creating anomaly function '{}'", jobName, e);
- return makeErrorStatus(-1, jobName, JobConstants.JobStatus.FAILED);
- }
-
- try {
- // launch minimal task (with job id == anomalyFunctionId)
- // NOTE: the existing task framework is an incredible hack
-
- ReplayTaskInfo taskInfo = new ReplayTaskInfo();
- taskInfo.setJobName(jobName);
- taskInfo.setProperties(properties);
-
- String taskInfoJson = OBJECT_MAPPER.writeValueAsString(taskInfo);
-
- TaskDTO task = new TaskDTO();
- task.setTaskType(TaskConstants.TaskType.REPLAY);
- task.setJobName(jobName);
- task.setStatus(TaskConstants.TaskStatus.WAITING);
- task.setTaskInfo(taskInfoJson);
- task.setJobId(anomalyFunctionId);
- this.taskDAO.save(task);
-
- return detectionOnboardJobStatusToJsonString(new DetectionOnboardJobStatus(anomalyFunctionId, jobName, JobConstants.JobStatus.SCHEDULED, ""));
- } catch (Exception e) {
- LOG.error("Error creating onboarding job '{}'", jobName, e);
- this.anomalyDAO.deleteById(anomalyFunctionId);
- return makeErrorStatus(-1, jobName, JobConstants.JobStatus.FAILED);
- }
-
- }
-
- /**
- * Returns the job status in JSON string.
- *
- * @param jobId the name of the job.
- *
- * @return the job status in JSON string.
- */
- @GET
- @Path("/get-status")
- @ApiOperation("GET request for job status (a sequence of events including create, replay, autotune)")
- public String getDetectionOnboardingJobStatus(@QueryParam("jobId") long jobId) {
- AnomalyFunctionDTO anomalyFunction = this.anomalyDAO.findById(jobId);
- if (anomalyFunction == null) {
- return makeErrorStatus(jobId, "Unknown Job", JobConstants.JobStatus.UNKNOWN);
- }
-
- DetectionOnboardJobStatus detectionOnboardingJobStatus = anomalyFunction.getOnboardJobStatus();
- if (detectionOnboardingJobStatus == null) {
- return makeErrorStatus(jobId, anomalyFunction.getFunctionName(), JobConstants.JobStatus.SCHEDULED);
- }
-
- return detectionOnboardJobStatusToJsonString(detectionOnboardingJobStatus);
- }
-
- /**
- * Helper. Returns serialized job status
- *
- * @param jobId job id
- * @param jobName job name
- * @param jobStatus job status
- * @return
- */
- private String makeErrorStatus(long jobId, String jobName, JobConstants.JobStatus jobStatus) {
- return detectionOnboardJobStatusToJsonString(
- new DetectionOnboardJobStatus(jobId, jobName, jobStatus, String.format("Job %d", jobId)));
- }
-
- /**
- * Converts job status to a JSON string or returns the plain text of any exception that is thrown during the
- * conversion.
- *
- * @param detectionOnboardingJobStatus the job status to be converted to a JSON string.
- *
- * @return the JSON string of the given job status.
- */
- private String detectionOnboardJobStatusToJsonString(DetectionOnboardJobStatus detectionOnboardingJobStatus) {
- try {
- return OBJECT_MAPPER.writeValueAsString(detectionOnboardingJobStatus);
- } catch (JsonProcessingException e) {
- LOG.error("Failed to convert job status to a json string.", e);
- return ExceptionUtils.getStackTrace(e);
- }
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskInfo.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskInfo.java
deleted file mode 100644
index fb9874b..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskInfo.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import org.apache.pinot.thirdeye.anomaly.task.TaskInfo;
-import java.util.Map;
-
-
-@Deprecated
-public class ReplayTaskInfo implements TaskInfo {
- private String jobName;
- private Map<String, String> properties;
-
- public String getJobName() {
- return jobName;
- }
-
- public void setJobName(String jobName) {
- this.jobName = jobName;
- }
-
- public Map<String, String> getProperties() {
- return properties;
- }
-
- public void setProperties(Map<String, String> properties) {
- this.properties = properties;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskRunner.java
deleted file mode 100644
index 393c741..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/ReplayTaskRunner.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.SmtpConfiguration;
-import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnBoardJobRunner;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobStatus;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.tasks.DefaultDetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.task.TaskContext;
-import org.apache.pinot.thirdeye.anomaly.task.TaskInfo;
-import org.apache.pinot.thirdeye.anomaly.task.TaskResult;
-import org.apache.pinot.thirdeye.anomaly.task.TaskRunner;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.MapConfiguration;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.pinot.thirdeye.anomaly.SmtpConfiguration.SMTP_CONFIG_KEY;
-
-
-/**
- * Traditional ThirdEye task runner wrapping an onboarding framework job
- */
-@Deprecated
-public class ReplayTaskRunner implements TaskRunner {
- private static final Logger LOG = LoggerFactory.getLogger(ReplayTaskRunner.class);
-
- private final AnomalyFunctionManager anomalyFunctionDAO;
-
- public ReplayTaskRunner() {
- this.anomalyFunctionDAO = DAORegistry.getInstance().getAnomalyFunctionDAO();
- }
-
- @Override
- public List<TaskResult> execute(TaskInfo taskInfo, TaskContext taskContext) throws Exception {
- ReplayTaskInfo replayTaskInfo = (ReplayTaskInfo) taskInfo;
-
- // fetch anomaly function
- final String jobName = replayTaskInfo.getJobName();
- final AnomalyFunctionDTO anomalyFunction = this.anomalyFunctionDAO.findWhereNameEquals(jobName);
- Preconditions.checkNotNull(anomalyFunction, String.format("Could not find anomaly function '%s'", jobName));
-
- final long jobId = anomalyFunction.getId();
-
- try {
- // Put System Configuration into properties
- Map<String, String> properties = new HashMap<>(replayTaskInfo.getProperties());
- Configuration systemConfig = toConfiguration(taskContext.getThirdEyeAnomalyConfiguration());
- Iterator systemConfigKeyIterator = systemConfig.getKeys();
- while (systemConfigKeyIterator.hasNext()) {
- String systemConfigKey = systemConfigKeyIterator.next().toString();
- properties.put(systemConfigKey, systemConfig.getString(systemConfigKey));
- }
-
- LOG.info("Creating replay job with properties: {}", properties);
-
- DetectionOnboardJob job = new DefaultDetectionOnboardJob(replayTaskInfo.getJobName(), properties);
-
- Preconditions.checkNotNull(job, "Job cannot be null.");
- Preconditions.checkNotNull(job.getName(), "Job name cannot be null.");
-
- // Initialize the tasks and their configuration
- Configuration configuration = job.getTaskConfiguration();
- Preconditions.checkNotNull(configuration, String.format("Job %s returns a null configuration.", jobName));
-
- List<DetectionOnboardTask> tasks = job.getTasks();
- Preconditions.checkNotNull(tasks, "Job %s returns a null task list.", jobName);
-
- DetectionOnboardJobStatus jobStatus = new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
- DetectionOnBoardJobRunner jobRunner = new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus);
-
- // execute
- jobRunner.run();
-
- // update job status
- updateJobStatus(jobId, jobStatus);
-
- } catch (Exception e) {
- LOG.error("Replay job failed", e);
- updateJobStatus(jobId, new DetectionOnboardJobStatus(jobId, jobName,
- JobConstants.JobStatus.FAILED, String.format("Execution Error: %s", ExceptionUtils.getStackTrace(e))));
- }
-
- return Collections.emptyList();
- }
-
- private void updateJobStatus(long jobId, DetectionOnboardJobStatus jobStatus) {
- final AnomalyFunctionDTO anomalyFunction = this.anomalyFunctionDAO.findById(jobId);
- anomalyFunction.setOnboardJobStatus(jobStatus);
- this.anomalyFunctionDAO.save(anomalyFunction);
- }
-
- private static Configuration toConfiguration(ThirdEyeAnomalyConfiguration thirdeyeConfigs) {
- Preconditions.checkNotNull(thirdeyeConfigs);
- SmtpConfiguration smtpConfiguration = SmtpConfiguration.createFromProperties(
- thirdeyeConfigs.getAlerterConfiguration().get(SMTP_CONFIG_KEY));
- Preconditions.checkNotNull(smtpConfiguration);
-
- Map<String, String> systemConfig = new HashMap<>();
- systemConfig.put(DefaultDetectionOnboardJob.FUNCTION_FACTORY_CONFIG_PATH, thirdeyeConfigs.getFunctionConfigPath());
- systemConfig.put(DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY_CONFIG_PATH, thirdeyeConfigs.getAlertFilterConfigPath());
- systemConfig.put(DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH, thirdeyeConfigs.getFilterAutotuneConfigPath());
- systemConfig.put(DefaultDetectionOnboardJob.SMTP_HOST, smtpConfiguration.getSmtpHost());
- systemConfig.put(DefaultDetectionOnboardJob.SMTP_PORT, Integer.toString(smtpConfiguration.getSmtpPort()));
- systemConfig.put(DefaultDetectionOnboardJob.THIRDEYE_DASHBOARD_HOST, thirdeyeConfigs.getDashboardHost());
- systemConfig.put(DefaultDetectionOnboardJob.PHANTON_JS_PATH, thirdeyeConfigs.getPhantomJsPath());
- systemConfig.put(DefaultDetectionOnboardJob.ROOT_DIR, thirdeyeConfigs.getRootDir());
- systemConfig.put(DefaultDetectionOnboardJob.DEFAULT_ALERT_SENDER_ADDRESS, thirdeyeConfigs.getFailureFromAddress());
- systemConfig.put(DefaultDetectionOnboardJob.DEFAULT_ALERT_RECEIVER_ADDRESS, thirdeyeConfigs.getFailureToAddress());
-
- return new MapConfiguration(systemConfig);
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardJob.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardJob.java
deleted file mode 100644
index f073a44..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardJob.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
-
-public abstract class BaseDetectionOnboardJob implements DetectionOnboardJob {
- private final String jobName;
- protected ImmutableMap<String, String> properties;
-
- public BaseDetectionOnboardJob(String jobName, Map<String, String> properties) {
- Preconditions.checkNotNull(jobName);
- Preconditions.checkArgument(StringUtils.isNotBlank(jobName.trim()));
- Preconditions.checkNotNull(properties);
-
- this.jobName = jobName;
- this.properties = ImmutableMap.copyOf(properties);
- }
-
- public String getName() {
- return jobName;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardTask.java
deleted file mode 100644
index 6f9f03f..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/BaseDetectionOnboardTask.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.StringUtils;
-
-public abstract class BaseDetectionOnboardTask implements DetectionOnboardTask {
- private final String taskName;
- protected DetectionOnboardTaskContext taskContext = new DetectionOnboardTaskContext();
-
- public BaseDetectionOnboardTask(String taskName) {
- Preconditions.checkNotNull(taskName);
- Preconditions.checkArgument(StringUtils.isNotBlank(taskName.trim()));
- this.taskName = taskName;
- }
-
- @Override
- public String getTaskName() {
- return taskName;
- }
-
- @Override
- public void setTaskContext(DetectionOnboardTaskContext taskContext) {
- Preconditions.checkNotNull(taskContext);
- this.taskContext = taskContext;
- }
-
- @Override
- public DetectionOnboardTaskContext getTaskContext() {
- return taskContext;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnBoardJobRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnBoardJobRunner.java
deleted file mode 100644
index a334a82..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnBoardJobRunner.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.SmtpConfiguration;
-import org.apache.pinot.thirdeye.anomaly.alert.util.EmailHelper;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import org.apache.pinot.thirdeye.anomaly.onboard.tasks.DefaultDetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
-import org.apache.pinot.thirdeye.anomaly.utils.EmailUtils;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.apache.commons.mail.EmailException;
-import org.apache.commons.mail.HtmlEmail;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.quartz.JobExecutionException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DetectionOnBoardJobRunner implements Runnable {
- private static final Logger LOG = LoggerFactory.getLogger(DetectionOnBoardJobRunner.class);
- private final ExecutorService executorService = Executors.newCachedThreadPool();
- private final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- public static final String ABORT_ON_FAILURE= "abortOnFailure";
- private final DetectionOnboardJobContext jobContext;
- private final List<DetectionOnboardTask> tasks;
- private final DetectionOnboardJobStatus jobStatus;
- private final int taskTimeOutSize;
- private final TimeUnit taskTimeOutUnit;
- private final SmtpConfiguration smtpConfiguration;
- private final String failureNotificationSender;
- private final DetectionAlertFilterRecipients failureNotificationReceiver;
- private final boolean notifyIfFails;
-
- public DetectionOnBoardJobRunner(DetectionOnboardJobContext jobContext, List<DetectionOnboardTask> tasks,
- DetectionOnboardJobStatus jobStatus) {
- this(jobContext, tasks, jobStatus, 30, TimeUnit.MINUTES);
- }
-
- public DetectionOnBoardJobRunner(DetectionOnboardJobContext jobContext, List<DetectionOnboardTask> tasks,
- DetectionOnboardJobStatus jobStatus, int taskTimeOutSize, TimeUnit taskTimeOutUnit) {
- Preconditions.checkNotNull(jobContext);
- Preconditions.checkNotNull(tasks);
- Preconditions.checkNotNull(jobStatus);
- Preconditions.checkNotNull(taskTimeOutUnit);
-
- this.jobContext = jobContext;
- this.tasks = tasks;
- this.jobStatus = jobStatus;
- this.taskTimeOutSize = taskTimeOutSize;
- this.taskTimeOutUnit = taskTimeOutUnit;
- Configuration configuration = jobContext.getConfiguration();
- if (configuration.containsKey(DefaultDetectionOnboardJob.SMTP_HOST)) {
- smtpConfiguration = new SmtpConfiguration();
- smtpConfiguration.setSmtpHost(configuration.getString(DefaultDetectionOnboardJob.SMTP_HOST));
- smtpConfiguration.setSmtpPort(configuration.getInt(DefaultDetectionOnboardJob.SMTP_PORT));
- } else {
- smtpConfiguration = null;
- }
- failureNotificationSender = configuration.getString(DefaultDetectionOnboardJob.DEFAULT_ALERT_SENDER_ADDRESS);
- Set<String> toAddresses = EmailUtils.getValidEmailAddresses(configuration.getString(DefaultDetectionOnboardJob.DEFAULT_ALERT_RECEIVER_ADDRESS));
- failureNotificationReceiver = new DetectionAlertFilterRecipients(toAddresses);
- notifyIfFails = configuration.getBoolean(DefaultDetectionOnboardJob.NOTIFY_IF_FAILS,
- DefaultDetectionOnboardJob.DEFAULT_NOTIFY_IF_FAILS);
- }
-
- @Override
- public void run() {
- Preconditions.checkNotNull(jobContext);
- Preconditions.checkNotNull(tasks);
- Preconditions.checkNotNull(jobStatus);
-
- for (DetectionOnboardTask task : tasks) {
- DetectionOnboardTaskStatus taskStatus =
- new DetectionOnboardTaskStatus(task.getTaskName(), TaskConstants.TaskStatus.WAITING, "");
- jobStatus.addTaskStatus(taskStatus);
-
- // Construct Task context and configuration
- Configuration taskConfig = jobContext.getConfiguration().subset(task.getTaskName());
- final boolean abortOnFailure = taskConfig.getBoolean(ABORT_ON_FAILURE, true);
- DetectionOnboardTaskContext taskContext = new DetectionOnboardTaskContext();
- taskContext.setConfiguration(taskConfig);
- taskContext.setExecutionContext(jobContext.getExecutionContext());
-
- // Submit task
- try {
- task.setTaskContext(taskContext);
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.RUNNING);
- Future<DetectionOnboardTaskStatus> taskFuture = executorService.submit(new DetectionOnboardTaskRunner(task));
- // Wait until time out
- DetectionOnboardTaskStatus returnedTaskStatus = taskFuture.get(taskTimeOutSize, taskTimeOutUnit);
- taskStatus.setTaskStatus(returnedTaskStatus.getTaskStatus());
- taskStatus.setMessage(returnedTaskStatus.getMessage());
- } catch (TimeoutException e) {
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.TIMEOUT);
- LOG.warn("Task {} timed out.", task.getTaskName());
- } catch (InterruptedException e) {
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.FAILED);
- taskStatus.setMessage("Job execution is interrupted.");
- jobStatus.setJobStatus(JobConstants.JobStatus.FAILED);
- jobStatus.setMessage(String.format("Job execution is interrupted: %s", ExceptionUtils.getStackTrace(e)));
- LOG.error("Job execution is interrupted.", e);
- return; // Stop executing the job because the thread to execute the job is interrupted.
- } catch (Exception e) {
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.FAILED);
- taskStatus.setMessage(String.format("Execution Error: %s", ExceptionUtils.getStackTrace(e)));
- LOG.error("Encountered unknown error while running job {}.", jobContext.getJobName(), e);
- }
-
- // Notify upon exception
- if (notifyIfFails && !TaskConstants.TaskStatus.COMPLETED.equals(taskStatus.getTaskStatus())) {
- if (smtpConfiguration == null || StringUtils.isBlank(failureNotificationSender)
- || failureNotificationReceiver.getTo().isEmpty()) {
- LOG.warn("SmtpConfiguration, and email sender/recipients cannot be null or empty");
- } else {
- try {
- sendFailureEmail();
- } catch (JobExecutionException e) {
- LOG.warn("Unable to send failure emails");
- }
- }
- }
-
- if (abortOnFailure && !TaskConstants.TaskStatus.COMPLETED.equals(taskStatus.getTaskStatus())) {
- jobStatus.setJobStatus(JobConstants.JobStatus.FAILED);
- LOG.error("Failed to execute job {}.", jobContext.getJobName());
- return;
- }
- }
- jobStatus.setJobStatus(JobConstants.JobStatus.COMPLETED);
- }
-
- private void sendFailureEmail() throws JobExecutionException {
- HtmlEmail email = new HtmlEmail();
- String subject = String
- .format("[ThirdEye Onboarding Job] FAILED Onboarding Job Id=%d for config %s", jobContext.getJobId(),
- jobContext.getJobName());
- String jobStatusString;
- try {
- jobStatusString = OBJECT_MAPPER.writeValueAsString(jobStatus);
- } catch (IOException e) {
- LOG.warn("Unable to parse job context {}", jobContext);
- jobStatusString = jobStatus.toString();
- }
- String htmlBody = String
- .format("<h1>Job Status</h1><p>%s</p>", jobStatusString);
- try {
- EmailHelper
- .sendEmailWithHtml(email, smtpConfiguration, subject, htmlBody,
- failureNotificationSender, failureNotificationReceiver);
- } catch (EmailException e) {
- throw new JobExecutionException(e);
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardExecutionContext.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardExecutionContext.java
deleted file mode 100644
index 0092527..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardExecutionContext.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A container to store the execution context (i.e., the input for a task or the output from a task) of a job.
- * Currently, we assume that the order of tasks forms a total order and therefore there is no concurrent tasks.
- */
-public class DetectionOnboardExecutionContext {
- private Map<String, Object> executionResults = new HashMap<>();
-
- /**
- * Sets the execution context (i.e., result or input) with the given key.
- *
- * @param key the key to associate with the given execution context (value).
- * @param value the execution context.
- *
- * @return the previous value associated with the key, or null if there was no mapping for the key.
- */
- public Object setExecutionResult(String key, Object value) {
- return this.executionResults.put(key, value);
- }
-
- /**
- * Returns the execution context to which the specified key is associated, or null if no such mapping for the key.
- *
- * @param key key with which the specified execution context is to be associated.
- *
- * @return the execution context to which the specified key is associated, or null if no such mapping for the key.
- */
- public Object getExecutionResult(String key) {
- return this.executionResults.get(key);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJob.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJob.java
deleted file mode 100644
index 5c89eb2..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJob.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import java.util.List;
-import org.apache.commons.configuration2.Configuration;
-
-public interface DetectionOnboardJob {
-
- /**
- * Returns the unique name of this job.
- * @return the unique name of this job.
- */
- String getName();
-
- /**
- * Returns the configuration for the tasks in this job execution. The configuration should be built from the
- * properties map that is given in the initialized method. The property for each task in the built configuration
- * should has the corresponding task's name. Assume that a job has two tasks with names: "task1" and "task2",
- * respectively. The property for "task1" must have the prefix "task1.". Similarly, the configuration for "task2" have
- * the prefix "task2".
- *
- * @return the configuration for the tasks in this job execution.
- */
- Configuration getTaskConfiguration();
-
- /**
- * Returns the list of tasks of this job. The tasks will be executed following their order in the list.
- *
- * @return the list of tasks of this job.
- */
- List<DetectionOnboardTask> getTasks();
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobContext.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobContext.java
deleted file mode 100644
index 04c5e5d..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobContext.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.lang3.StringUtils;
-
-public class DetectionOnboardJobContext {
- private String jobName;
- private long jobId;
- private Configuration configuration;
- private DetectionOnboardExecutionContext executionContext = new DetectionOnboardExecutionContext();
-
- public DetectionOnboardJobContext(long jobId, String jobName, Configuration configuration) {
- setJobName(jobName);
- setJobId(jobId);
- setConfiguration(configuration);
- }
-
- /**
- * Returns the unique name of the job.
- * @return the unique name of the job.
- */
- public String getJobName() {
- return jobName;
- }
-
- /**
- * Sets the name of the job. The name cannot be null or an empty string. Any white space before and after the name
- * will be trimmed.
- *
- * @param jobName the name of the job.
- */
- private void setJobName(String jobName) {
- Preconditions.checkNotNull(jobName);
- Preconditions.checkArgument(StringUtils.isNotBlank(jobName.trim()), "Job name cannot be empty.");
- this.jobName = jobName.trim();
- }
-
- /**
- * Returns the id of the job.
- *
- * @return the id of the job.
- */
- public long getJobId() {
- return jobId;
- }
-
- /**
- * Sets the id of the job.
- *
- * @param jobId the id of the job.
- */
- private void setJobId(long jobId) {
- this.jobId = jobId;
- }
-
- /**
- * Returns the configuration of the job.
- *
- * @return the configuration of the job.
- */
- public Configuration getConfiguration() {
- return configuration;
- }
-
- /**
- * Sets the configuration of the job.
- *
- * @param configuration the configuration of the job.
- */
- private void setConfiguration(Configuration configuration) {
- Preconditions.checkNotNull(configuration);
- this.configuration = configuration;
- }
-
- /**
- * Returns the execution context (i.e., execution results from all tasks) of this job.
- *
- * @return the execution context of this job.
- */
- public DetectionOnboardExecutionContext getExecutionContext() {
- return executionContext;
- }
-
- /**
- * Sets the execution context (i.e., execution results from all tasks) of this job. The context cannot be null.
- *
- * @param executionContext the execution context.
- */
- public void setExecutionContext(DetectionOnboardExecutionContext executionContext) {
- Preconditions.checkNotNull(executionContext);
- this.executionContext = executionContext;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobStatus.java
deleted file mode 100644
index f254cc7..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardJobStatus.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import java.util.ArrayList;
-import java.util.List;
-
-public class DetectionOnboardJobStatus {
- private long jobId = -1;
- private String jobName = "Unknown Job Name";
- private JobConstants.JobStatus jobStatus = JobConstants.JobStatus.SCHEDULED;
- private String message = "";
- private List<DetectionOnboardTaskStatus> taskStatuses = new ArrayList<>();
-
- public DetectionOnboardJobStatus() { }
-
- public DetectionOnboardJobStatus(long jobId, String jobName) {
- this.setJobId(jobId);
- this.setJobName(jobName);
- }
-
- public DetectionOnboardJobStatus(long jobId, String jobName, JobConstants.JobStatus jobStatus, String message) {
- this.setJobId(jobId);
- this.setJobName(jobName);
- this.setJobStatus(jobStatus);
- this.setMessage(message);
- }
-
- public long getJobId() {
- return jobId;
- }
-
- public void setJobId(long jobId) {
- this.jobId = jobId;
- }
-
- public String getJobName() {
- return jobName;
- }
-
- public void setJobName(String jobName) {
- Preconditions.checkArgument(!Strings.isNullOrEmpty(jobName));
- this.jobName = jobName;
- }
-
- public JobConstants.JobStatus getJobStatus() {
- return jobStatus;
- }
-
- public void setJobStatus(JobConstants.JobStatus jobStatus) {
- Preconditions.checkNotNull(jobStatus);
- this.jobStatus = jobStatus;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- Preconditions.checkNotNull(message);
- this.message = message;
- }
-
- public void addTaskStatus(DetectionOnboardTaskStatus taskStatus) {
- Preconditions.checkNotNull(taskStatus);
- taskStatuses.add(taskStatus);
- }
-
- public List<DetectionOnboardTaskStatus> getTaskStatuses() {
- return ImmutableList.copyOf(taskStatuses);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTask.java
deleted file mode 100644
index 890f157..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTask.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-
-public interface DetectionOnboardTask {
-
- /**
- * Returns the unique name of the task.
- *
- * @return the unique name of the task.
- */
- String getTaskName();
-
- /**
- * Sets the task context of this task.
- *
- * @param taskContext the task context of this task.
- */
- void setTaskContext(DetectionOnboardTaskContext taskContext);
-
- /**
- * Returns the task context of this task.
- * @return the task context of this task.
- */
- DetectionOnboardTaskContext getTaskContext();
-
- /**
- * Executes the task. To fail this task, throw exceptions. The job executor will catch the exception and store
- * it in the message in the execution status of this task.
- */
- void run();
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskContext.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskContext.java
deleted file mode 100644
index 7ee48c5..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskContext.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import java.util.Collections;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.MapConfiguration;
-
-
-public class DetectionOnboardTaskContext {
- private Configuration configuration = new MapConfiguration(Collections.emptyMap());
- private DetectionOnboardExecutionContext executionContext = new DetectionOnboardExecutionContext();
-
- public DetectionOnboardTaskContext() {
- }
-
- public Configuration getConfiguration() {
- return configuration;
- }
-
- public void setConfiguration(Configuration configuration) {
- Preconditions.checkNotNull(configuration);
- this.configuration = configuration;
- }
-
- public DetectionOnboardExecutionContext getExecutionContext() {
- return executionContext;
- }
-
- public void setExecutionContext(DetectionOnboardExecutionContext executionContext) {
- Preconditions.checkNotNull(executionContext);
- this.executionContext = executionContext;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskRunner.java
deleted file mode 100644
index 8e705d3..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskRunner.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
-import java.util.concurrent.Callable;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class DetectionOnboardTaskRunner implements Callable<DetectionOnboardTaskStatus> {
- private static final Logger LOG = LoggerFactory.getLogger(DetectionOnboardTaskRunner.class);
-
- private final DetectionOnboardTask task;
-
- public DetectionOnboardTaskRunner(DetectionOnboardTask task) {
- Preconditions.checkNotNull(task);
- this.task = task;
- }
-
- @Override
- public DetectionOnboardTaskStatus call() throws Exception {
- DetectionOnboardTaskStatus taskStatus = new DetectionOnboardTaskStatus(task.getTaskName());
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.RUNNING);
-
- try {
- task.run();
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.COMPLETED);
- } catch (Exception e) {
- taskStatus.setTaskStatus(TaskConstants.TaskStatus.FAILED);
- taskStatus.setMessage(ExceptionUtils.getStackTrace(e));
- LOG.error("Error encountered when running task: {}", task.getTaskName(), e);
- }
-
- return taskStatus;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskStatus.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskStatus.java
deleted file mode 100644
index 31a0963..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/framework/DetectionOnboardTaskStatus.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.framework;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
-
-public class DetectionOnboardTaskStatus {
- private String taskName = "Unknown Task Name";
- private TaskConstants.TaskStatus taskStatus = TaskConstants.TaskStatus.WAITING;
- private String message = "";
-
- public DetectionOnboardTaskStatus() { }
-
- public DetectionOnboardTaskStatus(String taskName) {
- this.setTaskName(taskName);
- }
-
- public DetectionOnboardTaskStatus(String taskName, TaskConstants.TaskStatus taskStatus, String message) {
- this.setTaskName(taskName);
- this.setTaskStatus(taskStatus);
- this.setMessage(message);
- }
-
- public String getTaskName() {
- return taskName;
- }
-
- public void setTaskName(String taskName) {
- Preconditions.checkArgument(!Strings.isNullOrEmpty(taskName));
- this.taskName = taskName;
- }
-
- public TaskConstants.TaskStatus getTaskStatus() {
- return taskStatus;
- }
-
- public void setTaskStatus(TaskConstants.TaskStatus taskStatus) {
- Preconditions.checkNotNull(taskStatus);
- this.taskStatus = taskStatus;
- }
-
- public String getMessage() {
- return message;
- }
-
- public void setMessage(String message) {
- Preconditions.checkNotNull(message);
- this.message = message.trim();
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/AlertFilterAutoTuneOnboardingTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/AlertFilterAutoTuneOnboardingTask.java
deleted file mode 100644
index ad3290b..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/AlertFilterAutoTuneOnboardingTask.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.pinot.thirdeye.anomaly.detection.DetectionJobScheduler;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
-import org.apache.pinot.thirdeye.dashboard.resources.DetectionJobResource;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import java.io.IOException;
-import java.util.List;
-import javax.ws.rs.core.Response;
-import org.apache.commons.configuration2.Configuration;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * This task takes the output of replay result and tune the alert filter for the given detected anomalies
- */
-public class AlertFilterAutoTuneOnboardingTask extends BaseDetectionOnboardTask {
- private static final Logger LOG = LoggerFactory.getLogger(AlertFilterAutoTuneOnboardingTask.class);
- private final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- public static final String TASK_NAME = "AlertFilterAutotune";
-
- public static final String ALERT_FILTER_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY;
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY;
- public static final String ANOMALY_FUNCTION_CONFIG = DefaultDetectionOnboardJob.ANOMALY_FUNCTION_CONFIG;
- public static final String BACKFILL_PERIOD = DefaultDetectionOnboardJob.PERIOD;
- public static final String BACKFILL_START = DefaultDetectionOnboardJob.START;
- public static final String BACKFILL_END = DefaultDetectionOnboardJob.END;
- public static final String AUTOTUNE_PATTERN = DefaultDetectionOnboardJob.AUTOTUNE_PATTERN;
- public static final String AUTOTUNE_TYPE = DefaultDetectionOnboardJob.AUTOTUNE_TYPE;
- public static final String AUTOTUNE_PATTERN_ONLY = DefaultDetectionOnboardJob.AUTOTUNE_PATTERN_ONLY;
- public static final String AUTOTUNE_FEATURES = DefaultDetectionOnboardJob.AUTOTUNE_FEATURES;
- public static final String AUTOTUNE_MTTD = DefaultDetectionOnboardJob.AUTOTUNE_MTTD;
- public static final String HOLIDAY_STARTS = DefaultDetectionOnboardJob.HOLIDAY_STARTS;
- public static final String HOLIDAY_ENDS = DefaultDetectionOnboardJob.HOLIDAY_ENDS;
-
- public static final String DEFAULT_AUTOTUNE_PATTERN = "UP,DOWN";
- public static final String DEFAULT_AUTOTUNE_TYPE = "AUTOTUNE";
-
- public static final String DEFAULT_BACKFILL_PERIOD = FunctionReplayOnboardingTask.DEFAULT_BACKFILL_PERIOD;
-
- public AlertFilterAutoTuneOnboardingTask() {
- super(TASK_NAME);
- }
-
- /**
- * Executes the task. To fail this task, throw exceptions. The job executor will catch the exception and store
- * it in the message in the execution status of this task.
- */
- @Override
- public void run() {
- Configuration taskConfiguration = taskContext.getConfiguration();
- DetectionOnboardExecutionContext executionContext = taskContext.getExecutionContext();
-
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_FACTORY));
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_AUTOTUNE_FACTORY));
-
- AlertFilterFactory alertFilterFactory =
- (AlertFilterFactory) executionContext.getExecutionResult(ALERT_FILTER_FACTORY);
- AlertFilterAutotuneFactory alertFilterAutotuneFactory =
- (AlertFilterAutotuneFactory) executionContext.getExecutionResult(ALERT_FILTER_AUTOTUNE_FACTORY);
-
- Preconditions.checkNotNull(alertFilterFactory);
- Preconditions.checkNotNull(alertFilterAutotuneFactory);
-
- DetectionJobResource detectionJobResource =
- new DetectionJobResource(new DetectionJobScheduler(), alertFilterFactory, alertFilterAutotuneFactory);
-
- AnomalyFunctionDTO anomalyFunctionSpec =
- (AnomalyFunctionDTO) executionContext.getExecutionResult(ANOMALY_FUNCTION_CONFIG);
- long functionId = anomalyFunctionSpec.getId();
- DateTime start = ((DateTime) executionContext.getExecutionResult(BACKFILL_START)).minusDays(1);
- DateTime end = ((DateTime) executionContext.getExecutionResult(BACKFILL_END)).plusDays(1);
-
- Response autotuneResponse = detectionJobResource.
- tuneAlertFilter(Long.toString(functionId), start.toString(), end.toString(),
- taskConfiguration.getString(AUTOTUNE_TYPE, DEFAULT_AUTOTUNE_TYPE),
- taskConfiguration.getString(HOLIDAY_STARTS, ""), taskConfiguration.getString(HOLIDAY_ENDS, ""),
- taskConfiguration.getString(AUTOTUNE_FEATURES), taskConfiguration.getString(AUTOTUNE_MTTD),
- taskConfiguration.getString(AUTOTUNE_PATTERN, DEFAULT_AUTOTUNE_PATTERN),
- taskConfiguration.getString(AUTOTUNE_PATTERN_ONLY,
- Boolean.toString(Strings.isNullOrEmpty(taskConfiguration.getString(AUTOTUNE_MTTD)))));
-
- if (autotuneResponse.getEntity() != null) {
- List<Long> autotuneIds;
- try {
- autotuneIds = OBJECT_MAPPER.readValue(autotuneResponse.getEntity().toString(), List.class);
- } catch (IOException e) {
- throw new IllegalStateException("Unable to parse autotune response: " + autotuneResponse.getEntity().toString(),
- e);
- }
- for (int i = 0; i < autotuneIds.size(); i++) {
- detectionJobResource.updateAlertFilterToFunctionSpecByAutoTuneId(((Number) autotuneIds.get(i)).longValue());
- }
- LOG.info("Initial alert filter applied");
- } else {
- LOG.info("AutoTune doesn't applied");
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DataPreparationOnboardingTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DataPreparationOnboardingTask.java
deleted file mode 100644
index 1fc7d49..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DataPreparationOnboardingTask.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTaskContext;
-import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
-import org.apache.commons.configuration2.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class DataPreparationOnboardingTask extends BaseDetectionOnboardTask {
- private static final Logger LOG = LoggerFactory.getLogger(DataPreparationOnboardingTask.class);
-
- public static final String TASK_NAME = "DataPreparation";
-
- public static final String FUNCTION_FACTORY_CONFIG_PATH = DefaultDetectionOnboardJob.FUNCTION_FACTORY_CONFIG_PATH;
- public static final String ALERT_FILTER_FACTORY_CONFIG_PATH = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY_CONFIG_PATH;
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH = DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH;
- public static final String FUNCTION_FACTORY = DefaultDetectionOnboardJob.FUNCTION_FACTORY;
- public static final String ALERT_FILTER_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY;
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY;
-
- public DataPreparationOnboardingTask(){
- super(TASK_NAME);
- }
-
- @Override
- public void run(){
- Preconditions.checkNotNull(getTaskContext());
-
- DetectionOnboardTaskContext taskContext = getTaskContext();
- DetectionOnboardExecutionContext executionContext = taskContext.getExecutionContext();
- Configuration configuration = taskContext.getConfiguration();
-
- Preconditions.checkNotNull(executionContext);
- Preconditions.checkNotNull(configuration);
- Preconditions.checkNotNull(configuration.getString(FUNCTION_FACTORY_CONFIG_PATH));
- Preconditions.checkNotNull(configuration.getString(ALERT_FILTER_FACTORY_CONFIG_PATH));
- Preconditions.checkNotNull(configuration.getString(ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH));
-
- AnomalyFunctionFactory anomalyFunctionFactory =
- new AnomalyFunctionFactory(configuration.getString(FUNCTION_FACTORY_CONFIG_PATH));
- AlertFilterFactory alertFilterFactory =
- new AlertFilterFactory(configuration.getString(ALERT_FILTER_FACTORY_CONFIG_PATH));
- AlertFilterAutotuneFactory alertFilterAutotuneFactory =
- new AlertFilterAutotuneFactory(configuration.getString(ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH));
-
- Preconditions.checkNotNull(anomalyFunctionFactory);
- Preconditions.checkNotNull(alertFilterFactory);
- Preconditions.checkNotNull(alertFilterAutotuneFactory);
-
- executionContext.setExecutionResult(FUNCTION_FACTORY, anomalyFunctionFactory);
- executionContext.setExecutionResult(ALERT_FILTER_FACTORY, alertFilterFactory);
- executionContext.setExecutionResult(ALERT_FILTER_AUTOTUNE_FACTORY, alertFilterAutotuneFactory);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DefaultDetectionOnboardJob.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DefaultDetectionOnboardJob.java
deleted file mode 100644
index 9559a27..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/DefaultDetectionOnboardJob.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnBoardJobRunner;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.utils.PropertyCheckUtils;
-import org.apache.pinot.thirdeye.dashboard.resources.DetectionJobResource;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.MapConfiguration;
-
-
-/**
- * This job is for self-serve onboarding. During self-serve, a list of tasks needs to be done. Using a wrapper may block
- * users on browser, and it is hard for them to check onboarding status. This job is sent to back-end scheduler and enable
- * the status-check functionality.
- */
-public class DefaultDetectionOnboardJob extends BaseDetectionOnboardJob {
- public static final String ABORT_ON_FAILURE = DetectionOnBoardJobRunner.ABORT_ON_FAILURE;
-
- public static final String FUNCTION_FACTORY_CONFIG_PATH = "functionFactoryConfigPath";
- public static final String ALERT_FILTER_FACTORY_CONFIG_PATH = "alertFilterFactoryConfigPath";
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH = "alertFilterAutotuneFactoryConfigPath";
- public static final String FUNCTION_FACTORY = "functionFactory";
- public static final String ALERT_FILTER_FACTORY = "alertFilterFactory";
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY = "alertFilterAutotuneFactory";
- public static final String NOTIFY_IF_FAILS = "notifyIfFails";
- public static final String FUNCTION_NAME = "functionName";
- public static final String COLLECTION_NAME = "collection";
- public static final String METRIC_NAME = "metric";
- public static final String EXPLORE_DIMENSION = "exploreDimensions";
- public static final String FILTERS = "filters";
- public static final String METRIC_FUNCTION = "metricFunction";
- public static final String FUNCTION_TYPE = "functionType";
- public static final String WINDOW_SIZE = "windowSize";
- public static final String WINDOW_UNIT = "windowUnit";
- public static final String WINDOW_DELAY = "windowDelay";
- public static final String WINDOW_DELAY_UNIT = "windowDelayUnit";
- public static final String DATA_GRANULARITY = "dataGranularity";
- public static final String FUNCTION_PROPERTIES = "properties";
- public static final String FUNCTION_IS_ACTIVE = "isActive";
- public static final String CRON_EXPRESSION = "cron";
- public static final String REQUIRE_DATA_COMPLETENESS = "requireDataCompleteness";
- public static final String ALERT_ID = "alertId";
- public static final String ALERT_NAME = "alertName";
- public static final String ALERT_CRON = "alertCron";
- public static final String ALERT_FROM = "alertSender";
- public static final String ALERT_TO = "alertRecipients";
- public static final String ALERT_CC = "ccRecipients";
- public static final String ALERT_BCC = "bccRecipients";
- public static final String ALERT_APPLICATION = "application";
- public static final String ANOMALY_FUNCTION_CONFIG = "anomalyFuncitonConfig";
- public static final String ALERT_CONFIG = "alertConfig";
- public static final String AUTOTUNE_PATTERN = DetectionJobResource.AUTOTUNE_PATTERN_KEY;
- public static final String AUTOTUNE_TYPE = "autoTuneType";
- public static final String AUTOTUNE_PATTERN_ONLY = DetectionJobResource.AUTOTUNE_PATTERN_ONLY;
- public static final String AUTOTUNE_FEATURES = DetectionJobResource.AUTOTUNE_FEATURE_KEY;
- public static final String AUTOTUNE_MTTD = DetectionJobResource.AUTOTUNE_MTTD_KEY;
- public static final String HOLIDAY_STARTS = "holidayStarts";
- public static final String HOLIDAY_ENDS = "holidayEnds";
- public static final String REMOVE_ANOMALY_IN_WINDOW = "removeAnomaliesInWindow";
- public static final String PERIOD = "period";
- public static final String START = "start";
- public static final String END = "end";
- public static final String FORCE = "force";
- public static final String SPEEDUP = "speedup";
- public static final String SMTP_HOST = "smtpHost";
- public static final String SMTP_PORT = "smtpPort";
- public static final String THIRDEYE_DASHBOARD_HOST = "thirdeyeDashboardHost";
- public static final String DEFAULT_ALERT_SENDER_ADDRESS = "alertSender";
- public static final String DEFAULT_ALERT_RECEIVER_ADDRESS = "alertReceiver";
- public static final String PHANTON_JS_PATH = "phantonJsPath";
- public static final String ROOT_DIR = "rootDir";
-
- protected AnomalyFunctionFactory anomalyFunctionFactory;
- protected AlertFilterFactory alertFilterFactory;
-
- public static final String MISSING_PARAMETER_ERROR_MESSAGE_TEMPLATE = "Require parameter field: %s";
- public static final Boolean DEFAULT_NOTIFY_IF_FAILS = Boolean.TRUE;
-
- public DefaultDetectionOnboardJob(String jobName, Map<String, String> properties) {
- super(jobName, properties);
- }
-
- /**
- * Return a task configuration with task name as its prefix, e.g. task1.property1
- * Note that, if a property is used by multiple tasks, the property key is reused for each task,
- * e.g. task1.property1, task2.property2, and so on
- * @return a task configuration with task name as the property key prefix
- */
- @Override
- public Configuration getTaskConfiguration() {
- PropertyCheckUtils.checkNotNull(this.properties,
- Arrays.asList(FUNCTION_FACTORY_CONFIG_PATH,
- ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH,
- ALERT_FILTER_FACTORY_CONFIG_PATH,
- FUNCTION_NAME,
- COLLECTION_NAME,
- METRIC_NAME,
- SMTP_HOST,
- SMTP_PORT,
- DEFAULT_ALERT_RECEIVER_ADDRESS,
- DEFAULT_ALERT_SENDER_ADDRESS,
- THIRDEYE_DASHBOARD_HOST,
- PHANTON_JS_PATH,
- ROOT_DIR));
- Preconditions.checkArgument(properties.containsKey(ALERT_ID) || properties.containsKey(ALERT_NAME),
- String.format(MISSING_PARAMETER_ERROR_MESSAGE_TEMPLATE, ALERT_ID + " OR " + ALERT_NAME));
- if (!properties.containsKey(ALERT_ID)) {
- Preconditions.checkNotNull(properties.get(ALERT_TO),
- String.format(MISSING_PARAMETER_ERROR_MESSAGE_TEMPLATE, ALERT_TO));
- }
-
- Map<String, String> taskConfigs = new HashMap<>();
- taskConfigs.put (SMTP_HOST, this.properties.get(SMTP_HOST));
- taskConfigs.put (SMTP_PORT, this.properties.get(SMTP_PORT));
- taskConfigs.put (DEFAULT_ALERT_SENDER_ADDRESS, this.properties.get(DEFAULT_ALERT_SENDER_ADDRESS));
- taskConfigs.put (DEFAULT_ALERT_RECEIVER_ADDRESS, this.properties.get(DEFAULT_ALERT_RECEIVER_ADDRESS));
-
- String taskPrefix = DataPreparationOnboardingTask.TASK_NAME + ".";
- taskConfigs.put(taskPrefix + ABORT_ON_FAILURE, Boolean.TRUE.toString());
- taskConfigs.put(taskPrefix + FUNCTION_FACTORY_CONFIG_PATH, this.properties.get(FUNCTION_FACTORY_CONFIG_PATH));
- taskConfigs.put(taskPrefix + ALERT_FILTER_FACTORY_CONFIG_PATH, this.properties.get(ALERT_FILTER_FACTORY_CONFIG_PATH));
- taskConfigs.put(taskPrefix + ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH, this.properties.get(
- ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH));
-
- taskPrefix = FunctionCreationOnboardingTask.TASK_NAME + ".";
- taskConfigs.put(taskPrefix + ABORT_ON_FAILURE, Boolean.TRUE.toString());
- taskConfigs.put(taskPrefix + FUNCTION_NAME, this.properties.get(FUNCTION_NAME));
- taskConfigs.put(taskPrefix + COLLECTION_NAME, this.properties.get(COLLECTION_NAME));
- taskConfigs.put(taskPrefix + METRIC_NAME, this.properties.get(METRIC_NAME));
- if (this.properties.containsKey(EXPLORE_DIMENSION)) {
- taskConfigs.put(taskPrefix + EXPLORE_DIMENSION, this.properties.get(EXPLORE_DIMENSION));
- }
- if (this.properties.containsKey(FILTERS)) {
- taskConfigs.put(taskPrefix + FILTERS, this.properties.get(FILTERS));
- }
- if (this.properties.containsKey(METRIC_FUNCTION)) {
- taskConfigs.put(taskPrefix + METRIC_FUNCTION, this.properties.get(METRIC_FUNCTION));
- }
- if (this.properties.containsKey(WINDOW_SIZE)) {
- taskConfigs.put(taskPrefix + WINDOW_SIZE, this.properties.get(WINDOW_SIZE));
- }
- if (this.properties.containsKey(WINDOW_UNIT)) {
- taskConfigs.put(taskPrefix + WINDOW_UNIT, this.properties.get(WINDOW_UNIT));
- }
- if (this.properties.containsKey(WINDOW_DELAY)) {
- taskConfigs.put(taskPrefix + WINDOW_DELAY, this.properties.get(WINDOW_DELAY));
- }
- if (this.properties.containsKey(WINDOW_DELAY_UNIT)) {
- taskConfigs.put(taskPrefix + WINDOW_DELAY_UNIT, this.properties.get(WINDOW_DELAY_UNIT));
- }
- if (this.properties.containsKey(DATA_GRANULARITY)) {
- taskConfigs.put(taskPrefix + DATA_GRANULARITY, this.properties.get(DATA_GRANULARITY));
- }
- if (this.properties.containsKey(FUNCTION_PROPERTIES)) {
- taskConfigs.put(taskPrefix + FUNCTION_PROPERTIES, this.properties.get(FUNCTION_PROPERTIES));
- }
- if (this.properties.containsKey(FUNCTION_IS_ACTIVE)) {
- taskConfigs.put(taskPrefix + FUNCTION_IS_ACTIVE, this.properties.get(FUNCTION_IS_ACTIVE));
- }
- if (this.properties.containsKey(AUTOTUNE_PATTERN)) {
- taskConfigs.put(taskPrefix + AUTOTUNE_PATTERN, this.properties.get(AUTOTUNE_PATTERN));
- }
- if (this.properties.containsKey(AUTOTUNE_TYPE)) {
- taskConfigs.put(taskPrefix + AUTOTUNE_TYPE, this.properties.get(AUTOTUNE_TYPE));
- }
- if (this.properties.containsKey(AUTOTUNE_FEATURES)) {
- taskConfigs.put(taskPrefix + AUTOTUNE_FEATURES, this.properties.get(AUTOTUNE_FEATURES));
- }
- if (this.properties.containsKey(AUTOTUNE_MTTD)) {
- taskConfigs.put(taskPrefix + AUTOTUNE_MTTD, this.properties.get(AUTOTUNE_MTTD));
- }
- taskConfigs.put(taskPrefix + CRON_EXPRESSION, this.properties.get(CRON_EXPRESSION));
- if (this.properties.containsKey(ALERT_ID)) {
- taskConfigs.put(taskPrefix + ALERT_ID, this.properties.get(ALERT_ID));
- }
- if (this.properties.containsKey(ALERT_NAME)) {
- taskConfigs.put(taskPrefix + ALERT_NAME, this.properties.get(ALERT_NAME));
- }
- if (this.properties.containsKey(ALERT_CRON)) {
- taskConfigs.put(taskPrefix + ALERT_CRON, this.properties.get(ALERT_CRON));
- }
- if (this.properties.containsKey(ALERT_FROM)) {
- taskConfigs.put(taskPrefix + ALERT_FROM, this.properties.get(ALERT_FROM));
- }
- if (this.properties.containsKey(ALERT_TO)) {
- taskConfigs.put(taskPrefix + ALERT_TO, this.properties.get(ALERT_TO));
- }
- if (this.properties.containsKey(ALERT_CC)) {
- taskConfigs.put(taskPrefix + ALERT_CC, this.properties.get(ALERT_CC));
- }
- if (this.properties.containsKey(ALERT_BCC)) {
- taskConfigs.put(taskPrefix + ALERT_BCC, this.properties.get(ALERT_BCC));
- }
- if (this.properties.containsKey(ALERT_APPLICATION)) {
- taskConfigs.put(taskPrefix + ALERT_APPLICATION, this.properties.get(ALERT_APPLICATION));
- }
- if (this.properties.containsKey(DEFAULT_ALERT_RECEIVER_ADDRESS)) {
- taskConfigs.put (taskPrefix + DEFAULT_ALERT_RECEIVER_ADDRESS, this.properties.get(DEFAULT_ALERT_RECEIVER_ADDRESS));
- }
-
- taskPrefix = FunctionReplayOnboardingTask.TASK_NAME + ".";
- taskConfigs.put(taskPrefix + ABORT_ON_FAILURE, Boolean.FALSE.toString());
- if (this.properties.containsKey(PERIOD)) {
- taskConfigs.put (taskPrefix + PERIOD, this.properties.get(PERIOD));
- }
- if (this.properties.containsKey(START)) {
- taskConfigs.put (taskPrefix + START, this.properties.get(START));
- }
- if (this.properties.containsKey(END)) {
- taskConfigs.put (taskPrefix + END, this.properties.get(END));
- }
- if (this.properties.containsKey(FORCE)) {
- taskConfigs.put (taskPrefix + FORCE, this.properties.get(FORCE));
- }
- if (this.properties.containsKey(SPEEDUP)) {
- taskConfigs.put (taskPrefix + SPEEDUP, this.properties.get(SPEEDUP));
- }
- if (this.properties.containsKey(REMOVE_ANOMALY_IN_WINDOW)) {
- taskConfigs.put (taskPrefix + REMOVE_ANOMALY_IN_WINDOW, this.properties.get(REMOVE_ANOMALY_IN_WINDOW));
- }
-
- taskPrefix = AlertFilterAutoTuneOnboardingTask.TASK_NAME + ".";
- taskConfigs.put(taskPrefix + ABORT_ON_FAILURE, Boolean.FALSE.toString());
- if (this.properties.containsKey(PERIOD)) {
- taskConfigs.put (taskPrefix + PERIOD, this.properties.get(PERIOD));
- }
- if (this.properties.containsKey(START)) {
- taskConfigs.put (taskPrefix + START, this.properties.get(START));
- }
- if (this.properties.containsKey(END)) {
- taskConfigs.put (taskPrefix + END, this.properties.get(END));
- }
- if (this.properties.containsKey(AUTOTUNE_PATTERN)) {
- taskConfigs.put (taskPrefix + AUTOTUNE_PATTERN, this.properties.get(AUTOTUNE_PATTERN));
- }
- if (this.properties.containsKey(AUTOTUNE_TYPE)) {
- taskConfigs.put (taskPrefix + AUTOTUNE_TYPE, this.properties.get(AUTOTUNE_TYPE));
- }
- if (this.properties.containsKey(AUTOTUNE_FEATURES)) {
- taskConfigs.put (taskPrefix + AUTOTUNE_FEATURES, this.properties.get(AUTOTUNE_FEATURES));
- }
- if (this.properties.containsKey(AUTOTUNE_MTTD)) {
- taskConfigs.put (taskPrefix + AUTOTUNE_MTTD, this.properties.get(AUTOTUNE_MTTD));
- }
- if (this.properties.containsKey(HOLIDAY_STARTS)) {
- taskConfigs.put (taskPrefix + HOLIDAY_STARTS, this.properties.get(HOLIDAY_STARTS));
- }
- if (this.properties.containsKey(HOLIDAY_ENDS)) {
- taskConfigs.put (taskPrefix + HOLIDAY_ENDS, this.properties.get(HOLIDAY_ENDS));
- }
-
- taskPrefix = NotificationOnboardingTask.TASK_NAME + ".";
- taskConfigs.put(taskPrefix + ABORT_ON_FAILURE, Boolean.TRUE.toString());
- if (this.properties.containsKey(START)) {
- taskConfigs.put (taskPrefix + START, this.properties.get(START));
- }
- if (this.properties.containsKey(END)) {
- taskConfigs.put (taskPrefix + END, this.properties.get(END));
- }
- taskConfigs.put (taskPrefix + SMTP_HOST, this.properties.get(SMTP_HOST));
- taskConfigs.put (taskPrefix + SMTP_PORT, this.properties.get(SMTP_PORT));
- taskConfigs.put (taskPrefix + THIRDEYE_DASHBOARD_HOST, this.properties.get(THIRDEYE_DASHBOARD_HOST));
- taskConfigs.put (taskPrefix + DEFAULT_ALERT_SENDER_ADDRESS, this.properties.get(DEFAULT_ALERT_SENDER_ADDRESS));
- taskConfigs.put (taskPrefix + DEFAULT_ALERT_RECEIVER_ADDRESS, this.properties.get(DEFAULT_ALERT_RECEIVER_ADDRESS));
- taskConfigs.put (taskPrefix + PHANTON_JS_PATH, this.properties.get(PHANTON_JS_PATH));
- taskConfigs.put (taskPrefix + ROOT_DIR, this.properties.get(ROOT_DIR));
-
- // By default list splitting is disabled which means the comma separated values will be retained.
- MapConfiguration mapConfiguration = new MapConfiguration(taskConfigs);
- return mapConfiguration;
- }
-
- /**
- * Return a list of tasks will be run in the job
- * @return a list of tasks
- */
- @Override
- public List<DetectionOnboardTask> getTasks() {
- List<DetectionOnboardTask> detectionOnboardTasks = new ArrayList<>();
- detectionOnboardTasks.add(new DataPreparationOnboardingTask());
- detectionOnboardTasks.add(new FunctionCreationOnboardingTask());
- detectionOnboardTasks.add(new FunctionReplayOnboardingTask());
- detectionOnboardTasks.add(new AlertFilterAutoTuneOnboardingTask());
- detectionOnboardTasks.add(new NotificationOnboardingTask());
- return detectionOnboardTasks;
- }
-
- public static String getAbortOnFailure() {
- return ABORT_ON_FAILURE;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionCreationOnboardingTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionCreationOnboardingTask.java
deleted file mode 100644
index c849ab3..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionCreationOnboardingTask.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTaskContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.utils.FunctionCreationUtils;
-import org.apache.pinot.thirdeye.anomaly.utils.EmailUtils;
-import org.apache.pinot.thirdeye.common.time.TimeGranularity;
-import org.apache.pinot.thirdeye.common.time.TimeSpec;
-import org.apache.pinot.thirdeye.constant.MetricAggFunction;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean.EmailConfig;
-import org.apache.pinot.thirdeye.datalayer.util.ThirdEyeStringUtils;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
-import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
-import java.net.URLDecoder;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * This task runs the function creation and assign function id to an existing or new alert config
- * Three steps are included:
- * - create a new anomaly function
- * - assign the function id to an existing or new alert config
- */
-public class FunctionCreationOnboardingTask extends BaseDetectionOnboardTask {
- private static final Logger LOG = LoggerFactory.getLogger(FunctionCreationOnboardingTask.class);
-
- public static final String TASK_NAME = "FunctionAlertCreation";
-
- public static final String ANOMALY_FUNCTION_CONFIG = DefaultDetectionOnboardJob.ANOMALY_FUNCTION_CONFIG;
- public static final String ALERT_CONFIG = DefaultDetectionOnboardJob.ALERT_CONFIG;
- public static final String FUNCTION_FACTORY = DefaultDetectionOnboardJob.FUNCTION_FACTORY;
- public static final String ALERT_FILTER_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY;
- public static final String FUNCTION_NAME = DefaultDetectionOnboardJob.FUNCTION_NAME;
- public static final String COLLECTION_NAME = DefaultDetectionOnboardJob.COLLECTION_NAME;
- public static final String METRIC_NAME = DefaultDetectionOnboardJob.METRIC_NAME;
- public static final String EXPLORE_DIMENSION = DefaultDetectionOnboardJob.EXPLORE_DIMENSION;
- public static final String FILTERS = DefaultDetectionOnboardJob.FILTERS;
- public static final String FUNCTION_TYPE = DefaultDetectionOnboardJob.FUNCTION_TYPE;
- public static final String METRIC_FUNCTION = DefaultDetectionOnboardJob.METRIC_FUNCTION;
- public static final String WINDOW_SIZE = DefaultDetectionOnboardJob.WINDOW_SIZE;
- public static final String WINDOW_UNIT = DefaultDetectionOnboardJob.WINDOW_UNIT;
- public static final String WINDOW_DELAY = DefaultDetectionOnboardJob.WINDOW_DELAY;
- public static final String WINDOW_DELAY_UNIT = DefaultDetectionOnboardJob.WINDOW_DELAY_UNIT;
- public static final String DATA_GRANULARITY = DefaultDetectionOnboardJob.DATA_GRANULARITY;
- public static final String PROPERTIES = DefaultDetectionOnboardJob.FUNCTION_PROPERTIES;
- public static final String IS_ACTIVE = DefaultDetectionOnboardJob.FUNCTION_IS_ACTIVE;
- public static final String CRON_EXPRESSION = DefaultDetectionOnboardJob.CRON_EXPRESSION;
- public static final String REQUIRE_DATA_COMPLETENESS = DefaultDetectionOnboardJob.REQUIRE_DATA_COMPLETENESS;
- public static final String ALERT_FILTER_PATTERN = DefaultDetectionOnboardJob.AUTOTUNE_PATTERN;
- public static final String ALERT_FILTER_TYPE = DefaultDetectionOnboardJob.AUTOTUNE_TYPE;
- public static final String ALERT_FILTER_FEATURES = DefaultDetectionOnboardJob.AUTOTUNE_FEATURES;
- public static final String ALERT_FILTER_MTTD = DefaultDetectionOnboardJob.AUTOTUNE_MTTD;
- public static final String ALERT_ID = DefaultDetectionOnboardJob.ALERT_ID;
- public static final String ALERT_NAME = DefaultDetectionOnboardJob.ALERT_NAME;
- public static final String ALERT_CRON = DefaultDetectionOnboardJob.ALERT_CRON;
- public static final String ALERT_FROM = DefaultDetectionOnboardJob.ALERT_FROM;
- public static final String ALERT_TO = DefaultDetectionOnboardJob.ALERT_TO;
- public static final String ALERT_CC = DefaultDetectionOnboardJob.ALERT_CC;
- public static final String ALERT_BCC = DefaultDetectionOnboardJob.ALERT_BCC;
- public static final String ALERT_APPLICATION = DefaultDetectionOnboardJob.ALERT_APPLICATION;
- public static final String DEFAULT_ALERT_SENDER = DefaultDetectionOnboardJob.DEFAULT_ALERT_SENDER_ADDRESS;
- public static final String DEFAULT_ALERT_RECEIVER = DefaultDetectionOnboardJob.DEFAULT_ALERT_RECEIVER_ADDRESS;
-
- public static final Boolean DEFAULT_IS_ACTIVE = true;
- public static final Integer DEFAULT_WINDOW_DELAY = 0;
- public static final String DEFAULT_ALERT_CRON = "0 0/5 * * * ? *"; // Every 5 min
- public static final String DEFAULT_ALERT_FILTER_PATTERN = AlertFilterAutoTuneOnboardingTask.DEFAULT_AUTOTUNE_PATTERN;
- public static final String DEFAULT_ALERT_FILTER_TYPE = "AUTOTUNE";
- public static final String DEFAULT_URL_DECODER = "UTF-8";
-
- private AnomalyFunctionManager anomalyFunctionDAO;
- private AlertConfigManager alertConfigDAO;
- private DatasetConfigManager datasetConfigDAO;
- private MetricConfigManager metricConfigDAO;
-
- public FunctionCreationOnboardingTask() {
- super(TASK_NAME);
- DAORegistry daoRegistry = DAORegistry.getInstance();
- this.alertConfigDAO = daoRegistry.getAlertConfigDAO();
- this.anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
- this.datasetConfigDAO = daoRegistry.getDatasetConfigDAO();
- this.metricConfigDAO = daoRegistry.getMetricConfigDAO();
- }
-
- /**
- * Executes the task. To fail this task, throw exceptions. The job executor will catch the exception and store
- * it in the message in the execution status of this task.
- */
- @Override
- public void run() {
- DetectionOnboardTaskContext taskContext = getTaskContext();
- DetectionOnboardExecutionContext executionContext = taskContext.getExecutionContext();
- Configuration configuration = taskContext.getConfiguration();
-
- Preconditions.checkNotNull(executionContext.getExecutionResult(FUNCTION_FACTORY));
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_FACTORY));
-
- AnomalyFunctionFactory anomalyFunctionFactory = (AnomalyFunctionFactory)
- executionContext.getExecutionResult(FUNCTION_FACTORY);
- AlertFilterFactory alertFilterFactory = (AlertFilterFactory) executionContext.getExecutionResult(ALERT_FILTER_FACTORY);
-
- Preconditions.checkNotNull(anomalyFunctionFactory);
- Preconditions.checkNotNull(alertFilterFactory);
-
- // Assert if null
- Preconditions.checkNotNull(taskContext);
- Preconditions.checkNotNull(configuration.getString(FUNCTION_NAME));
- Preconditions.checkNotNull(configuration.getString(COLLECTION_NAME));
- Preconditions.checkNotNull(configuration.getString(METRIC_NAME));
- Preconditions.checkArgument(configuration.containsKey(ALERT_ID) || configuration.containsKey(ALERT_NAME));
- if (!configuration.containsKey(ALERT_ID)) {
- Preconditions.checkNotNull(configuration.getString(ALERT_TO));
- }
-
- // Get pre-created function name
- // TODO Once pipeline refactor is done, this logic should be changed to be new function creation
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findWhereNameEquals(configuration.getString(FUNCTION_NAME));
- if (anomalyFunction == null) {
- throw new IllegalArgumentException(String.format("No function with name %s is found in the system",
- configuration.getString(FUNCTION_NAME)));
- }
-
- // check if duplicate name exists
- if (StringUtils.isNotBlank(configuration.getString(ALERT_NAME))) {
- AlertConfigDTO duplicateAlert = alertConfigDAO.findWhereNameEquals(configuration.getString(ALERT_NAME));
- if (duplicateAlert != null) {
- throw new IllegalArgumentException("Duplicate alert name " + configuration.getString(ALERT_NAME)
- + " is found");
- }
- }
-
- // update datasetConfig
- DatasetConfigDTO datasetConfig = datasetConfigDAO.findByDataset(configuration.getString(COLLECTION_NAME));
- if (datasetConfig == null) {
- throw new NoSuchElementException("Cannot find collection: " + configuration.getString(COLLECTION_NAME));
- }
- TimeSpec timeSpec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
- TimeGranularity dataGranularity = timeSpec.getDataGranularity();
- if (configuration.containsKey(DATA_GRANULARITY)) {
- TimeGranularity userAssignedDataGranularity = null;
- try {
- userAssignedDataGranularity = TimeGranularity.fromString(configuration.getString(DATA_GRANULARITY));
- } catch (Exception e) {
- LOG.error("Unable to parse user input data granularity: {}",
- configuration.getString(DATA_GRANULARITY));
- throw new IllegalArgumentException("Unsupported time granularity: " + configuration.getString(DATA_GRANULARITY));
- }
- dataGranularity = userAssignedDataGranularity;
- }
-
- // use the aggregate function in MetricConfig as default function
- MetricConfigDTO metricConfig = metricConfigDAO.findByMetricAndDataset(configuration.getString(METRIC_NAME), configuration.getString(COLLECTION_NAME));
- String defaultMetricFunction = metricConfig.getDefaultAggFunction().name();
-
- // create function
- try {
- AnomalyFunctionDTO defaultFunctionSpec = getDefaultFunctionSpecByTimeGranularity(dataGranularity);
-
- // Merge user properties with default properties; the user assigned property can override default property
- Properties userAssignedFunctionProperties = ThirdEyeStringUtils
- .decodeCompactedProperties(configuration.getString(PROPERTIES, ""));
- Properties defaultFunctionProperties = ThirdEyeStringUtils
- .decodeCompactedProperties(defaultFunctionSpec.getProperties());
- for (Map.Entry propertyEntry : userAssignedFunctionProperties.entrySet()) {
- defaultFunctionProperties.setProperty((String) propertyEntry.getKey(), (String) propertyEntry.getValue());
- }
-
- anomalyFunction.setMetricId(metricConfig.getId());
- anomalyFunction.setMetric(metricConfig.getName());
- anomalyFunction.setTopicMetric(metricConfig.getName());
- anomalyFunction.setMetrics(Arrays.asList(metricConfig.getName()));
- anomalyFunction.setCollection(datasetConfig.getDataset());
- anomalyFunction.setCron(configuration.getString(CRON_EXPRESSION, defaultFunctionSpec.getCron()));
- anomalyFunction.setMetricFunction(MetricAggFunction.valueOf(
- configuration.getString(METRIC_FUNCTION, defaultMetricFunction)));
- String filters = configuration.getString(FILTERS);
- if (!org.apache.commons.lang3.StringUtils.isBlank(filters)) {
- filters = URLDecoder.decode(filters, DEFAULT_URL_DECODER);
- String filterString = ThirdEyeUtils.getSortedFiltersFromJson(filters);
- anomalyFunction.setFilters(filterString);
- }
- if (org.apache.commons.lang3.StringUtils.isNotEmpty(configuration.getString(EXPLORE_DIMENSION))) {
- anomalyFunction.setExploreDimensions(FunctionCreationUtils.getDimensions(datasetConfig, configuration.getString(EXPLORE_DIMENSION)));
- }
-
- anomalyFunction.setWindowSize(configuration.getInt(WINDOW_SIZE, defaultFunctionSpec.getWindowSize()));
- anomalyFunction.setWindowUnit(TimeUnit.valueOf(configuration
- .getString(WINDOW_UNIT, defaultFunctionSpec.getWindowUnit().name())));
- anomalyFunction.setWindowDelay(configuration.getInt(WINDOW_DELAY, DEFAULT_WINDOW_DELAY));
- anomalyFunction.setWindowDelayUnit(TimeUnit.valueOf(
- configuration.getString(WINDOW_DELAY_UNIT, dataGranularity.getUnit().toString())));
- anomalyFunction.setType(configuration.getString(FUNCTION_TYPE, defaultFunctionSpec.getType()));
- anomalyFunction.setProperties(ThirdEyeStringUtils.
- encodeCompactedProperties(defaultFunctionProperties));
- if (defaultFunctionSpec.getFrequency() != null) {
- anomalyFunction.setFrequency(defaultFunctionSpec.getFrequency());
- }
- anomalyFunction.setBucketSize(dataGranularity.getSize());
- anomalyFunction.setBucketUnit(dataGranularity.getUnit());
- anomalyFunction.setIsActive(configuration.getBoolean(IS_ACTIVE, DEFAULT_IS_ACTIVE));
- anomalyFunction.setRequiresCompletenessCheck(configuration.getBoolean(REQUIRE_DATA_COMPLETENESS,
- defaultFunctionSpec.isRequiresCompletenessCheck()));
-
- anomalyFunctionDAO.update(anomalyFunction);
-
- executionContext.setExecutionResult(ANOMALY_FUNCTION_CONFIG, anomalyFunction);
- } catch (Exception e) {
- throw new IllegalArgumentException(e);
- }
-
- // Assign Default Alert Filter
- Map<String, String> alertFilter = new HashMap<>();
- alertFilter.put(ALERT_FILTER_PATTERN, configuration.getString(ALERT_FILTER_PATTERN, DEFAULT_ALERT_FILTER_PATTERN));
- alertFilter.put(ALERT_FILTER_TYPE, configuration.getString(ALERT_FILTER_TYPE, DEFAULT_ALERT_FILTER_TYPE));
- if (configuration.containsKey(ALERT_FILTER_FEATURES)) {
- alertFilter.put(ALERT_FILTER_FEATURES, configuration.getString(ALERT_FILTER_FEATURES));
- }
- if (configuration.containsKey(ALERT_FILTER_MTTD)) {
- alertFilter.put(ALERT_FILTER_MTTD, configuration.getString(ALERT_FILTER_MTTD));
- }
- anomalyFunction.setAlertFilter(alertFilter);
- this.anomalyFunctionDAO.update(anomalyFunction);
-
- // create alert config
- AlertConfigDTO alertConfig = null;
- if (configuration.containsKey(ALERT_ID)) {
- alertConfig = alertConfigDAO.findById(configuration.getLong(ALERT_ID));
- EmailConfig emailConfig = alertConfig.getEmailConfig();
- if (emailConfig == null) {
- EmailConfig emailConf = new EmailConfig();
- emailConf.setFunctionIds(Arrays.asList(anomalyFunction.getId()));
- } else {
- emailConfig.getFunctionIds().add(anomalyFunction.getId());
- }
- alertConfig.setEmailConfig(emailConfig);
-
- // Add recipients to existing alert group
- DetectionAlertFilterRecipients recipients = alertConfig.getReceiverAddresses();
- Set<String> toAddr = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_TO));
- if (recipients.getTo() == null) {
- recipients.setTo(toAddr);
- } else {
- recipients.getTo().addAll(toAddr);
- }
- Set<String> ccAddr = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_CC));
- if (recipients.getCc() == null) {
- recipients.setCc(ccAddr);
- } else {
- recipients.getCc().addAll(ccAddr);
- }
- Set<String> bccAddr = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_BCC));
- if (recipients.getBcc() == null) {
- recipients.setBcc(bccAddr);
- } else {
- recipients.getBcc().addAll(bccAddr);
- }
-
- alertConfigDAO.update(alertConfig);
- } else {
- alertConfig = new AlertConfigDTO();
- EmailConfig emailConfig = new EmailConfig();
- emailConfig.setFunctionIds(Arrays.asList(anomalyFunction.getId()));
- alertConfig.setEmailConfig(emailConfig);
- alertConfig.setName(configuration.getString(ALERT_NAME));
-
- Set<String> toAddresses = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_TO));
- Set<String> ccAddresses = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_CC));
- ccAddresses.addAll(EmailUtils.getValidEmailAddresses(configuration.getString(DEFAULT_ALERT_RECEIVER)));
- Set<String> bccAddresses = EmailUtils.getValidEmailAddresses(configuration.getString(ALERT_BCC));
-
- Set<String> defaultSender = EmailUtils.getValidEmailAddresses(configuration.getString(DEFAULT_ALERT_SENDER));
- if (defaultSender.isEmpty()) {
- throw new IllegalArgumentException("No sender configured for the emails. Please set " + DEFAULT_ALERT_SENDER);
- }
- alertConfig.setFromAddress(configuration.getString(ALERT_FROM, defaultSender.iterator().next()));
-
- alertConfig.setReceiverAddresses(new DetectionAlertFilterRecipients(toAddresses, ccAddresses, bccAddresses));
- alertConfig.setApplication(configuration.getString(ALERT_APPLICATION));
- alertConfig.setCronExpression(configuration.getString(ALERT_CRON, DEFAULT_ALERT_CRON));
- alertConfigDAO.save(alertConfig);
- }
- executionContext.setExecutionResult(ALERT_CONFIG, alertConfig);
- }
-
- protected AnomalyFunctionDTO getDefaultFunctionSpecByTimeGranularity(TimeGranularity timeGranularity) {
- AnomalyFunctionDTO anomalyFunctionSpec = new AnomalyFunctionDTO();
- switch (timeGranularity.getUnit()) {
- case MINUTES:
- anomalyFunctionSpec.setType("SIGN_TEST_WRAPPER");
- anomalyFunctionSpec.setCron("0 0/15 * * * ? *");
- anomalyFunctionSpec.setWindowSize(6);
- anomalyFunctionSpec.setWindowUnit(TimeUnit.HOURS);
- anomalyFunctionSpec.setFrequency(new TimeGranularity(15, TimeUnit.MINUTES));
- anomalyFunctionSpec.setProperties("variables.seasonalPeriod=P7D;module.training=nonparametric.SeasonalSlidingWindowTrainingModule;variables.slidingWindowWidth=8;variables.pattern=UP,DOWN;variables.anomalyRemovalThreshold=0.6,-0.6;module.data=SeasonalDataModule;variables.signTestStepSize=1;variables.pValueThreshold=0.05;function=ConfigurableAnomalyDetectionFunction;module.testingPreprocessors=DummyPreprocessModule;variables.seasonalCount=3;variables.signTestWindowSize=24;module.de [...]
- anomalyFunctionSpec.setRequiresCompletenessCheck(false);
- break;
- case HOURS:
- anomalyFunctionSpec.setType("REGRESSION_GAUSSIAN_SCAN_WRAPPER");
- anomalyFunctionSpec.setCron("0 0 * * * ? *");
- anomalyFunctionSpec.setWindowSize(24);
- anomalyFunctionSpec.setWindowUnit(TimeUnit.HOURS);
- anomalyFunctionSpec.setProperties("variables.isMajor=false;downgrade.variables.seasonalities=;function=SelfRecoverableAnomalyDetectionFunction;variables.pValueThreshold=0.01;variables.continuumOffset=P60D;module.detection=GaussianScanDetectionModule;variables.anomalyRemovalThreshold=1.0,-1.0;workflow=RegressionWorkflow;module.training=parametric.NullBasisRegressionTrainingModule;variables.seasonalities=HOURLY_SEASONALITY,DAILY_SEASONALITY;module.data=ContinuumDataModule;variables [...]
- anomalyFunctionSpec.setRequiresCompletenessCheck(false);
- break;
- case DAYS:
- anomalyFunctionSpec.setType("SPLINE_REGRESSION_WRAPPER");
- anomalyFunctionSpec.setCron("0 0 14 * * ? *");
- anomalyFunctionSpec.setWindowSize(1);
- anomalyFunctionSpec.setWindowUnit(TimeUnit.DAYS);
- anomalyFunctionSpec.setProperties("variables.continuumOffset=P90D;module.training=parametric.GenericSplineTrainingModule;variables.numberOfKnots=0;variables.degree=3;variables.predictionMode=TRENDING;variables.anomalyRemovalThreshold=0.6,-0.6;module.data=ContinuumDataModule;variables.pValueThreshold=0.025;function=SelfRecoverableAnomalyDetectionFunction;variables.seasonalities=DAILY_SEASONALITY;module.detection=ConfidenceIntervalDetectionModule;module.testingPreprocessors=DummyPr [...]
- anomalyFunctionSpec.setRequiresCompletenessCheck(true);
- break;
- default:
- anomalyFunctionSpec.setType("WEEK_OVER_WEEK_RULE");
- anomalyFunctionSpec.setCron("0 0 0 * * ?");
- anomalyFunctionSpec.setWindowSize(6);
- anomalyFunctionSpec.setWindowUnit(TimeUnit.HOURS);
- anomalyFunctionSpec.setProperties("");
- anomalyFunctionSpec.setRequiresCompletenessCheck(false);
- break;
- }
- return anomalyFunctionSpec;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionReplayOnboardingTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionReplayOnboardingTask.java
deleted file mode 100644
index 07ffebd..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/FunctionReplayOnboardingTask.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.anomaly.detection.DetectionJobScheduler;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants.JobStatus;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
-import org.apache.pinot.thirdeye.dashboard.resources.DetectionJobResource;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import java.util.Map;
-import javax.ws.rs.core.Response;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.lang3.exception.ExceptionUtils;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * This task performs replay on anomaly functions
- */
-public class FunctionReplayOnboardingTask extends BaseDetectionOnboardTask {
- private static final Logger LOG = LoggerFactory.getLogger(FunctionReplayOnboardingTask.class);
- public static final String TASK_NAME = "FunctionReplay";
-
- public static final String ANOMALY_FUNCTION_CONFIG = DefaultDetectionOnboardJob.ANOMALY_FUNCTION_CONFIG;
- public static final String ALERT_FILTER_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY;
- public static final String ALERT_FILTER_AUTOTUNE_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY;
- public static final String BACKFILL_PERIOD = DefaultDetectionOnboardJob.PERIOD;
- public static final String BACKFILL_START = DefaultDetectionOnboardJob.START;
- public static final String BACKFILL_END = DefaultDetectionOnboardJob.END;
- public static final String BACKFILL_FORCE = DefaultDetectionOnboardJob.FORCE;
- public static final String BACKFILL_SPEEDUP = DefaultDetectionOnboardJob.SPEEDUP;
- public static final String BACKFILL_REMOVE_ANOMALY_IN_WINDOW = DefaultDetectionOnboardJob.REMOVE_ANOMALY_IN_WINDOW;
-
- public static final String DEFAULT_BACKFILL_PERIOD = "P30D";
- public static final Boolean DEFAULT_BACKFILL_FORCE = true;
- public static final Boolean DEFAULT_BACKFILL_SPEEDUP = false;
- public static final Boolean DEFAULT_BACKFILL_REMOVE_ANOMALY_IN_WINDOW = false;
-
- private DetectionJobScheduler detectionJobScheduler;
-
- public FunctionReplayOnboardingTask() {
- super(TASK_NAME);
- }
-
- /**
- * Executes the task. To fail this task, throw exceptions. The job executor will catch the exception and store
- * it in the message in the execution status of this task.
- */
- @Override
- public void run() {
-
- try {
- Response response = initDetectionJob();
- Map<Long, Long> functionIdToJobIdMap = (Map<Long, Long>) response.getEntity();
- for (long jobId : functionIdToJobIdMap.values()) {
- JobStatus jobStatus = detectionJobScheduler.waitForJobDone(jobId);
- if (JobStatus.FAILED.equals(jobStatus) || JobStatus.TIMEOUT.equals(jobStatus)) {
- throw new InterruptedException("Get Job Status: " + jobStatus);
- }
- }
-
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- public Response initDetectionJob() throws Exception{
- Configuration taskConfiguration = taskContext.getConfiguration();
- DetectionOnboardExecutionContext executionContext = taskContext.getExecutionContext();
-
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_FACTORY));
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_AUTOTUNE_FACTORY));
-
- AlertFilterFactory alertFilterFactory = (AlertFilterFactory) executionContext.getExecutionResult(ALERT_FILTER_FACTORY);
- AlertFilterAutotuneFactory alertFilterAutotuneFactory = (AlertFilterAutotuneFactory)
- executionContext.getExecutionResult(ALERT_FILTER_AUTOTUNE_FACTORY);
-
- Preconditions.checkNotNull(alertFilterFactory);
- Preconditions.checkNotNull(alertFilterAutotuneFactory);
-
- detectionJobScheduler = new DetectionJobScheduler();
- DetectionJobResource detectionJobResource = new DetectionJobResource(detectionJobScheduler,
- alertFilterFactory, alertFilterAutotuneFactory);
- AnomalyFunctionDTO anomalyFunction = (AnomalyFunctionDTO) executionContext.getExecutionResult(ANOMALY_FUNCTION_CONFIG);
- long functionId = anomalyFunction.getId();
- Period backfillPeriod = Period.parse(taskConfiguration.getString(BACKFILL_PERIOD, DEFAULT_BACKFILL_PERIOD));
- DateTime start = DateTime.parse(taskConfiguration.getString(BACKFILL_START, DateTime.now().minus(backfillPeriod).toString()));
- DateTime end = DateTime.parse(taskConfiguration.getString(BACKFILL_END, DateTime.now().toString()));
- executionContext.setExecutionResult(BACKFILL_START, start);
- executionContext.setExecutionResult(BACKFILL_END, end);
-
- Response response = null;
- try {
- LOG.info("Running replay task for {} from {} to {}", anomalyFunction, start, end);
- response = detectionJobResource.generateAnomaliesInRange(functionId, start.toString(), end.toString(),
- Boolean.toString(taskConfiguration.getBoolean(BACKFILL_FORCE, DEFAULT_BACKFILL_FORCE)),
- taskConfiguration.getBoolean(BACKFILL_SPEEDUP, DEFAULT_BACKFILL_SPEEDUP),
- taskConfiguration.getBoolean(BACKFILL_REMOVE_ANOMALY_IN_WINDOW, DEFAULT_BACKFILL_REMOVE_ANOMALY_IN_WINDOW));
- } catch (Exception e){
- throw new IllegalStateException(String.format("Unable to create detection job for %d from %s to %s\n%s",
- functionId, start.toString(), end.toString(), ExceptionUtils.getStackTrace(e)));
- }
- return response;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/NotificationOnboardingTask.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/NotificationOnboardingTask.java
deleted file mode 100644
index 23c82e7..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/NotificationOnboardingTask.java
+++ /dev/null
@@ -1,160 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import com.google.common.base.Preconditions;
-import org.apache.pinot.thirdeye.notification.commons.EmailEntity;
-import org.apache.pinot.thirdeye.notification.formatter.ADContentFormatterContext;
-import org.apache.pinot.thirdeye.notification.formatter.channels.EmailContentFormatter;
-import org.apache.pinot.thirdeye.notification.content.templates.OnboardingNotificationContent;
-import org.apache.pinot.thirdeye.anomaly.SmtpConfiguration;
-import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
-import org.apache.pinot.thirdeye.anomaly.alert.util.AlertFilterHelper;
-import org.apache.pinot.thirdeye.anomaly.alert.util.EmailHelper;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.mail.EmailException;
-import org.joda.time.DateTime;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Send out email notifications
- */
-public class NotificationOnboardingTask extends BaseDetectionOnboardTask {
- private static final Logger LOG = LoggerFactory.getLogger(NotificationOnboardingTask.class);
-
- public static final String TASK_NAME = "Notification";
-
- public static final String ALERT_CONFIG = DefaultDetectionOnboardJob.ALERT_CONFIG;
- public static final String ALERT_FILTER_FACTORY = DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY;
- public static final String ANOMALY_FUNCTION_CONFIG = DefaultDetectionOnboardJob.ANOMALY_FUNCTION_CONFIG;
- public static final String NOTIFICATION_START = DefaultDetectionOnboardJob.START;
- public static final String NOTIFICATION_END = DefaultDetectionOnboardJob.END;
- public static final String SMTP_HOST = DefaultDetectionOnboardJob.SMTP_HOST;
- public static final String SMTP_PORT = DefaultDetectionOnboardJob.SMTP_PORT;
- public static final String THIRDEYE_DASHBOARD_HOST = DefaultDetectionOnboardJob.THIRDEYE_DASHBOARD_HOST;
- public static final String DEFAULT_ALERT_SENDER_ADDRESS = DefaultDetectionOnboardJob.DEFAULT_ALERT_SENDER_ADDRESS;
- public static final String DEFAULT_ALERT_RECEIVER_ADDRESS = DefaultDetectionOnboardJob.DEFAULT_ALERT_RECEIVER_ADDRESS;
- public static final String PHANTON_JS_PATH = DefaultDetectionOnboardJob.PHANTON_JS_PATH;
- public static final String ROOT_DIR = DefaultDetectionOnboardJob.ROOT_DIR;
-
- public NotificationOnboardingTask(){
- super(TASK_NAME);
- }
-
- /**
- * Executes the task. To fail this task, throw exceptions. The job executor will catch the exception and store
- * it in the message in the execution status of this task.
- */
- @Override
- public void run() {
- Configuration taskConfigs = taskContext.getConfiguration();
- DetectionOnboardExecutionContext executionContext = taskContext.getExecutionContext();
-
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_FILTER_FACTORY));
-
- AlertFilterFactory alertFilterFactory = (AlertFilterFactory) executionContext.getExecutionResult(ALERT_FILTER_FACTORY);
-
- Preconditions.checkNotNull(alertFilterFactory);
-
- Preconditions.checkNotNull(executionContext.getExecutionResult(ALERT_CONFIG));
- Preconditions.checkNotNull(executionContext.getExecutionResult(ANOMALY_FUNCTION_CONFIG));
- Preconditions.checkNotNull(executionContext.getExecutionResult(NOTIFICATION_START));
- Preconditions.checkNotNull(executionContext.getExecutionResult(NOTIFICATION_END));
- Preconditions.checkNotNull(taskConfigs.getString(SMTP_HOST));
- Preconditions.checkNotNull(taskConfigs.getString(SMTP_PORT));
- Preconditions.checkNotNull(taskConfigs.getString(DEFAULT_ALERT_RECEIVER_ADDRESS));
- Preconditions.checkNotNull(taskConfigs.getString(DEFAULT_ALERT_SENDER_ADDRESS));
- Preconditions.checkNotNull(taskConfigs.getString(THIRDEYE_DASHBOARD_HOST));
- Preconditions.checkNotNull(taskConfigs.getString(PHANTON_JS_PATH));
- Preconditions.checkNotNull(taskConfigs.getString(ROOT_DIR));
-
- AnomalyFunctionDTO anomalyFunctionSpec = (AnomalyFunctionDTO) executionContext.getExecutionResult(ANOMALY_FUNCTION_CONFIG);
- Long functionId = anomalyFunctionSpec.getId();
- DateTime start = (DateTime) executionContext.getExecutionResult(NOTIFICATION_START);
- DateTime end = (DateTime) executionContext.getExecutionResult(NOTIFICATION_END);
-
- // Get alert config from previous output
- AlertConfigDTO alertConfig = (AlertConfigDTO) executionContext.getExecutionResult(ALERT_CONFIG);
-
- SmtpConfiguration smtpConfiguration = new SmtpConfiguration();
- smtpConfiguration.setSmtpHost(taskConfigs.getString(SMTP_HOST));
- smtpConfiguration.setSmtpPort(taskConfigs.getInt(SMTP_PORT));
-
- MergedAnomalyResultManager mergeAnomalyDAO = DAORegistry.getInstance().getMergedAnomalyResultDAO();
- List<MergedAnomalyResultDTO> anomalyCandidates = mergeAnomalyDAO
- .findByFunctionId(functionId);
- anomalyCandidates = AlertFilterHelper.applyFiltrationRule(anomalyCandidates, alertFilterFactory);
- List<AnomalyResult> filteredAnomalyResults = new ArrayList<>();
- for (MergedAnomalyResultDTO anomaly : anomalyCandidates) {
- filteredAnomalyResults.add(anomaly);
- }
-
- // Email Subject
- String subject = String.format("Replay results for %s is ready for review!",
- DAORegistry.getInstance().getAnomalyFunctionDAO().findById(functionId).getFunctionName());
-
- // construct context
- ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAnomalyFunctionSpec(anomalyFunctionSpec);
- context.setAlertConfig(alertConfig);
- context.setStart(start);
- context.setEnd(end);
-
- // Set up thirdeye config
- ThirdEyeAnomalyConfiguration thirdEyeAnomalyConfig = new ThirdEyeAnomalyConfiguration();
- thirdEyeAnomalyConfig.setDashboardHost(taskConfigs.getString(THIRDEYE_DASHBOARD_HOST));
- thirdEyeAnomalyConfig.setPhantomJsPath(taskConfigs.getString(PHANTON_JS_PATH));
- thirdEyeAnomalyConfig.setRootDir(taskConfigs.getString(ROOT_DIR));
-
- EmailContentFormatter
- emailFormatter = new EmailContentFormatter(new OnboardingNotificationContent(), thirdEyeAnomalyConfig);
- EmailEntity emailEntity = emailFormatter.getEmailEntity(alertConfig.getReceiverAddresses(),
- subject, filteredAnomalyResults, context);
- try {
- EmailHelper.sendEmailWithEmailEntity(emailEntity, smtpConfiguration);
- } catch (EmailException e) {
- LOG.error("Unable to send out email to recipients");
- throw new IllegalStateException("Unable to send out email to recipients", e);
- }
-
- // Set the alert to be active after everything is successful
- alertConfig.setActive(true);
- DAORegistry.getInstance().getAlertConfigDAO().update(alertConfig);
-
- // Update Notified flag
- for (MergedAnomalyResultDTO notifiedAnomaly : anomalyCandidates) {
- notifiedAnomaly.setNotified(true);
- mergeAnomalyDAO.update(notifiedAnomaly);
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/FunctionCreationUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/FunctionCreationUtils.java
deleted file mode 100644
index 1e61b00..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/FunctionCreationUtils.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.utils;
-
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import org.apache.commons.lang3.StringUtils;
-
-
-public class FunctionCreationUtils {
-
- /**
- * Return the valid dimension string
- * @param dataset the dataset configuration
- * @param exploreDimensions the dimensions to be explored
- * @return a dimension string with valid dimensions
- * @throws Exception
- */
- public static String getDimensions(DatasetConfigDTO dataset, String exploreDimensions) throws Exception {
- // Ensure that the explore dimension names are ordered as schema dimension names
- List<String> schemaDimensionNames = dataset.getDimensions();
- Set<String> splitExploreDimensions = new HashSet<>(Arrays.asList(exploreDimensions.trim().split(",")));
- List<String> reorderedDimensions = new ArrayList<>();
- for (String dimensionName : schemaDimensionNames) {
- if (splitExploreDimensions.contains(dimensionName)) {
- reorderedDimensions.add(dimensionName);
- }
- }
- return StringUtils.join(reorderedDimensions, ",");
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/PropertyCheckUtils.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/PropertyCheckUtils.java
deleted file mode 100644
index befd3ea..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/onboard/utils/PropertyCheckUtils.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.utils;
-
-import com.google.common.base.Preconditions;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-
-public class PropertyCheckUtils {
- /**
- * check if the list of property keys exist in the given properties
- * @param properties
- * @param propertyKeys
- */
- public static void checkNotNull(Map<String, String> properties, List<String> propertyKeys) {
- Preconditions.checkNotNull(properties);
- Preconditions.checkNotNull(propertyKeys);
-
- List<String> missingPropertyKeys = new ArrayList<>();
-
- for (String propertyKey : propertyKeys) {
- if (properties.get(propertyKey) == null) {
- missingPropertyKeys.add(propertyKey);
- }
- }
-
- if (!missingPropertyKeys.isEmpty()) {
- throw new IllegalArgumentException("Missing Property Keys: " + missingPropertyKeys);
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskInfoFactory.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskInfoFactory.java
index ede19d7..4552a80 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskInfoFactory.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskInfoFactory.java
@@ -24,7 +24,6 @@ import org.apache.pinot.thirdeye.anomaly.alert.AlertTaskInfo;
import org.apache.pinot.thirdeye.anomaly.classification.ClassificationTaskInfo;
import org.apache.pinot.thirdeye.anomaly.detection.DetectionTaskInfo;
import org.apache.pinot.thirdeye.anomaly.monitor.MonitorTaskInfo;
-import org.apache.pinot.thirdeye.anomaly.onboard.ReplayTaskInfo;
import org.apache.pinot.thirdeye.anomaly.task.TaskConstants.TaskType;
import org.apache.pinot.thirdeye.completeness.checker.DataCompletenessTaskInfo;
import org.apache.pinot.thirdeye.detection.DetectionPipelineTaskInfo;
@@ -75,9 +74,6 @@ public class TaskInfoFactory {
case CLASSIFICATION:
taskInfo = OBJECT_MAPPER.readValue(taskInfoString, ClassificationTaskInfo.class);
break;
- case REPLAY:
- taskInfo = OBJECT_MAPPER.readValue(taskInfoString, ReplayTaskInfo.class);
- break;
default:
LOG.error("TaskType must be one of ANOMALY_DETECTION, MONITOR, ALERT, ALERT2, DATA_COMPLETENESS, CLASSIFICATION");
break;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskRunnerFactory.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskRunnerFactory.java
index deac910..2f6851f 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskRunnerFactory.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/anomaly/task/TaskRunnerFactory.java
@@ -23,7 +23,6 @@ import org.apache.pinot.thirdeye.anomaly.alert.v2.AlertTaskRunnerV2;
import org.apache.pinot.thirdeye.anomaly.classification.ClassificationTaskRunner;
import org.apache.pinot.thirdeye.anomaly.detection.DetectionTaskRunner;
import org.apache.pinot.thirdeye.anomaly.monitor.MonitorTaskRunner;
-import org.apache.pinot.thirdeye.anomaly.onboard.ReplayTaskRunner;
import org.apache.pinot.thirdeye.anomaly.task.TaskConstants.TaskType;
import org.apache.pinot.thirdeye.completeness.checker.DataCompletenessTaskRunner;
import org.apache.pinot.thirdeye.detection.DetectionPipelineTaskRunner;
@@ -66,8 +65,6 @@ public class TaskRunnerFactory {
case CLASSIFICATION:
taskRunner = new ClassificationTaskRunner();
break;
- case REPLAY:
- taskRunner = new ReplayTaskRunner();
default:
}
return taskRunner;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index a08b0d4..a2a86b5 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -24,7 +24,6 @@ import com.google.common.cache.CacheBuilder;
import io.dropwizard.auth.AuthValueFactoryProvider;
import io.dropwizard.auth.Authenticator;
import org.apache.pinot.thirdeye.anomaly.detection.DetectionJobScheduler;
-import org.apache.pinot.thirdeye.anomaly.onboard.DetectionOnboardResource;
import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
import org.apache.pinot.thirdeye.api.application.ApplicationResource;
import org.apache.pinot.thirdeye.auth.ThirdEyeCredentials;
@@ -51,7 +50,6 @@ import org.apache.pinot.thirdeye.dashboard.resources.EntityManagerResource;
import org.apache.pinot.thirdeye.dashboard.resources.EntityMappingResource;
import org.apache.pinot.thirdeye.dashboard.resources.MetricConfigResource;
import org.apache.pinot.thirdeye.dashboard.resources.OnboardDatasetMetricResource;
-import org.apache.pinot.thirdeye.dashboard.resources.OnboardResource;
import org.apache.pinot.thirdeye.dashboard.resources.SummaryResource;
import org.apache.pinot.thirdeye.dashboard.resources.ThirdEyeResource;
import org.apache.pinot.thirdeye.dashboard.resources.v2.AnomaliesResource;
@@ -175,7 +173,6 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new ThirdEyeResource());
env.jersey().register(new DataResource(anomalyFunctionFactory, alertFilterFactory));
env.jersey().register(new AnomaliesResource(anomalyFunctionFactory, alertFilterFactory));
- env.jersey().register(new OnboardResource(config));
env.jersey().register(new EntityMappingResource());
env.jersey().register(new OnboardDatasetMetricResource());
env.jersey().register(new AutoOnboardResource(config));
@@ -190,8 +187,6 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new ApplicationResource(
DAO_REGISTRY.getApplicationDAO(), DAO_REGISTRY.getMergedAnomalyResultDAO(),
DAO_REGISTRY.getDetectionConfigManager(), DAO_REGISTRY.getDetectionAlertConfigManager()));
- env.jersey().register(new DetectionOnboardResource(
- DAO_REGISTRY.getTaskDAO(), DAO_REGISTRY.getAnomalyFunctionDAO()));
env.jersey().register(new DetectionResource());
env.jersey().register(new DetectionAlertResource(DAO_REGISTRY.getDetectionAlertConfigManager()));
env.jersey().register(new YamlResource(config.getDetectionPreviewConfig()));
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyResource.java
index 1ada50c..90a357a 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/AnomalyResource.java
@@ -23,7 +23,6 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.cache.LoadingCache;
import org.apache.pinot.pql.parsers.utils.Pair;
import org.apache.pinot.thirdeye.anomaly.alert.util.AlertFilterHelper;
-import org.apache.pinot.thirdeye.anomaly.onboard.utils.FunctionCreationUtils;
import org.apache.pinot.thirdeye.anomaly.views.AnomalyTimelinesView;
import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyFeedback;
@@ -31,9 +30,7 @@ import org.apache.pinot.thirdeye.anomalydetection.context.TimeSeries;
import org.apache.pinot.thirdeye.api.Constants;
import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
-import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.constant.AnomalyResultSource;
-import org.apache.pinot.thirdeye.constant.MetricAggFunction;
import org.apache.pinot.thirdeye.dashboard.resources.v2.AnomaliesResource;
import org.apache.pinot.thirdeye.dashboard.views.TimeBucket;
import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
@@ -41,11 +38,9 @@ import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
import org.apache.pinot.thirdeye.datalayer.bao.AutotuneConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AutotuneConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
@@ -54,7 +49,6 @@ import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
import org.apache.pinot.thirdeye.detector.email.filter.BaseAlertFilter;
import org.apache.pinot.thirdeye.detector.email.filter.DummyAlertFilter;
import org.apache.pinot.thirdeye.detector.function.AnomalyFunctionFactory;
-import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
import org.apache.pinot.thirdeye.util.TimeSeriesUtils;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
@@ -88,13 +82,10 @@ import org.joda.time.Interval;
import org.joda.time.Period;
import org.joda.time.Weeks;
import org.joda.time.format.ISODateTimeFormat;
-import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
-import static org.apache.pinot.thirdeye.anomaly.detection.lib.AutotuneMethodType.*;
-
@Path(value = "/dashboard")
@Api(tags = { Constants.ANOMALY_TAG })
@@ -104,22 +95,17 @@ public class AnomalyResource {
private static final Logger LOG = LoggerFactory.getLogger(AnomalyResource.class);
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private static final String DEFAULT_CRON = "0 0 0 * * ?";
private static final String UTF8 = "UTF-8";
- private static final String DEFAULT_FUNCTION_TYPE = "WEEK_OVER_WEEK_RULE";
private AnomalyFunctionManager anomalyFunctionDAO;
private MergedAnomalyResultManager anomalyMergedResultDAO;
private AlertConfigManager emailConfigurationDAO;
- private MetricConfigManager metricConfigDAO;
private MergedAnomalyResultManager mergedAnomalyResultDAO;
private AutotuneConfigManager autotuneConfigDAO;
private DatasetConfigManager datasetConfigDAO;
private AnomalyFunctionFactory anomalyFunctionFactory;
private AlertFilterFactory alertFilterFactory;
- private AlertFilterAutotuneFactory alertFilterAutotuneFactory;
private LoadingCache<String, Long> collectionMaxDataTimeCache;
- private LoadingCache<String, String> dimensionFiltersCache;
private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance();
@@ -128,15 +114,12 @@ public class AnomalyResource {
this.anomalyFunctionDAO = DAO_REGISTRY.getAnomalyFunctionDAO();
this.anomalyMergedResultDAO = DAO_REGISTRY.getMergedAnomalyResultDAO();
this.emailConfigurationDAO = DAO_REGISTRY.getAlertConfigDAO();
- this.metricConfigDAO = DAO_REGISTRY.getMetricConfigDAO();
this.mergedAnomalyResultDAO = DAO_REGISTRY.getMergedAnomalyResultDAO();
this.autotuneConfigDAO = DAO_REGISTRY.getAutotuneConfigDAO();
this.datasetConfigDAO = DAO_REGISTRY.getDatasetConfigDAO();
this.anomalyFunctionFactory = anomalyFunctionFactory;
this.alertFilterFactory = alertFilterFactory;
- this.alertFilterAutotuneFactory = alertFilterAutotuneFactory;
this.collectionMaxDataTimeCache = CACHE_REGISTRY_INSTANCE.getDatasetMaxDataTimeCache();
- this.dimensionFiltersCache = CACHE_REGISTRY_INSTANCE.getDimensionFiltersCache();
}
/************** CRUD for anomalies of a collection ********************************************************/
@@ -261,203 +244,6 @@ public class AnomalyResource {
}
/**
- * Endpoint to be used for creating new anomaly function
- * @param dataset name of dataset for the new anomaly function
- * @param functionName name of the new anomaly function. Should follow convention "productName_metricName_dimensionName_other"
- * @param metric name of metric on Thirdeye of the new anomaly function
- * @param metric_function was using as a consolidation function, now by default use "SUM"
- * @param type type of anomaly function. Minutely metrics use SIGN_TEST_VANILLA,
- * Hourly metrics use REGRESSION_GAUSSIAN_SCAN, Daily metrics use SPLINE_REGRESSION
- * @param windowSize Detection window size
- * @param windowUnit Detection window unit
- * @param windowDelay Detection window delay (wait for data completeness)
- * @param windowDelayUnit Detection window delay unit
- * @param cron cron of detection
- * @param exploreDimensions explore dimensions in JSON
- * @param filters filters on dimensions
- * @param userInputDataGranularity user input metric granularity, if null uses dataset granularity
- * @param properties properties for anomaly function
- * @param isActive whether set the new anomaly function to be active or not
- * @return new anomaly function Id if successfully create new anomaly function
- * @throws Exception
- */
- @POST
- @Path("/anomaly-function")
- @ApiOperation("Endpoint to be used for creating new anomaly function")
- public Response createAnomalyFunction(@NotNull @QueryParam("dataset") String dataset,
- @NotNull @QueryParam("functionName") String functionName, @NotNull @QueryParam("metric") String metric,
- @NotNull @QueryParam("metricFunction") String metric_function, @QueryParam("type") String type,
- @NotNull @QueryParam("windowSize") String windowSize, @NotNull @QueryParam("windowUnit") String windowUnit,
- @QueryParam("windowDelay") @DefaultValue("0") String windowDelay, @QueryParam("cron") String cron,
- @QueryParam("windowDelayUnit") String windowDelayUnit, @QueryParam("exploreDimension") String exploreDimensions,
- @QueryParam("filters") String filters, @QueryParam("dataGranularity") String userInputDataGranularity,
- @NotNull @QueryParam("properties") String properties, @QueryParam("isActive") boolean isActive) throws Exception {
- if (StringUtils.isEmpty(dataset) || StringUtils.isEmpty(functionName) || StringUtils.isEmpty(metric)
- || StringUtils.isEmpty(windowSize) || StringUtils.isEmpty(windowUnit) || properties == null) {
- throw new IllegalArgumentException(String.format("Received nulll or emtpy String for one of the mandatory params: "
- + "dataset: %s, metric: %s, functionName %s, windowSize: %s, windowUnit: %s, and properties: %s", dataset,
- metric, functionName, windowSize, windowUnit, properties));
- }
-
- TimeGranularity dataGranularity;
- DatasetConfigDTO datasetConfig = DAO_REGISTRY.getDatasetConfigDAO().findByDataset(dataset);
- if (datasetConfig == null) {
- throw new IllegalArgumentException(String.format("No entry with dataset name %s exists", dataset));
- }
- if (userInputDataGranularity == null) {
- TimeSpec timespec = ThirdEyeUtils.getTimeSpecFromDatasetConfig(datasetConfig);
- dataGranularity = timespec.getDataGranularity();
- } else {
- dataGranularity = TimeGranularity.fromString(userInputDataGranularity);
- }
-
- AnomalyFunctionDTO anomalyFunctionSpec = new AnomalyFunctionDTO();
- anomalyFunctionSpec.setActive(isActive);
- anomalyFunctionSpec.setMetricFunction(MetricAggFunction.valueOf(metric_function));
- anomalyFunctionSpec.setCollection(dataset);
- anomalyFunctionSpec.setFunctionName(functionName);
- anomalyFunctionSpec.setTopicMetric(metric);
- anomalyFunctionSpec.setMetrics(Arrays.asList(metric));
- if (StringUtils.isEmpty(type)) {
- type = DEFAULT_FUNCTION_TYPE;
- }
- anomalyFunctionSpec.setType(type);
- anomalyFunctionSpec.setWindowSize(Integer.valueOf(windowSize));
- anomalyFunctionSpec.setWindowUnit(TimeUnit.valueOf(windowUnit));
-
- // Setting window delay time / unit
- TimeUnit dataGranularityUnit = dataGranularity.getUnit();
-
- // default windowDelay = 0, can be adjusted to cope with expected delay for given dataset/metric
- int windowDelayTime = Integer.valueOf(windowDelay);
- TimeUnit windowDelayTimeUnit;
- switch (dataGranularityUnit) {
- case MINUTES:
- windowDelayTimeUnit = TimeUnit.MINUTES;
- break;
- case DAYS:
- windowDelayTimeUnit = TimeUnit.DAYS;
- break;
- case HOURS:
- default:
- windowDelayTimeUnit = TimeUnit.HOURS;
- }
- if (StringUtils.isNotBlank(windowDelayUnit)) {
- windowDelayTimeUnit = TimeUnit.valueOf(windowDelayUnit.toUpperCase());
- }
- anomalyFunctionSpec.setWindowDelayUnit(windowDelayTimeUnit);
- anomalyFunctionSpec.setWindowDelay(windowDelayTime);
-
- // setup detection frequency if it's minutely function
- // the default frequency in AnomalyDetectionFunctionBean is 1 hour
- // when detection job is scheduled, the frequency may also be modified by data granularity
- // this frequency is a override to allow longer time period (lower frequency) and reduce system load
- if (dataGranularity.getUnit().equals(TimeUnit.MINUTES)) {
- TimeGranularity frequency = new TimeGranularity(15, TimeUnit.MINUTES);
- anomalyFunctionSpec.setFrequency(frequency);
- }
-
- // bucket size and unit are defaulted to the collection granularity
- anomalyFunctionSpec.setBucketSize(dataGranularity.getSize());
- anomalyFunctionSpec.setBucketUnit(dataGranularity.getUnit());
-
- if (StringUtils.isNotEmpty(exploreDimensions)) {
- anomalyFunctionSpec.setExploreDimensions(FunctionCreationUtils.getDimensions(datasetConfig, exploreDimensions));
- }
- if (!StringUtils.isBlank(filters)) {
- filters = URLDecoder.decode(filters, UTF8);
- String filterString = ThirdEyeUtils.getSortedFiltersFromJson(filters);
- anomalyFunctionSpec.setFilters(filterString);
- }
- anomalyFunctionSpec.setProperties(properties);
-
- if (StringUtils.isEmpty(cron)) {
- cron = DEFAULT_CRON;
- } else {
- // validate cron
- if (!CronExpression.isValidExpression(cron)) {
- throw new IllegalArgumentException("Invalid cron expression for cron : " + cron);
- }
- }
- anomalyFunctionSpec.setCron(cron);
-
- Long id = anomalyFunctionDAO.save(anomalyFunctionSpec);
-
- return Response.ok(id).build();
- }
-
- /**
- * Apply an autotune configuration to an existing function
- * @param id
- * The id of an autotune configuration
- * @param isCloneFunction
- * Should we clone the function or simply apply the autotune configuration to the existing function
- * @return
- * an activated anomaly detection function
- */
- @POST
- @Path("/anomaly-function/apply/{autotuneConfigId}")
- @ApiOperation("Apply an autotune configuration to an existing function")
- public Response applyAutotuneConfig(@PathParam("autotuneConfigId") @NotNull long id,
- @QueryParam("cloneFunction") @DefaultValue("false") boolean isCloneFunction,
- @QueryParam("cloneAnomalies") Boolean isCloneAnomalies) {
- Map<String, String> responseMessage = new HashMap<>();
- AutotuneConfigDTO autotuneConfigDTO = autotuneConfigDAO.findById(id);
- if (autotuneConfigDTO == null) {
- responseMessage.put("message", "Cannot find the autotune configuration entry " + id + ".");
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- if (autotuneConfigDTO.getConfiguration() == null || autotuneConfigDTO.getConfiguration().isEmpty()) {
- responseMessage.put("message",
- "Autotune configuration is null or empty. The original function is optimal. Nothing to change");
- return Response.ok(responseMessage).build();
- }
-
- if (isCloneAnomalies == null) { // if isCloneAnomalies is not given, assign a default value
- isCloneAnomalies = containsLabeledAnomalies(autotuneConfigDTO.getFunctionId());
- }
-
- AnomalyFunctionDTO originalFunction = anomalyFunctionDAO.findById(autotuneConfigDTO.getFunctionId());
- AnomalyFunctionDTO targetFunction = originalFunction;
-
- // clone anomaly function and its anomaly results if requested
- if (isCloneFunction) {
- OnboardResource onboardResource = new OnboardResource(anomalyFunctionDAO, mergedAnomalyResultDAO);
- long cloneId;
- String tag = "clone";
- try {
- cloneId = onboardResource.cloneAnomalyFunctionById(originalFunction.getId(), "clone", isCloneAnomalies);
- } catch (Exception e) {
- responseMessage.put("message",
- "Unable to clone function " + originalFunction.getId() + " with clone tag \"clone\"");
- LOG.warn("Unable to clone function {} with clone tag \"{}\"", originalFunction.getId(), "clone");
- return Response.status(Response.Status.CONFLICT).entity(responseMessage).build();
- }
- targetFunction = anomalyFunctionDAO.findById(cloneId);
- }
-
- // Verify if to update alert filter or function configuraions
- // if auto tune method is EXHAUSTIVE, which belongs to function auto tune, need to update function configurations
- // if auto tune method is ALERT_FILTER_LOGISITC_AUTO_TUNE or INITIATE_ALERT_FILTER_LOGISTIC_AUTO_TUNE, alert filter is to be updated
- if (autotuneConfigDTO.getAutotuneMethod() != EXHAUSTIVE) {
- targetFunction.setAlertFilter(autotuneConfigDTO.getConfiguration());
- } else {
- // Update function configuration
- targetFunction.updateProperties(autotuneConfigDTO.getConfiguration());
- }
- targetFunction.setActive(true);
- anomalyFunctionDAO.update(targetFunction);
-
- // Deactivate original function
- if (isCloneFunction) {
- originalFunction.setActive(false);
- anomalyFunctionDAO.update(originalFunction);
- }
-
- return Response.ok(targetFunction).build();
- }
-
- /**
* Get the timeseries with function baseline for an anomaly function.
* The function baseline can be 1) the baseline for online analysis, including baseline in training and testing range,
* and 2) the baseline for offline analysis, where only training time rage is included.
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/DetectionJobResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/DetectionJobResource.java
index 681a3c4..699b90e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/DetectionJobResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/DetectionJobResource.java
@@ -23,10 +23,8 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiOperation;
import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.BaseAlertFilterAutoTune;
import org.apache.pinot.thirdeye.api.Constants;
-import org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardConfiguration;
import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.ApplicationDTO;
@@ -59,7 +57,6 @@ import javax.ws.rs.core.Response.Status;
import org.apache.pinot.thirdeye.anomaly.detection.DetectionJobScheduler;
import org.apache.commons.lang3.StringUtils;
import org.joda.time.DateTime;
-import org.joda.time.Interval;
import org.joda.time.format.ISODateTimeFormat;
import org.json.JSONArray;
import org.json.JSONException;
@@ -68,13 +65,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.pinot.thirdeye.anomaly.detection.lib.AutotuneMethodType;
-import org.apache.pinot.thirdeye.anomaly.detection.lib.FunctionReplayRunnable;
import org.apache.pinot.thirdeye.anomaly.job.JobConstants.JobStatus;
import org.apache.pinot.thirdeye.anomaly.utils.AnomalyUtils;
import org.apache.pinot.thirdeye.anomalydetection.alertFilterAutotune.AlertFilterAutotuneFactory;
-import org.apache.pinot.thirdeye.anomalydetection.performanceEvaluation.PerformanceEvaluateHelper;
-import org.apache.pinot.thirdeye.anomalydetection.performanceEvaluation.PerformanceEvaluationMethod;
-import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
import org.apache.pinot.thirdeye.datalayer.bao.AutotuneConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
@@ -155,302 +148,6 @@ public class DetectionJobResource {
return Response.ok().build();
}
- /**
- * The wrapper endpoint to do first time replay, tuning and send out notification to user
- * TODO: Remove this wrapper method after funciton onboard is ready on FE
- * @param id anomaly function id
- * @param startTimeIso start time of replay
- * @param endTimeIso end time of replay
- * @param isForceBackfill whether force back fill or not, default is true
- * @param isRemoveAnomaliesInWindow whether need to remove exsiting anomalies within replay time window, default is false
- * @param speedup whether use speedUp or not
- * @param userDefinedPattern tuning parameter, user defined pattern can be "UP", "DOWN", or "UP&DOWN"
- * @param sensitivity sensitivity level for initial tuning
- * @param fromAddr email notification from address, if blank uses fromAddr of ThirdEyeConfiguration
- * @param toAddr email notification to address
- * @param teHost thirdeye host, if black uses thirdeye host configured in ThirdEyeConfiguration
- * @param smtpHost smtp host if black uses smtpHost configured in ThirdEyeConfiguration
- * @param smtpPort smtp port if black uses smtpPort configured in ThirdEyeConfiguration
- * @param phantomJsPath phantomJSpath
- * @return
- */
- @POST
- @Path("/{id}/notifyreplaytuning")
- public Response triggerReplayTuningAndNotification(@PathParam("id") @NotNull final long id,
- @QueryParam("start") @NotNull String startTimeIso, @QueryParam("end") @NotNull String endTimeIso,
- @QueryParam("force") @DefaultValue("true") String isForceBackfill,
- @QueryParam("removeAnomaliesInWindow") @DefaultValue("false") final Boolean isRemoveAnomaliesInWindow,
- @QueryParam("speedup") @DefaultValue("false") final Boolean speedup,
- @QueryParam("userDefinedPattern") @DefaultValue("UP") String userDefinedPattern,
- @QueryParam("sensitivity") @DefaultValue("MEDIUM") final String sensitivity, @QueryParam("from") String fromAddr,
- @QueryParam("to") String toAddr, @QueryParam("cc") String ccAddr, @QueryParam("bcc") String bccAddr,
- @QueryParam("teHost") String teHost, @QueryParam("smtpHost") String smtpHost,
- @QueryParam("smtpPort") Integer smtpPort, @QueryParam("phantomJsPath") String phantomJsPath) {
-
- if (emailResource == null) {
- LOG.error("Unable to proceed this function without email resource");
- return Response.status(Status.EXPECTATION_FAILED).entity("No email resource").build();
- }
- // run replay, update function with jobId
- long jobId;
- try {
- Response response =
- generateAnomaliesInRange(id, startTimeIso, endTimeIso, isForceBackfill, speedup, isRemoveAnomaliesInWindow);
- Map<Long, Long> entity = (Map<Long, Long>) response.getEntity();
- jobId = entity.get(id);
- } catch (Exception e) {
- return Response.status(Status.BAD_REQUEST).entity("Failed to start replay!").build();
- }
-
- long startTime = ISODateTimeFormat.dateTimeParser().parseDateTime(startTimeIso).getMillis();
- long endTime = ISODateTimeFormat.dateTimeParser().parseDateTime(endTimeIso).getMillis();
- JobStatus jobStatus = detectionJobScheduler.waitForJobDone(jobId);
- int numReplayedAnomalies = 0;
- if (!jobStatus.equals(JobStatus.COMPLETED)) {
- //TODO: cleanup done tasks and replay results under this failed job
- // send email to internal
- String replayFailureSubject =
- new StringBuilder("Replay failed on metric: " + anomalyFunctionDAO.findById(id).getMetric()).toString();
- String replayFailureText = new StringBuilder("Failed on Function: " + id + "with Job Id: " + jobId).toString();
- emailResource.sendEmailWithText(null, null, null, null, replayFailureSubject,
- replayFailureText, smtpHost, smtpPort);
- return Response.status(Status.BAD_REQUEST).entity("Replay job error with job status: {}" + jobStatus).build();
- } else {
- numReplayedAnomalies =
- mergedAnomalyResultDAO.findByStartTimeInRangeAndFunctionId(startTime, endTime, id).size();
- LOG.info("Replay completed with {} anomalies generated.", numReplayedAnomalies);
- }
-
- // create initial tuning and apply filter
- Response initialAutotuneResponse =
- initiateAlertFilterAutoTune(id, startTimeIso, endTimeIso, "AUTOTUNE", userDefinedPattern, sensitivity, "", "");
- if (initialAutotuneResponse.getEntity() != null) {
- updateAlertFilterToFunctionSpecByAutoTuneId(Long.valueOf(initialAutotuneResponse.getEntity().toString()));
- LOG.info("Initial alert filter applied");
- } else {
- LOG.info("AutoTune doesn't applied");
- }
-
- // send out email
- String subject = new StringBuilder(
- "Replay results for " + anomalyFunctionDAO.findById(id).getFunctionName() + " is ready for review!").toString();
-
- emailResource.generateAndSendAlertForFunctions(startTime, endTime, String.valueOf(id), fromAddr, toAddr, ccAddr,
- bccAddr, subject, false, true, teHost, smtpHost, smtpPort, phantomJsPath);
- LOG.info("Sent out email");
-
- return Response.ok("Replay, Tuning and Notification finished!").build();
- }
-
-
- /**
- * Breaks down the given range into consecutive monitoring windows as per function definition
- * Regenerates anomalies for each window separately
- *
- * As the anomaly result regeneration is a heavy job, we move the function from Dashboard to worker
- * @param id an anomaly function id
- * @param startTimeIso The start time of the monitoring window (in ISO Format), ex: 2016-5-23T00:00:00Z
- * @param endTimeIso The start time of the monitoring window (in ISO Format)
- * @param isForceBackfill false to resume backfill from the latest left off
- * @param speedup
- * whether this backfill should speedup with 7-day window. The assumption is that the functions are using
- * WoW-based algorithm, or Seasonal Data Model.
- * @return HTTP response of this request with a job execution id
- * @throws Exception
- */
- @POST
- @Path("/{id}/replay")
- @ApiOperation(" Breaks down the given range into consecutive monitoring windows as per function definition\n "
- + "Regenerates anomalies for each window separately")
- public Response generateAnomaliesInRange(@PathParam("id") @NotNull final long id,
- @QueryParam("start") @NotNull String startTimeIso, @QueryParam("end") @NotNull String endTimeIso,
- @QueryParam("force") @DefaultValue("false") String isForceBackfill,
- @QueryParam("speedup") @DefaultValue("false") final Boolean speedup,
- @QueryParam("removeAnomaliesInWindow") @DefaultValue("false") final Boolean isRemoveAnomaliesInWindow)
- throws Exception {
- Response response =
- generateAnomaliesInRangeForFunctions(Long.toString(id), startTimeIso, endTimeIso, isForceBackfill, speedup,
- isRemoveAnomaliesInWindow);
- return response;
- }
-
- /**
- * Breaks down the given range into consecutive monitoring windows as per function definition
- * Regenerates anomalies for each window separately
- *
- * Different from the previous replay function, this replay function takes multiple function ids, and is able to send
- * out alerts to user once the replay is done.
- *
- * Enable replay on inactive function, but still keep the original function status after replay
- * If the anomaly function has historical anomalies, will only remove anomalies within the replay period, making replay capable to take historical information
- *
- * As the anomaly result regeneration is a heavy job, we move the function from Dashboard to worker
- * @param ids a string containing multiple anomaly function ids, separated by comma (e.g. f1,f2,f3)
- * @param startTimeIso The start time of the monitoring window (in ISO Format), ex: 2016-5-23T00:00:00Z
- * @param endTimeIso The start time of the monitoring window (in ISO Format)
- * @param isForceBackfill false to resume backfill from the latest left off
- * @param speedup
- * whether this backfill should speedup with 7-day window. The assumption is that the functions are using
- * WoW-based algorithm, or Seasonal Data Model.
- * @param isRemoveAnomaliesInWindow whether remove existing anomalies in replay window
- * @return HTTP response of this request with a map from function id to its job execution id
- * @throws Exception
- */
- @POST
- @Path("/replay")
- public Response generateAnomaliesInRangeForFunctions(@QueryParam("ids") @NotNull String ids,
- @QueryParam("start") @NotNull String startTimeIso, @QueryParam("end") @NotNull String endTimeIso,
- @QueryParam("force") @DefaultValue("false") String isForceBackfill,
- @QueryParam("speedup") @DefaultValue("false") final Boolean speedup,
- @QueryParam("removeAnomaliesInWindow") @DefaultValue("false") final Boolean isRemoveAnomaliesInWindow)
- throws Exception {
- final boolean forceBackfill = Boolean.valueOf(isForceBackfill);
- final List<Long> functionIdList = new ArrayList<>();
- final Map<Long, Long> detectionJobIdMap = new HashMap<>();
- for (String functionId : ids.split(",")) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(Long.valueOf(functionId));
- if (anomalyFunction != null) {
- functionIdList.add(Long.valueOf(functionId));
- } else {
- LOG.warn("[Backfill] Unable to load function id {}", functionId);
- }
- }
- if (functionIdList.isEmpty()) {
- return Response.noContent().build();
- }
-
- // Check if the timestamps are available
- DateTime startTime = null;
- DateTime endTime = null;
- if (startTimeIso == null || startTimeIso.isEmpty()) {
- LOG.error("[Backfill] Monitoring start time is not found");
- throw new IllegalArgumentException(String.format("[Backfill] Monitoring start time is not found"));
- }
- if (endTimeIso == null || endTimeIso.isEmpty()) {
- LOG.error("[Backfill] Monitoring end time is not found");
- throw new IllegalArgumentException(String.format("[Backfill] Monitoring end time is not found"));
- }
-
- startTime = ISODateTimeFormat.dateTimeParser().parseDateTime(startTimeIso);
- endTime = ISODateTimeFormat.dateTimeParser().parseDateTime(endTimeIso);
-
- if (startTime.isAfter(endTime)) {
- LOG.error("[Backfill] Monitoring start time is after monitoring end time");
- throw new IllegalArgumentException(
- String.format("[Backfill] Monitoring start time is after monitoring end time"));
- }
- if (endTime.isAfterNow()) {
- endTime = DateTime.now();
- LOG.warn("[Backfill] End time is in the future. Force to now.");
- }
-
- final Map<Long, Integer> originalWindowSize = new HashMap<>();
- final Map<Long, TimeUnit> originalWindowUnit = new HashMap<>();
- final Map<Long, String> originalCron = new HashMap<>();
- saveFunctionWindow(functionIdList, originalWindowSize, originalWindowUnit, originalCron);
-
- // Update speed-up window and cron
- if (speedup) {
- for (long functionId : functionIdList) {
- anomalyFunctionSpeedup(functionId);
- }
- }
-
- // Run backfill : for each function, set to be active to enable runBackfill,
- // remove existing anomalies if there is any already within the replay window (to avoid duplicated anomaly results)
- // set the original activation status back after backfill
- for (long functionId : functionIdList) {
- // Activate anomaly function if it's inactive
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- Boolean isActive = anomalyFunction.getIsActive();
- if (!isActive) {
- anomalyFunction.setActive(true);
- anomalyFunctionDAO.update(anomalyFunction);
- }
- // if isRemoveAnomaliesInWindow is true, remove existing anomalies within same replay window
- if (isRemoveAnomaliesInWindow) {
- OnboardResource onboardResource = new OnboardResource(new ThirdEyeDashboardConfiguration());
- onboardResource.deleteExistingAnomalies(functionId, startTime.getMillis(), endTime.getMillis());
- }
-
- // run backfill
- long detectionJobId = detectionJobScheduler.runBackfill(functionId, startTime, endTime, forceBackfill);
- // Put back activation status
- anomalyFunction.setActive(isActive);
- anomalyFunctionDAO.update(anomalyFunction);
- detectionJobIdMap.put(functionId, detectionJobId);
- }
-
- /**
- * Check the job status in thread and recover the function
- */
- new Thread(new Runnable() {
- @Override
- public void run() {
- for (long detectionJobId : detectionJobIdMap.values()) {
- detectionJobScheduler.waitForJobDone(detectionJobId);
- }
- // Revert window setup
- revertFunctionWindow(functionIdList, originalWindowSize, originalWindowUnit, originalCron);
- }
- }).start();
-
- return Response.ok(detectionJobIdMap).build();
- }
-
- /**
- * Under current infrastructure, we are not able to determine whether and how we accelerate the backfill.
- * Currently, the requirement for speedup is to increase the window up to 1 week.
- * For WoW-based models, if we enlarge the window to 7 days, it can significantly increase the backfill speed.
- * Now, the hard-coded code is a contemporary solution to this problem. It can be fixed under new infra.
- *
- * TODO Data model provide information on how the function can speed up, and user determines if they wnat to speed up
- * TODO the replay
- * @param functionId
- */
- private void anomalyFunctionSpeedup(long functionId) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- TimeUnit dataTimeUnit = anomalyFunction.getBucketUnit();
- switch (dataTimeUnit) {
- case MINUTES:
- anomalyFunction.setWindowSize(170);
- anomalyFunction.setWindowUnit(TimeUnit.HOURS);
- anomalyFunction.setCron("0 0 0 ? * MON *");
- break;
- case HOURS:
- anomalyFunction.setCron("0 0 0/6 1/1 * ? *");
- default:
- }
- anomalyFunctionDAO.update(anomalyFunction);
- }
-
- private void saveFunctionWindow(List<Long> functionIdList, Map<Long, Integer> windowSize,
- Map<Long, TimeUnit> windowUnit, Map<Long, String> cron) {
- for (long functionId : functionIdList) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- windowSize.put(functionId, anomalyFunction.getWindowSize());
- windowUnit.put(functionId, anomalyFunction.getWindowUnit());
- cron.put(functionId, anomalyFunction.getCron());
- }
- }
-
- private void revertFunctionWindow(List<Long> functionIdList, Map<Long, Integer> windowSize,
- Map<Long, TimeUnit> windowUnit, Map<Long, String> cron) {
- for (long functionId : functionIdList) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if (windowSize.containsKey(functionId)) {
- anomalyFunction.setWindowSize(windowSize.get(functionId));
- }
- if (windowUnit.containsKey(functionId)) {
- anomalyFunction.setWindowUnit(windowUnit.get(functionId));
- }
- if (cron.containsKey(functionId)) {
- anomalyFunction.setCron(cron.get(functionId));
- }
- anomalyFunctionDAO.update(anomalyFunction);
- }
- }
-
@POST
@Path("/{id}/offlineAnalysis")
public Response generateAnomaliesInTrainingData(@PathParam("id") @NotNull long id,
@@ -930,142 +627,6 @@ public class DetectionJobResource {
}
/**
- * Perform anomaly function autotune:
- * - run backfill on all possible combinations of tuning parameters
- * - keep all the parameter combinations which lie in the goal range
- * - return list of parameter combinations along with their performance evaluation
- * @param functionId
- * the id of the target anomaly function
- * @param replayTimeIso
- * the end time of the anomaly function replay in ISO format, e.g. 2017-02-27T00:00:00.000Z
- * @param replayDuration
- * the duration of the replay ahead of the replayStartTimeIso
- * @param durationUnit
- * the time unit of the duration, DAYS, HOURS, MINUTES and so on
- * @param speedup
- * whether we speedup the replay process
- * @param tuningJSON
- * the json object includes all tuning fields and list of parameters
- * ex: {"baselineLift": [0.9, 0.95, 1, 1.05, 1.1], "baselineSeasonalPeriod": [2, 3, 4]}
- * @param goal
- * the expected performance assigned by user
- * @param includeOrigin
- * to include the performance of original setup into comparison
- * If we perform offline analysis before hand, we don't get the correct performance about the current configuration
- * setup. Therefore, we need to exclude the performance from the comparison.
- * @return
- * A response containing all satisfied properties with their evaluation result
- */
- @Deprecated
- @POST
- @Path("replay/function/{id}")
- public Response anomalyFunctionReplay(@PathParam("id") @NotNull long functionId,
- @QueryParam("time") String replayTimeIso, @QueryParam("duration") @DefaultValue("30") int replayDuration,
- @QueryParam("durationUnit") @DefaultValue("DAYS") String durationUnit,
- @QueryParam("speedup") @DefaultValue("true") boolean speedup,
- @QueryParam("tune") @DefaultValue("{\"pValueThreshold\":[0.05, 0.01]}") String tuningJSON,
- @QueryParam("goal") @DefaultValue("0.05") double goal,
- @QueryParam("includeOriginal") @DefaultValue("true") boolean includeOrigin,
- @QueryParam("evalMethod") @DefaultValue("ANOMALY_PERCENTAGE") String performanceEvaluationMethod) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if (anomalyFunction == null) {
- LOG.warn("Unable to find anomaly function {}", functionId);
- return Response.status(Response.Status.BAD_REQUEST).entity("Cannot find function").build();
- }
- DateTime replayStart = null;
- DateTime replayEnd = null;
- try {
- TimeUnit timeUnit = TimeUnit.valueOf(durationUnit.toUpperCase());
-
- TimeGranularity timeGranularity = new TimeGranularity(replayDuration, timeUnit);
- replayEnd = DateTime.now();
- if (StringUtils.isNotEmpty(replayTimeIso)) {
- replayEnd = ISODateTimeFormat.dateTimeParser().parseDateTime(replayTimeIso);
- }
- replayStart = replayEnd.minus(timeGranularity.toPeriod());
- } catch (Exception e) {
- throw new WebApplicationException("Unable to parse strings, " + replayTimeIso + ", in ISO DateTime format", e);
- }
-
- // List all tuning parameter sets
- List<Map<String, String>> tuningParameters = null;
- try {
- tuningParameters = listAllTuningParameters(new JSONObject(tuningJSON));
- } catch (JSONException e) {
- LOG.error("Unable to parse json string: {}", tuningJSON, e);
- return Response.status(Response.Status.BAD_REQUEST).build();
- }
- if (tuningParameters.size() == 0) { // no tuning combinations
- LOG.warn("No tuning parameter is found in json string {}", tuningJSON);
- return Response.status(Response.Status.BAD_REQUEST).build();
- }
- AutotuneMethodType autotuneMethodType = AutotuneMethodType.EXHAUSTIVE;
- PerformanceEvaluationMethod performanceEvalMethod =
- PerformanceEvaluationMethod.valueOf(performanceEvaluationMethod.toUpperCase());
-
- Map<String, Double> originalPerformance = new HashMap<>();
- originalPerformance.put(performanceEvalMethod.name(),
- PerformanceEvaluateHelper.getPerformanceEvaluator(performanceEvalMethod, functionId, functionId,
- new Interval(replayStart.getMillis(), replayEnd.getMillis()), mergedAnomalyResultDAO).evaluate());
-
- // select the functionAutotuneConfigDTO in DB
- //TODO: override existing autotune results by a method "autotuneConfigDAO.udpate()"
- AutotuneConfigDTO targetDTO = null;
- List<AutotuneConfigDTO> functionAutoTuneConfigDTOList =
- autotuneConfigDAO.findAllByFuctionIdAndWindow(functionId, replayStart.getMillis(), replayEnd.getMillis());
- for (AutotuneConfigDTO configDTO : functionAutoTuneConfigDTOList) {
- if (configDTO.getAutotuneMethod().equals(autotuneMethodType) && configDTO.getPerformanceEvaluationMethod()
- .equals(performanceEvalMethod) && configDTO.getStartTime() == replayStart.getMillis()
- && configDTO.getEndTime() == replayEnd.getMillis() && configDTO.getGoal() == goal) {
- targetDTO = configDTO;
- break;
- }
- }
-
- if (targetDTO == null) { // Cannot find existing dto
- targetDTO = new AutotuneConfigDTO();
- targetDTO.setFunctionId(functionId);
- targetDTO.setAutotuneMethod(autotuneMethodType);
- targetDTO.setPerformanceEvaluationMethod(performanceEvalMethod);
- targetDTO.setStartTime(replayStart.getMillis());
- targetDTO.setEndTime(replayEnd.getMillis());
- targetDTO.setGoal(goal);
- autotuneConfigDAO.save(targetDTO);
- }
-
- // clear message;
- targetDTO.setMessage("");
- if (includeOrigin) {
- targetDTO.setPerformance(originalPerformance);
- } else {
- targetDTO.setPerformance(Collections.EMPTY_MAP);
- }
- autotuneConfigDAO.update(targetDTO);
-
- // Setup threads and start to run
- for (Map<String, String> config : tuningParameters) {
- LOG.info("Running backfill replay with parameter configuration: {}" + config.toString());
- FunctionReplayRunnable backfillRunnable =
- new FunctionReplayRunnable(detectionJobScheduler, anomalyFunctionDAO, mergedAnomalyResultDAO,
- autotuneConfigDAO);
- backfillRunnable.setTuningFunctionId(functionId);
- backfillRunnable.setFunctionAutotuneConfigId(targetDTO.getId());
- backfillRunnable.setReplayStart(replayStart);
- backfillRunnable.setReplayEnd(replayEnd);
- backfillRunnable.setForceBackfill(true);
- backfillRunnable.setGoal(goal);
- backfillRunnable.setSpeedUp(speedup);
- backfillRunnable.setPerformanceEvaluationMethod(performanceEvalMethod);
- backfillRunnable.setAutotuneMethodType(autotuneMethodType);
- backfillRunnable.setTuningParameter(config);
-
- new Thread(backfillRunnable).start();
- }
-
- return Response.ok(targetDTO.getId()).build();
- }
-
- /**
* Parse the jsonobject and list all the possible configuration combinations
* @param tuningJSON the input json string from user
* @return full list of all the possible configurations
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/OnboardResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/OnboardResource.java
deleted file mode 100644
index 6b6da81..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/OnboardResource.java
+++ /dev/null
@@ -1,681 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.dashboard.resources;
-
-import com.codahale.metrics.Counter;
-import com.google.common.base.CaseFormat;
-import org.apache.pinot.thirdeye.anomaly.onboard.DetectionOnboardResource;
-import org.apache.pinot.thirdeye.anomaly.onboard.tasks.DefaultDetectionOnboardJob;
-import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardUtility;
-import org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardConfiguration;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import io.swagger.annotations.ApiOperation;
-import io.swagger.annotations.ApiParam;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import javax.validation.constraints.NotNull;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.joda.time.DateTime;
-import org.joda.time.format.ISODateTimeFormat;
-import org.quartz.CronExpression;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.pinot.thirdeye.anomaly.onboard.tasks.FunctionCreationOnboardingTask.*;
-import static org.apache.pinot.thirdeye.dashboard.resources.EntityManagerResource.*;
-
-
-@Path("/onboard")
-@Produces(MediaType.APPLICATION_JSON)
-@Consumes(MediaType.APPLICATION_JSON)
-@Deprecated
-public class OnboardResource {
- private final AnomalyFunctionManager anomalyFunctionDAO;
- private final MergedAnomalyResultManager mergedAnomalyResultDAO;
- private MetricConfigManager metricConfigDAO;
- private DatasetConfigManager datasetFunctionDAO;
- private AlertConfigManager emailConfigurationDAO;
- private TaskManager taskDAO;
- private ThirdEyeDashboardConfiguration config;
-
- private static final String DEFAULT_FUNCTION_PREFIX = "thirdEyeAutoOnboard_";
- private static final String DEFAULT_ALERT_GROUP = "te_bulk_onboard_alerts";
- private static final String DEFAULT_ALERT_GROUP_APPLICATION = "others";
- private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance();
- private static final Logger LOG = LoggerFactory.getLogger(OnboardResource.class);
-
- public OnboardResource(ThirdEyeDashboardConfiguration config) {
- this.config = config;
- this.anomalyFunctionDAO = DAO_REGISTRY.getAnomalyFunctionDAO();
- this.datasetFunctionDAO = DAO_REGISTRY.getDatasetConfigDAO();
- this.mergedAnomalyResultDAO = DAO_REGISTRY.getMergedAnomalyResultDAO();
- this.metricConfigDAO = DAO_REGISTRY.getMetricConfigDAO();
- this.emailConfigurationDAO = DAO_REGISTRY.getAlertConfigDAO();
- this.taskDAO = DAO_REGISTRY.getTaskDAO();
- }
-
- public OnboardResource(AnomalyFunctionManager anomalyFunctionManager,
- MergedAnomalyResultManager mergedAnomalyResultManager) {
- this.anomalyFunctionDAO = anomalyFunctionManager;
- this.mergedAnomalyResultDAO = mergedAnomalyResultManager;
- }
-
- /**
- * Endpoint for bulk onboarding of metrics
- *
- * This endpoint will create anomaly functions for all the metrics under the given tag
- * and also has the ability to auto create alert config groups with dataset owners as
- * recipients and send out email alerts.
- *
- * @param tag the tag belonging to the metrics which you would like to onboard
- * @param dataset the dataset belonging to the metrics which you would like to onboard
- * @param functionPrefix (optional, DEFAULT_FUNCTION_PREFIX) a custom anomaly function prefix
- * @param forceSyncAlertGroup (optional, true) force create alert groups based on dataset owners
- * @param alertGroupName (optional) subscribe to a custom subscription alert group
- * @param application (optional) the application to which this alert belongs to.
- * @return HTTP response containing onboard statistics and warnings
- */
- @POST
- @Path("/bulk-onboard")
- @ApiOperation("Endpoint used for bulk on-boarding alerts leveraging the create-job endpoint.")
- public Response bulkOnboardAlert(
- @QueryParam("tag") String tag,
- @QueryParam("dataset") String dataset,
- @DefaultValue(DEFAULT_FUNCTION_PREFIX) @QueryParam("functionPrefix") String functionPrefix,
- @DefaultValue("true") @QueryParam("forceSyncAlertGroup") boolean forceSyncAlertGroup,
- @QueryParam("alertGroupName") String alertGroupName, @QueryParam("alertGroupCron") String alertGroupCron,
- @QueryParam("application") String application,
- @QueryParam("sleep") Long sleep)
- throws Exception {
- Map<String, String> responseMessage = new HashMap<>();
- Counter counter = new Counter();
-
- if (StringUtils.isBlank(tag) && StringUtils.isBlank(dataset)) {
- responseMessage.put("message", "Must provide either tag or dataset");
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
-
- AlertConfigDTO alertConfigDTO = null;
- if (StringUtils.isNotEmpty(alertGroupName)) {
- alertConfigDTO = emailConfigurationDAO.findWhereNameEquals(alertGroupName);
- if (alertConfigDTO == null) {
- responseMessage.put("message", "cannot find an alert group with name " + alertGroupName + ".");
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- }
-
- List<MetricConfigDTO> metrics = new ArrayList<>();
-
- if (StringUtils.isNotBlank(tag)) {
- List<MetricConfigDTO> tagMetrics = fetchMetricsByTag(tag);
- LOG.info("Number of metrics with tag {} fetched is {}.", tag, tagMetrics.size());
- metrics.addAll(tagMetrics);
- }
-
- if (StringUtils.isNotBlank(dataset)) {
- List<MetricConfigDTO> datasetMetrics = fetchMetricsByDataset(dataset);
- LOG.info("Number of metrics with dataset {} fetched is {}.", dataset, datasetMetrics.size());
- metrics.addAll(datasetMetrics);
- }
-
- // For each metric create a new anomaly function & replay it
- List<Long> ids = new ArrayList<>();
- for (MetricConfigDTO metric : metrics) {
- String functionName = functionPrefix
- + CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, metric.getName()) + "_"
- + CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, metric.getDataset());
-
- LOG.info("[bulk-onboard] Onboarding anomaly function {}.", functionName);
-
- if (alertConfigDTO == null) {
- alertConfigDTO = getAlertConfigGroupForMetric(metric, forceSyncAlertGroup, application, alertGroupCron);
- if (alertConfigDTO == null) {
- responseMessage.put("message", "cannot find an alert group for metric " + metric.getName() + ".");
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
- }
- }
-
- AnomalyFunctionDTO anomalyFunctionDTO = anomalyFunctionDAO.findWhereNameEquals(functionName);
- if (anomalyFunctionDTO != null) {
- LOG.error("[bulk-onboard] Anomaly function {} already exists.", anomalyFunctionDTO.getFunctionName());
- responseMessage.put("metric " + metric.getName(), "skipped! Anomaly function "
- + anomalyFunctionDTO.getFunctionName() + " already exists.");
- continue;
- }
-
- try {
- Map<String, String> properties = new HashMap<>();
- properties.put(DefaultDetectionOnboardJob.FUNCTION_NAME, functionName);
- properties.put(DefaultDetectionOnboardJob.METRIC_NAME, metric.getName());
- properties.put(DefaultDetectionOnboardJob.COLLECTION_NAME, metric.getDataset());
- properties.put("alertId", alertConfigDTO.getId().toString());
- String propertiesJson = OBJECT_MAPPER.writeValueAsString(properties);
- DetectionOnboardResource detectionOnboardResource = new DetectionOnboardResource(taskDAO, anomalyFunctionDAO);
- detectionOnboardResource.createDetectionOnboardingJob(functionName, propertiesJson);
- long functionId = anomalyFunctionDAO.findWhereNameEquals(functionName).getId();
- ids.add(functionId);
- subscribeAlertGroupToFunction(alertConfigDTO, functionId);
- responseMessage.put("metric " + metric.getName(), "success! onboarded and added function id " + functionId
- + " to subscription alertGroup = " + alertConfigDTO.getName());
- counter.inc();
-
- if (sleep != null) {
- Thread.sleep(sleep);
- }
-
- } catch (InterruptedException e) {
- responseMessage.put("message", "Operation interrupted");
- return Response.status(Response.Status.BAD_REQUEST).entity(responseMessage).build();
-
- } catch (Exception e) {
- LOG.error("[bulk-onboard] There was an exception onboarding metric {} function {}.", metric, functionName, e);
- responseMessage.put("skipped " + metric.getName(), "Exception onboarding metric : " + e);
- }
- }
-
- responseMessage.put("message", "successfully onboarded " + counter.getCount() + " metrics with function ids " + ids);
- return Response.ok(responseMessage).build();
- }
-
- private void subscribeAlertGroupToFunction(AlertConfigDTO alertConfigDTO, long functionId) {
- if (alertConfigDTO.getEmailConfig() == null) {
- AlertConfigBean.EmailConfig emailConfig = new AlertConfigBean.EmailConfig();
- List<Long> functionIds = new ArrayList<>();
- functionIds.add(functionId);
- emailConfig.setFunctionIds(functionIds);
- alertConfigDTO.setEmailConfig(emailConfig);
- } else {
- alertConfigDTO.getEmailConfig().getFunctionIds().add(functionId);
- }
- emailConfigurationDAO.update(alertConfigDTO);
- }
-
- private AlertConfigDTO getAlertConfigGroupForMetric(MetricConfigDTO metric, boolean forceSyncAlertGroup,
- String application, String cron) {
- String alertGroupName = AutoOnboardUtility.getAutoAlertGroupName(metric.getDataset());
- AlertConfigDTO alertConfigDTO = emailConfigurationDAO.findWhereNameEquals(alertGroupName);
-
- if (forceSyncAlertGroup) {
- syncAlertConfig(alertConfigDTO, alertGroupName, metric, application, cron);
- alertConfigDTO = emailConfigurationDAO.findWhereNameEquals(alertGroupName);
- } else {
- if (alertConfigDTO == null) {
- LOG.warn("Cannot find alert group {} corresponding to dataset {} for metric {}. Loading default alert group {}",
- alertGroupName, metric.getDataset(), metric.getName(), DEFAULT_ALERT_GROUP);
- alertConfigDTO = emailConfigurationDAO.findWhereNameEquals(DEFAULT_ALERT_GROUP);
- }
- }
-
- return alertConfigDTO;
- }
-
- private void syncAlertConfig(AlertConfigDTO alertConfigDTO, String alertGroupName, MetricConfigDTO metric,
- String application, String cron) {
- Set<String> metricOwners = getOwners(metric);
- if (alertConfigDTO == null) {
- createAlertConfig(alertGroupName, application, cron, metricOwners);
- } else {
- // Note: Since we support only one subscription group per function in the legacy code, we append
- // dataset owners and interested stakeholders to the same auto created subscription group.
- // Side effect:
- // If a dataset owner is removed at source, which rarely is the case, then we will continue to
- // retain the owner in our subscription group and send him alerts unless manually removed.
- if (alertConfigDTO.getReceiverAddresses() == null) {
- alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients(metricOwners));
- } else if (alertConfigDTO.getReceiverAddresses().getTo() == null) {
- alertConfigDTO.getReceiverAddresses().setTo(metricOwners);
- } else {
- alertConfigDTO.getReceiverAddresses().getTo().addAll(metricOwners);
- }
- this.emailConfigurationDAO.update(alertConfigDTO);
- LOG.info("Alert config {} with id {} has been updated.", alertConfigDTO.getName(), alertConfigDTO.getId());
- }
- }
-
- private Set<String> getOwners(MetricConfigDTO metric) {
- Set<String> owners = new HashSet<>();
- if (metric != null && metric.getDataset() != null) {
- DatasetConfigDTO datasetConfigDTO = datasetFunctionDAO.findByDataset(metric.getDataset());
- if (datasetConfigDTO != null && datasetConfigDTO.getOwners() != null) {
- owners.addAll(datasetConfigDTO.getOwners());
- }
- }
- return owners;
- }
-
-
- private Long createAlertConfig(String alertGroupName, String application, String cron, Set<String> recipients) {
- if (StringUtils.isEmpty(cron)) {
- cron = DEFAULT_ALERT_CRON;
- } else {
- if (!CronExpression.isValidExpression(cron)) {
- throw new IllegalArgumentException("Invalid cron expression : " + cron);
- }
- }
-
- if (StringUtils.isEmpty(application)) {
- application = DEFAULT_ALERT_GROUP_APPLICATION;
- }
-
- AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
- alertConfigDTO.setName(alertGroupName);
- alertConfigDTO.setApplication(application);
- alertConfigDTO.setActive(true);
- alertConfigDTO.setFromAddress(config.getFailureFromAddress());
- alertConfigDTO.setCronExpression(cron);
- alertConfigDTO.setReceiverAddresses(new DetectionAlertFilterRecipients(recipients));
- return this.emailConfigurationDAO.save(alertConfigDTO);
- }
-
- private List<MetricConfigDTO> fetchMetricsByTag(String tag) {
- List<MetricConfigDTO> results = new ArrayList<>();
- for (MetricConfigDTO metricConfigDTO : this.metricConfigDAO.findAll()) {
- if (metricConfigDTO.getTags() != null) {
- if (metricConfigDTO.getTags().contains(tag)) {
- results.add(metricConfigDTO);
- }
- }
- }
- return results;
- }
-
- private List<MetricConfigDTO> fetchMetricsByDataset(String dataset) {
- return this.metricConfigDAO.findByDataset(dataset);
- }
-
- // endpoint clone function Ids to append a name defined in nameTags
-
- @GET
- @Path("function/{id}")
- @ApiOperation("GET a single function record by id")
- public AnomalyFunctionDTO getAnomalyFunction(@ApiParam("alert function id\n") @PathParam("id") Long id) {
- return anomalyFunctionDAO.findById(id);
- }
-
- // clone functions in batch
- @POST
- @Path("function/clone")
- public List<Long> cloneFunctionsGetIds(@QueryParam("functionId") String functionIds,
- @QueryParam("nameTags") String nameTags,
- @QueryParam("cloneAnomaly")@DefaultValue("false") String cloneAnomaly)
- throws Exception {
- ArrayList<Long> cloneFunctionIds = new ArrayList<>();
- try {
- Map<Long, String> idNameTagMap = parseFunctionIdsAndNameTags(functionIds, nameTags);
-
- for (Map.Entry<Long, String> entry : idNameTagMap.entrySet()) {
- cloneFunctionIds.add(cloneAnomalyFunctionById(entry.getKey(), entry.getValue(), Boolean.valueOf(cloneAnomaly)));
- }
- } catch (Exception e) {
- LOG.error("Can not clone function Ids {}, with name tags {}", functionIds, nameTags);
- throw new WebApplicationException(e);
- }
- return cloneFunctionIds;
- }
-
- // clone function 1 by 1
- @POST
- @Path("function/{id}/clone/{tag}")
- public Long cloneFunctionGetId(@PathParam("id") Long id,
- @PathParam("tag") String tag,
- @QueryParam("cloneAnomaly")@DefaultValue("false") String cloneAnomaly)
- throws Exception {
- try {
- return cloneAnomalyFunctionById(id, tag, Boolean.valueOf(cloneAnomaly));
- } catch (Exception e) {
- LOG.error("Can not clone function: Id {}, with name tag {}", id, tag);
- throw new WebApplicationException(e);
- }
- }
-
- /**
- * Copy total configurations of source anomaly function (with srcId) to destination anomaly function (with destId)
- * Explicit representation: denote source anomaly function with (srcId, srcFunctionName, srcConfigs)
- * denote destination anomaly function with (destId, destFunctionName, destConfigs)
- * "copyConfigFromSrcToDest" will update destination anomaly function's configurations by source anomaly function's configurations
- * After "copyConfigFromSrcToDest", the two functions will become:
- * (srcId, srcFunctionName, srcConfigs)
- * (destId, destFunctionName, srcConfigs)
- * This in fact updates source anomaly function's properties into destination function's properties
- * @param srcId : the source function with configurations to be copied to
- * @param destId : the destination function Id that will have its configurations being overwritten by source function
- * @return OK is success
- */
- @POST
- @Path("function/{srcId}/copyTo/{destId}")
- public Response copyConfigFromSrcToDest(@PathParam("srcId") @NotNull Long srcId,
- @PathParam("destId") @NotNull Long destId) {
- AnomalyFunctionDTO srcAnomalyFunction = anomalyFunctionDAO.findById(srcId);
- AnomalyFunctionDTO destAnomalyFunction = anomalyFunctionDAO.findById(destId);
- if (srcAnomalyFunction == null) {
- // LOG and exit
- LOG.error("Anomaly Function With id [{}] does not found", srcId);
- return Response.status(Response.Status.BAD_REQUEST).entity("Cannot find function with id: " + srcId).build();
- }
-
- if (destAnomalyFunction == null) {
- // LOG and exit
- LOG.error("Anomaly Function With id [{}] does not found", destId);
- return Response.status(Response.Status.BAD_REQUEST).entity("Cannot find function with id: " + srcId).build();
- }
- // Thirdeye database uses (functionId, functionName) as an identity for each anomaly function,
- // here by updating the identity of source anomaly function into destination anomaly function's Id and function name,
- // source anomaly function will inherit destination anomaly function's total configurations
- srcAnomalyFunction.setId(destId);
- srcAnomalyFunction.setFunctionName(destAnomalyFunction.getFunctionName());
- anomalyFunctionDAO.update(srcAnomalyFunction); // update configurations
- return Response.ok().build();
- }
-
- /**
- * Clone anomalies from source anomaly function to destination anomaly function in time range
- * @param srcId : function Id of source anomaly function
- * @param destId : function Id of destination anomaly function
- * @param startTimeIso : start time of anomalies to be cloned, time in ISO format ex: 2016-5-23T00:00:00Z
- * @param endTimeIso : end time of anomalies to be cloned, time in ISO format
- * @return true if at least one anomaly being cloned
- * @throws Exception
- */
- @POST
- @Path("function/{srcId}/cloneAnomalies/{destId}")
- public Response ClonedAnomalies(@PathParam("srcId") @NotNull long srcId,
- @PathParam("destId") @NotNull long destId,
- @QueryParam("start") String startTimeIso,
- @QueryParam("end") String endTimeIso) {
-
- long start = 0;
- long end = DateTime.now().getMillis();
- try {
- if (startTimeIso != null) {
- start = ISODateTimeFormat.dateTimeParser().parseDateTime(startTimeIso).getMillis();
- }
- if (endTimeIso != null) {
- end = ISODateTimeFormat.dateTimeParser().parseDateTime(endTimeIso).getMillis();
- }
- } catch (Exception e) {
- throw new WebApplicationException("Failed to parse time, startTime: " + startTimeIso + ", endTime: " + endTimeIso);
- }
- Boolean isAnyCloned = cloneAnomalyInstances(srcId, destId, start, end);
- return Response.ok(isAnyCloned).build();
- }
-
-
-
- // util functions for clone anomaly functions
- /**
- * Parse function ids to be cloned and clone name tags together
- * the length should be aligned otherwise name tags are all empty
- * or all replace with the first name tag (if length == 1)
- *
- * @param functionIds ',' separated string with numbers representing function ids to be cloned
- * @param nameTags ',' separated string with strings representing name tags for clone functions
- * @return a HashMap from function id to be cloned and the new name tag
- */
- private Map<Long, String> parseFunctionIdsAndNameTags(String functionIds, String nameTags) {
- List<Long> functionIdsList = new ArrayList<>();
- if (StringUtils.isNotBlank(functionIds)) {
- String[] tokens = functionIds.split(",");
- for (String token : tokens) {
- functionIdsList.add(Long.valueOf(token));
- }
- }
-
- int len = functionIdsList.size();
-
- List<String> nameTagList = new ArrayList<>();
- if (StringUtils.isNotBlank(nameTags)) {
- String[] tags = nameTags.split(",");
- if (tags.length == 1) {
- for (int i = 0; i < len; i++) {
- nameTagList.add(tags[0]);
- LOG.debug("only 1 tag, use the tag for all function");
- }
- } else {
- if (tags.length == len) {
- for (String tag : tags) {
- nameTagList.add(tag);
- }
- } else {
- LOG.debug("tag list and function id list does not mach, use empty strings as tags");
- for (int i = 0; i < len; i++) {
- nameTagList.add("");
- }
- }
- }
- }
-
- LOG.info("function ids set: {}", functionIds);
- LOG.info("name tag set: {}", nameTagList);
- // Construct Return results
- HashMap<Long, String> IdNameTagMap = new HashMap<>();
- for (int i=0; i < len; i++) {
- IdNameTagMap.put(functionIdsList.get(i), nameTagList.get(i));
- }
- return IdNameTagMap;
- }
-
- /**
- * clone Anomaly Function by Id to a function name appended with a name tag
- *
- * @param id function id to be cloned
- * @param cloneNameTag an name tag that will append to original function name as serve as the cloned function name
- * @param doCloneAnomaly does the associated anomaly instances will be clone to new function
- * @return id of the clone function
- */
- public Long cloneAnomalyFunctionById(long id, String cloneNameTag, boolean doCloneAnomaly) throws Exception {
- // get anomaly function definition
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(id);
- // if cannot find then return
- if (anomalyFunction == null) {
- // LOG and exit
- LOG.error("Anomaly Function With id [{}] does not found", id);
- return null;
- }
- String functionName = anomalyFunction.getFunctionName();
- String newFunctionName = functionName;
- if (cloneNameTag != null && cloneNameTag.length() > 0) {
- newFunctionName = newFunctionName + "_" + cloneNameTag; // update with input clone function name
- } else {
- newFunctionName = newFunctionName + "_" + "clone_0"; // set default clone function name
- }
- LOG.info("Try to clone anomaly Function Id: {}, Name: {}, to new Name: {}", id, functionName, newFunctionName);
- anomalyFunction.setFunctionName(newFunctionName);
- anomalyFunction.setActive(false); // deactivate the function
- anomalyFunction.setId(null);
- long newId = anomalyFunctionDAO.save(anomalyFunction); // exception should be handled when have duplicate name
- LOG.info("clone function id: {}, name: {} to id: {}, name: {}", id, functionName, newId, newFunctionName);
-
- if (doCloneAnomaly) {
- cloneAnomalyInstances(id, newId, 0, DateTime.now().getMillis());
- }
- return newId;
- }
-
- /**
- * Clone merged anomaly instances of one function to another
- * 1. get all merged anomaly instances with AnomalyFunctionId = srcId
- * 2. set the associated anomalyFunctionId = destId
- * 3. save the modified anomaly instances
- * @param srcId the source function Id with anomalies to be cloned.
- * @param destId the destination function Id which source anomalies to be cloned to.
- * @param start the start time of anomalies from source function Id to be cloned.
- * @param end the end time of anomalies from source function Id to be cloned.
- * @return boolean to indicate if the clone is success or not
- */
- public Boolean cloneAnomalyInstances(Long srcId, Long destId, long start, long end) {
-
- // make sure both function can be identified by IDs
-
- AnomalyFunctionDTO srcAnomalyFunction = anomalyFunctionDAO.findById(srcId);
- // if cannot find then return
- if (srcAnomalyFunction == null) {
- // LOG and exit
- LOG.error("Source Anomaly Function With id [{}] does not found", srcId);
- return false;
- }
-
- AnomalyFunctionDTO destAnomalyFunction = anomalyFunctionDAO.findById(destId);
- // if cannot find then return
- if (destAnomalyFunction == null) {
- // LOG and exit
- LOG.error("Destination Anomaly Function With id [{}] does not found", destId);
- return false;
- }
-
- LOG.info("clone merged anomaly results from source anomaly function id {} to id {}", srcId, destId);
-
- List<MergedAnomalyResultDTO> mergedAnomalyResultDTOs = mergedAnomalyResultDAO.findByStartTimeInRangeAndFunctionId(start, end, srcId);
- if (mergedAnomalyResultDTOs == null || mergedAnomalyResultDTOs.isEmpty()) {
- LOG.error("No merged anomaly results found for anomaly function Id: {}", srcId);
- return false;
- }
-
- for (MergedAnomalyResultDTO mergedAnomalyResultDTO : mergedAnomalyResultDTOs) {
- long oldId = mergedAnomalyResultDTO.getId();
- mergedAnomalyResultDTO.setId(null); // clean the Id, then will create a new Id when save
- mergedAnomalyResultDTO.setFunctionId(destId);
- mergedAnomalyResultDTO.setFunction(destAnomalyFunction);
- long newId = mergedAnomalyResultDAO.save(mergedAnomalyResultDTO);
- LOG.debug("clone merged anomaly {} to {}", oldId, newId);
- }
- return true;
- }
-
- /**
- * Delete raw and merged anomalies whose start time is located in the given time ranges
- * @param startTimeIso The start time of the monitoring window, if null, use smallest time
- * @param endTimeIso The start time of the monitoring window, if null, use current time
- */
- @DELETE
- @Path("function/{id}/anomalies")
- public Map<String, Integer> deleteAnomalies(@PathParam("id") Long functionId,
- @QueryParam("start") String startTimeIso,
- @QueryParam("end") String endTimeIso) {
-
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if (anomalyFunction == null) {
- LOG.info("Anomaly functionId {} is not found", functionId);
- return null;
- }
-
- long startTime = 0;
- long endTime = DateTime.now().getMillis();
- try {
- if (startTimeIso != null) {
- startTime = ISODateTimeFormat.dateTimeParser().parseDateTime(startTimeIso).getMillis();
- }
- if (endTimeIso != null) {
- endTime = ISODateTimeFormat.dateTimeParser().parseDateTime(endTimeIso).getMillis();
- }
- } catch (Exception e) {
- throw new WebApplicationException("Unable to parse strings, " + startTimeIso + " and " + endTimeIso
- + ", in ISO DateTime format", e);
- }
-
- LOG.info("Delete anomalies of function {} in the time range: {} -- {}", functionId, startTimeIso, endTimeIso);
-
- return deleteExistingAnomalies(functionId, startTime, endTime);
- }
-
- /**
- * Delete raw or merged anomalies whose start time is located in the given time ranges, except
- * the following two cases:
- *
- * 1. If a raw anomaly belongs to a merged anomaly whose start time is not located in the given
- * time ranges, then the raw anomaly will not be deleted.
- *
- * 2. If a raw anomaly belongs to a merged anomaly whose start time is located in the given
- * time ranges, then it is deleted regardless its start time.
- *
- * If monitoringWindowStartTime is not given, then start time is set to 0.
- * If monitoringWindowEndTime is not given, then end time is set to Long.MAX_VALUE.
- * @param functionId function id
- * @param monitoringWindowStartTime The start time of the monitoring window (in milli-second)
- * @param monitoringWindowEndTime The start time of the monitoring window (in milli-second)
- */
- public Map<String, Integer> deleteExistingAnomalies(long functionId,
- long monitoringWindowStartTime,
- long monitoringWindowEndTime) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if (anomalyFunction == null) {
- LOG.info("Anomaly functionId {} is not found", functionId);
- return null;
- }
- HashMap<String, Integer> returnInfo = new HashMap<>();
-
- // Find merged anomaly result and delete them first
- LOG.info("Deleting merged anomaly results in the time range: {} -- {}", new DateTime(monitoringWindowStartTime), new
- DateTime(monitoringWindowEndTime));
- LOG.info("Beginning cleanup merged anomaly results of functionId {} collection {} metric {}",
- functionId, anomalyFunction.getCollection(), anomalyFunction.getMetric());
- int mergedAnomaliesDeleted = 0;
- List<MergedAnomalyResultDTO> mergedResults =
- mergedAnomalyResultDAO.findByStartTimeInRangeAndFunctionId(monitoringWindowStartTime, monitoringWindowEndTime, functionId);
- if (CollectionUtils.isNotEmpty(mergedResults)) {
- mergedAnomaliesDeleted = deleteMergedResults(mergedResults);
- }
- returnInfo.put("mergedAnomaliesDeleted", mergedAnomaliesDeleted);
- LOG.info("{} merged anomaly results have been deleted", mergedAnomaliesDeleted);
-
- return returnInfo;
- }
-
- // Delete merged anomaly results from mergedAnomalyResultDAO
- private int deleteMergedResults(List<MergedAnomalyResultDTO> mergedResults) {
- LOG.info("Deleting merged results");
- int mergedAnomaliesDeleted = 0;
- for (MergedAnomalyResultDTO mergedResult : mergedResults) {
- LOG.info("...Deleting merged result id {} for functionId {}", mergedResult.getId(), mergedResult.getFunctionId());
- mergedAnomalyResultDAO.delete(mergedResult);
- mergedAnomaliesDeleted++;
- }
- return mergedAnomaliesDeleted;
- }
-}
\ No newline at end of file
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
index 1960c39..74e52cf 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java
@@ -23,8 +23,10 @@ import com.google.common.base.Preconditions;
import com.google.inject.Singleton;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFeedbackDTO;
import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AnomalyFeedbackBean;
import org.apache.pinot.thirdeye.datalayer.pojo.AnomalyFunctionBean;
+import org.apache.pinot.thirdeye.datalayer.pojo.DetectionConfigBean;
import org.apache.pinot.thirdeye.datalayer.pojo.MetricConfigBean;
import java.util.ArrayList;
import java.util.Collections;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/MergedAnomalyResultDTO.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/MergedAnomalyResultDTO.java
index eac43fe..d280cd4 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/MergedAnomalyResultDTO.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dto/MergedAnomalyResultDTO.java
@@ -35,6 +35,7 @@ public class MergedAnomalyResultDTO extends MergedAnomalyResultBean implements A
private AnomalyFeedbackDTO feedback;
private AnomalyFunctionDTO function;
+ private DetectionConfigDTO detectionConfig;
private Set<MergedAnomalyResultDTO> children = new HashSet<>();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/AnomalyFunctionBean.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/AnomalyFunctionBean.java
index 96e2c96..bb391c2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/AnomalyFunctionBean.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/pojo/AnomalyFunctionBean.java
@@ -24,7 +24,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Multimap;
import org.apache.pinot.thirdeye.anomaly.merge.AnomalyMergeConfig;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobStatus;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.constant.MetricAggFunction;
import org.apache.pinot.thirdeye.util.ThirdEyeUtils;
@@ -97,8 +96,6 @@ public class AnomalyFunctionBean extends AbstractBean {
private AnomalyMergeConfig anomalyMergeConfig;
- private DetectionOnboardJobStatus onboardJobStatus;
-
/**
* This flag always true.
* This flag would typically be unset, in backfill cases, where we want to override the completeness check,
@@ -334,14 +331,6 @@ public class AnomalyFunctionBean extends AbstractBean {
this.windowDelay = windowDelay;
}
- public DetectionOnboardJobStatus getOnboardJobStatus() {
- return onboardJobStatus;
- }
-
- public void setOnboardJobStatus(DetectionOnboardJobStatus onboardJobStatus) {
- this.onboardJobStatus = onboardJobStatus;
- }
-
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -362,7 +351,7 @@ public class AnomalyFunctionBean extends AbstractBean {
&& Objects.equals(windowDelay, that.windowDelay) && windowDelayUnit == that.windowDelayUnit && Objects.equals(
exploreDimensions, that.exploreDimensions) && Objects.equals(filters, that.filters) && Objects.equals(
dataFilter, that.dataFilter) && Objects.equals(alertFilter, that.alertFilter) && Objects.equals(
- anomalyMergeConfig, that.anomalyMergeConfig) && Objects.equals(onboardJobStatus, that.onboardJobStatus);
+ anomalyMergeConfig, that.anomalyMergeConfig);
}
@Override
@@ -370,6 +359,6 @@ public class AnomalyFunctionBean extends AbstractBean {
return Objects.hash(collection, functionName, metric, metrics, metricFunction, type, isActive, globalMetric,
globalMetricFilters, properties, cron, frequency, bucketSize, bucketUnit, windowSize, windowUnit, windowDelay,
windowDelayUnit, exploreDimensions, filters, metricId, dataFilter, alertFilter, anomalyMergeConfig,
- onboardJobStatus, requiresCompletenessCheck);
+ requiresCompletenessCheck);
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
index c85a806..79d4a79 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/alert/scheme/DetectionEmailAlerter.java
@@ -20,7 +20,6 @@
package org.apache.pinot.thirdeye.detection.alert.scheme;
import com.google.common.base.Preconditions;
-import com.google.common.collect.SetMultimap;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
@@ -37,7 +36,6 @@ import org.apache.commons.mail.HtmlEmail;
import org.apache.pinot.thirdeye.anomaly.SmtpConfiguration;
import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.AlertConfigBean;
@@ -193,7 +191,7 @@ public class DetectionEmailAlerter extends DetectionAlertScheme {
/**
* Plug the appropriate email subject style based on configuration
*/
- private AlertConfigBean.SubjectType makeSubject(Map<String, Object> emailParams) {
+ private AlertConfigBean.SubjectType makeEmailSubjectType(Map<String, Object> emailParams) {
AlertConfigBean.SubjectType subjectType;
if (emailParams != null && emailParams.containsKey(PROP_EMAIL_SUBJECT_STYLE)) {
subjectType = AlertConfigBean.SubjectType.valueOf(emailParams.get(PROP_EMAIL_SUBJECT_STYLE).toString());
@@ -212,6 +210,8 @@ public class DetectionEmailAlerter extends DetectionAlertScheme {
validateAlert(recipients, anomalies);
Map<String, Object> emailParams = ConfigUtils.getMap(this.config.getAlertSchemes().get(PROP_EMAIL_SCHEME));
+ this.config.setSubjectType(makeEmailSubjectType(emailParams));
+
Properties emailProps = new Properties();
emailProps.putAll(emailParams);
BaseNotificationContent content = makeTemplate(emailParams);
@@ -221,14 +221,8 @@ public class DetectionEmailAlerter extends DetectionAlertScheme {
List<AnomalyResult> anomalyResultListOfGroup = new ArrayList<>(anomalies);
anomalyResultListOfGroup.sort(COMPARATOR_DESC);
- AlertConfigDTO alertConfig = new AlertConfigDTO();
- alertConfig.setName(this.config.getName());
- alertConfig.setFromAddress(this.config.getFrom());
- alertConfig.setSubjectType(makeSubject(emailParams));
- alertConfig.setReferenceLinks(this.config.getReferenceLinks());
-
ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAlertConfig(alertConfig);
+ context.setNotificationConfig(this.config);
EmailEntity emailEntity = emailContentFormatter.getEmailEntity(null,
"Thirdeye Alert : " + this.config.getName(), anomalyResultListOfGroup,
context);
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/BaseNotificationContent.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/BaseNotificationContent.java
index f90a9a6..3d85f6d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/BaseNotificationContent.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/BaseNotificationContent.java
@@ -48,7 +48,7 @@ import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.dashboard.resources.v2.AnomaliesResource;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.EventDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
@@ -183,7 +183,7 @@ public abstract class BaseNotificationContent implements NotificationContent {
templateData.put("metricsMap", metricsMap);
}
- protected Map<String, Object> getTemplateData(AlertConfigDTO alertConfigDTO, Collection<AnomalyResult> anomalies) {
+ protected Map<String, Object> getTemplateData(DetectionAlertConfigDTO notificationConfig, Collection<AnomalyResult> anomalies) {
Map<String, Object> templateData = new HashMap<>();
DateTimeZone timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(DEFAULT_TIME_ZONE));
@@ -217,7 +217,7 @@ public abstract class BaseNotificationContent implements NotificationContent {
templateData.put("trueAlertCount", precisionRecallEvaluator.getTrueAnomalies());
templateData.put("falseAlertCount", precisionRecallEvaluator.getFalseAlarm());
templateData.put("newTrendCount", precisionRecallEvaluator.getTrueAnomalyNewTrend());
- templateData.put("alertConfigName", alertConfigDTO.getName());
+ templateData.put("alertConfigName", notificationConfig.getName());
templateData.put("includeSummary", includeSummary);
templateData.put("reportGenerationTimeMillis", System.currentTimeMillis());
if(precisionRecallEvaluator.getTotalResponses() > 0) {
@@ -225,8 +225,8 @@ public abstract class BaseNotificationContent implements NotificationContent {
templateData.put("recall", precisionRecallEvaluator.getRecall());
templateData.put("falseNegative", precisionRecallEvaluator.getFalseNegativeRate());
}
- if (alertConfigDTO.getReferenceLinks() != null) {
- templateData.put("referenceLinks", alertConfigDTO.getReferenceLinks());
+ if (notificationConfig.getReferenceLinks() != null) {
+ templateData.put("referenceLinks", notificationConfig.getReferenceLinks());
}
return templateData;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/EntityGroupKeyContent.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/EntityGroupKeyContent.java
index 278485b..c4fc289 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/EntityGroupKeyContent.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/EntityGroupKeyContent.java
@@ -96,7 +96,7 @@ public class EntityGroupKeyContent extends BaseNotificationContent {
@Override
public Map<String, Object> format(Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
- Map<String, Object> templateData = super.getTemplateData(context.getAlertConfig(), anomalies);
+ Map<String, Object> templateData = super.getTemplateData(context.getNotificationConfig(), anomalies);
DetectionConfigDTO config = null;
Preconditions.checkArgument(anomalies != null && !anomalies.isEmpty(), "Report has empty anomalies");
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/HierarchicalAnomaliesContent.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/HierarchicalAnomaliesContent.java
index e9bd63d..df8ccc1 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/HierarchicalAnomaliesContent.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/HierarchicalAnomaliesContent.java
@@ -88,7 +88,7 @@ public class HierarchicalAnomaliesContent extends BaseNotificationContent {
@Override
public Map<String, Object> format(Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
- Map<String, Object> templateData = super.getTemplateData(context.getAlertConfig(), anomalies);
+ Map<String, Object> templateData = super.getTemplateData(context.getNotificationConfig(), anomalies);
enrichMetricInfo(templateData, anomalies);
List<AnomalyReportEntity> rootAnomalyDetails = new ArrayList<>();
SortedMap<String, List<AnomalyReportEntity>> leafAnomalyDetails = new TreeMap<>();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/MetricAnomaliesContent.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/MetricAnomaliesContent.java
index 8f18abb..8f6d257 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/MetricAnomaliesContent.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/MetricAnomaliesContent.java
@@ -72,7 +72,7 @@ public class MetricAnomaliesContent extends BaseNotificationContent {
@Override
public Map<String, Object> format(Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
- Map<String, Object> templateData = super.getTemplateData(context.getAlertConfig(), anomalies);
+ Map<String, Object> templateData = super.getTemplateData(context.getNotificationConfig(), anomalies);
enrichMetricInfo(templateData, anomalies);
DateTime windowStart = DateTime.now();
@@ -118,10 +118,7 @@ public class MetricAnomaliesContent extends BaseNotificationContent {
String funcDescription = "";
Long id = -1L;
- if (anomaly.getFunction() != null){
- functionName = anomaly.getFunction().getFunctionName();
- id = anomaly.getFunction().getId();
- } else if ( anomaly.getDetectionConfigId() != null){
+ if ( anomaly.getDetectionConfigId() != null) {
DetectionConfigDTO config = this.configDAO.findById(anomaly.getDetectionConfigId());
Preconditions.checkNotNull(config, String.format("Cannot find detection config %d", anomaly.getDetectionConfigId()));
functionName = config.getName();
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/OnboardingNotificationContent.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/OnboardingNotificationContent.java
deleted file mode 100644
index 29b2872..0000000
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/content/templates/OnboardingNotificationContent.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.pinot.thirdeye.notification.content.templates;
-
-import org.apache.pinot.thirdeye.notification.formatter.ADContentFormatterContext;
-import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
-import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
-import org.apache.pinot.thirdeye.dashboard.resources.DetectionJobResource;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Properties;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.pinot.thirdeye.notification.content.BaseNotificationContent;
-import org.joda.time.Days;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-@Deprecated
-public class OnboardingNotificationContent extends BaseNotificationContent {
- private static final Logger LOG = LoggerFactory.getLogger(OnboardingNotificationContent.class);
-
- private static final String EMAIL_TEMPLATE = "emailTemplate";
- private static final String DEFAULT_TEMPLATE = "onboard-notification-email-template.ftl";
- private static final String DEFAULT_NULL_STRING_VALUE = "N/A";
- private static final String ALERT_FILTER_PATTERN_KEY = DetectionJobResource.AUTOTUNE_PATTERN_KEY;
- private static final int DEFAULT_ONBOARDING_REPLAY_DAYS = 30;
-
- public OnboardingNotificationContent() {}
-
- @Override
- public void init(Properties properties, ThirdEyeAnomalyConfiguration config) {
- super.init(properties, config);
- }
-
- @Override
- public String getTemplate() {
- return properties.getProperty(EMAIL_TEMPLATE, DEFAULT_TEMPLATE);
- }
-
- /**
- * The actual function that convert anomalies into parameter map
- */
- @Override
- public Map<String, Object> format(Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
- Map<String, Object> templateData = super.getTemplateData(context.getAlertConfig(), anomalies);
- enrichMetricInfo(templateData, anomalies);
- AnomalyFunctionDTO anomalyFunctionSpec = context.getAnomalyFunctionSpec();
- for (AnomalyResult anomalyResult : anomalies) {
- if (!(anomalyResult instanceof MergedAnomalyResultDTO)) {
- throw new IllegalArgumentException("Input anomalies should be instance of MergedAnomalyResultDTO");
- }
- if (anomalyFunctionSpec == null) {
- anomalyFunctionSpec = ((MergedAnomalyResultDTO) anomalyResult).getFunction();
- } else if (!anomalyFunctionSpec.getId().equals(((MergedAnomalyResultDTO) anomalyResult).getFunction().getId())) {
- throw new IllegalArgumentException("Input anomalies should be generated by the same anomaly function");
- }
- }
-
- // calculate times in between
- int onboardingReplayDays = DEFAULT_ONBOARDING_REPLAY_DAYS;
- if (context.getStart() != null && context.getEnd() != null) {
- onboardingReplayDays = Days.daysBetween(context.getStart(), context.getEnd()).getDays();
- }
-
- templateData.put("functionName", anomalyFunctionSpec.getFunctionName());
- templateData.put("functionId", Long.toString(anomalyFunctionSpec.getId()));
- templateData.put("metrics", anomalyFunctionSpec.getMetric());
- templateData.put("filters", returnValueOrDefault(anomalyFunctionSpec.getFilters(), DEFAULT_NULL_STRING_VALUE));
- templateData.put("dimensionDrillDown", returnValueOrDefault(anomalyFunctionSpec.getExploreDimensions(), DEFAULT_NULL_STRING_VALUE));
- templateData.put("repalyDays", Integer.toString(onboardingReplayDays));
- String alertPattern = DEFAULT_NULL_STRING_VALUE;
- Map<String, String> alertFilter = anomalyFunctionSpec.getAlertFilter();
- if (alertFilter != null && alertFilter.containsKey(ALERT_FILTER_PATTERN_KEY)) {
- alertPattern = alertFilter.get(ALERT_FILTER_PATTERN_KEY);
- }
- templateData.put("alertPattern", alertPattern);
-
- AlertConfigDTO alertConfig = context.getAlertConfig();
- if (alertConfig == null) {
- alertConfig = new AlertConfigDTO();
- }
- templateData.put("application", returnValueOrDefault(alertConfig.getApplication(), DEFAULT_NULL_STRING_VALUE));
- templateData.put("recipients", returnValueOrDefault(StringUtils.join(alertConfig.getReceiverAddresses().getTo(), ','), DEFAULT_NULL_STRING_VALUE));
- templateData.put("ccRecipients", returnValueOrDefault(StringUtils.join(alertConfig.getReceiverAddresses().getCc(), ','), DEFAULT_NULL_STRING_VALUE));
- templateData.put("bccRecipients", returnValueOrDefault(StringUtils.join(alertConfig.getReceiverAddresses().getBcc(), ','), DEFAULT_NULL_STRING_VALUE));
-
- return templateData;
- }
-
- private String returnValueOrDefault(String value, String defaultValue) {
- return StringUtils.isEmpty(value) ? defaultValue: value;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/ADContentFormatterContext.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/ADContentFormatterContext.java
index 77a35f7..a470b43 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/ADContentFormatterContext.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/ADContentFormatterContext.java
@@ -19,8 +19,8 @@
package org.apache.pinot.thirdeye.notification.formatter;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.joda.time.DateTime;
@@ -29,25 +29,25 @@ import org.joda.time.DateTime;
* which can be rendered into the alert(email, jira) content.
*/
public class ADContentFormatterContext {
- private AnomalyFunctionDTO anomalyFunctionSpec;
- private AlertConfigDTO alertConfig;
+ private DetectionConfigDTO detectionConfig;
+ private DetectionAlertConfigDTO notificationConfig;
private DateTime start; // anomaly search region starts
private DateTime end; // anomaly search region ends
- public AnomalyFunctionDTO getAnomalyFunctionSpec() {
- return anomalyFunctionSpec;
+ public DetectionConfigDTO getDetectionConfig() {
+ return detectionConfig;
}
- public void setAnomalyFunctionSpec(AnomalyFunctionDTO anomalyFunctionSpec) {
- this.anomalyFunctionSpec = anomalyFunctionSpec;
+ public void setDetectionConfig(DetectionConfigDTO detectionConfig) {
+ this.detectionConfig = detectionConfig;
}
- public AlertConfigDTO getAlertConfig() {
- return alertConfig;
+ public DetectionAlertConfigDTO getNotificationConfig() {
+ return notificationConfig;
}
- public void setAlertConfig(AlertConfigDTO alertConfig) {
- this.alertConfig = alertConfig;
+ public void setNotificationConfig(DetectionAlertConfigDTO notificationConfig) {
+ this.notificationConfig = notificationConfig;
}
public DateTime getStart() {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/EmailContentFormatter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/EmailContentFormatter.java
index 42aaba8..dba313b 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/EmailContentFormatter.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/notification/formatter/channels/EmailContentFormatter.java
@@ -70,12 +70,13 @@ public class EmailContentFormatter {
content.init(emailProps, teConfig);
}
- public EmailEntity getEmailEntity(DetectionAlertFilterRecipients recipients, String subject, Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
+ public EmailEntity getEmailEntity(DetectionAlertFilterRecipients recipients, String subject,
+ Collection<AnomalyResult> anomalies, ADContentFormatterContext context) {
Map<String, Object> templateData = notificationContent.format(anomalies, context);
templateData.put("dashboardHost", thirdEyeAnomalyConfig.getDashboardHost());
- String outputSubject = notificationContent.makeSubject(subject, context.getAlertConfig().getSubjectType(), templateData);
- return buildEmailEntity(templateData, outputSubject, recipients, context.getAlertConfig().getFromAddress(), notificationContent.getTemplate());
+ String outputSubject = notificationContent.makeSubject(subject, context.getNotificationConfig().getSubjectType(), templateData);
+ return buildEmailEntity(templateData, outputSubject, recipients, context.getNotificationConfig().getFrom(), notificationContent.getTemplate());
}
/**
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/feed/TestUnionAnomalyFeed.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/feed/TestUnionAnomalyFeed.java
deleted file mode 100644
index d96af77..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/feed/TestUnionAnomalyFeed.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.alert.feed;
-
-import org.apache.pinot.thirdeye.alert.commons.AnomalyFeedConfig;
-import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
-import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertSnapshotManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertSnapshotDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detector.email.filter.AlertFilterFactory;
-import java.util.Collection;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class TestUnionAnomalyFeed {
- private DAOTestBase testDAOProvider;
- private AlertFilterFactory alertFilterFactory;
- private AlertSnapshotManager alertSnapshotDAO;
- private AnomalyFeedConfig anomalyFeedConfig;
- private MergedAnomalyResultManager mergedAnomalyResultDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
-
- private static String TEST = "test";
- private long alertSnapshotId;
- @BeforeClass
- void beforeClass() {
- testDAOProvider = DAOTestBase.getInstance();
- String mappingsPath = ClassLoader.getSystemResource("sample-alertfilter.properties").getPath();
- alertFilterFactory = new AlertFilterFactory(mappingsPath);
- DAORegistry daoRegistry = DAORegistry.getInstance();
- alertSnapshotDAO = daoRegistry.getAlertSnapshotDAO();
- mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
- init();
- }
-
- @AfterClass(alwaysRun = true)
- void afterClass() {
- testDAOProvider.cleanup();
- }
-
- private void init() {
- AlertSnapshotDTO alertSnapshotDTO = DaoTestUtils.getTestAlertSnapshot();
- alertSnapshotId = alertSnapshotDAO.save(alertSnapshotDTO);
-
- anomalyFeedConfig = DaoTestUtils.getTestAnomalyFeedConfig();
- anomalyFeedConfig.setAlertSnapshotId(alertSnapshotId);
-
-
- AnomalyFunctionDTO anomalyFunction = DaoTestUtils.getTestFunctionSpec(TEST, TEST);
- anomalyFunction.setFilters("dimension=test;");
- long functionId = anomalyFunctionDAO.save(anomalyFunction);
-
- // Add mock anomalies
- MergedAnomalyResultDTO anomaly = DaoTestUtils.getTestMergedAnomalyResult(1l, 12l, TEST, TEST,
- -0.1, functionId, 1l);
- mergedAnomalyResultDAO.save(anomaly);
-
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(6l, 14l, TEST, TEST,-0.2, functionId, 5l);
- DimensionMap dimensions = new DimensionMap();
- dimensions.put("dimension", "test2");
- anomaly.setDimensions(dimensions);
- mergedAnomalyResultDAO.save(anomaly);
-
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(3l, 9l, TEST, TEST,-0.2, functionId, 3l);
- mergedAnomalyResultDAO.save(anomaly);
- }
-
- @Test
- public void testAnomalyFeed() {
- AnomalyFeed anomalyFeed = new UnionAnomalyFeed();
- anomalyFeed.init(alertFilterFactory, anomalyFeedConfig);
-
- Collection<MergedAnomalyResultDTO> mergedAnomalyResults = anomalyFeed.getAnomalyFeed();
- Assert.assertEquals(mergedAnomalyResults.size(), 2);
-
- anomalyFeed.updateSnapshot(mergedAnomalyResults);
- AlertSnapshotDTO alertSnapshotDTO = alertSnapshotDAO.findById(alertSnapshotId);
- Assert.assertEquals(alertSnapshotDTO.getSnapshot().size(), 2);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestContinuumAnomalyFetcher.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestContinuumAnomalyFetcher.java
deleted file mode 100644
index 3d0a87d..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestContinuumAnomalyFetcher.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.alert.fetcher;
-
-import org.apache.pinot.thirdeye.alert.commons.AnomalyFetcherConfig;
-import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertSnapshotDTO;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datalayer.util.ThirdEyeStringUtils;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.util.Collection;
-import java.util.Properties;
-import org.joda.time.DateTime;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class TestContinuumAnomalyFetcher {
- private static final String TEST = "test";
- private MergedAnomalyResultManager mergedAnomalyResultDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private DAOTestBase testDAOProvider;
- @BeforeClass
- public void beforeClass(){
- testDAOProvider = DAOTestBase.getInstance();
- DAORegistry daoRegistry = DAORegistry.getInstance();
- mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
-
- AnomalyFunctionDTO anomalyFunction = DaoTestUtils.getTestFunctionSpec(TEST, TEST);
- anomalyFunction.setFilters("dimension=test;");
- long functionId = anomalyFunctionDAO.save(anomalyFunction);
-
- // Add mock anomalies
- MergedAnomalyResultDTO anomaly = DaoTestUtils.getTestMergedAnomalyResult(1l, 12l, TEST, TEST,
- -0.1, functionId, 1l);
- mergedAnomalyResultDAO.save(anomaly);
-
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(3l, 14l, TEST, TEST,-0.2, functionId, 3l);
- mergedAnomalyResultDAO.save(anomaly);
-
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(3l, 9l, TEST, TEST,-0.2, functionId, 3l);
- mergedAnomalyResultDAO.save(anomaly);
- }
-
- @AfterClass(alwaysRun = true)
- void afterClass() {
- testDAOProvider.cleanup();
- }
-
- @Test
- public void testGetAlertCandidates(){
- AlertSnapshotDTO alertSnapshot = DaoTestUtils.getTestAlertSnapshot();
- AnomalyFetcherConfig anomalyFetcherConfig = DaoTestUtils.getTestAnomalyFetcherConfig();
- Properties properties = ThirdEyeStringUtils.decodeCompactedProperties(anomalyFetcherConfig.getProperties());
- properties.put(ContinuumAnomalyFetcher.REALERT_FREQUENCY, "5_MILLISECONDS");
- anomalyFetcherConfig.setProperties(ThirdEyeStringUtils.encodeCompactedProperties(properties));
-
- AnomalyFetcher anomalyFetcher = new ContinuumAnomalyFetcher();
- anomalyFetcher.init(anomalyFetcherConfig);
- Collection<MergedAnomalyResultDTO>
- alertCandidates = anomalyFetcher.getAlertCandidates(new DateTime(15l), alertSnapshot);
- Assert.assertEquals(alertCandidates.size(), 2);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestUnnotifiedAnomalyFetcher.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestUnnotifiedAnomalyFetcher.java
deleted file mode 100644
index f1d26ec..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/alert/fetcher/TestUnnotifiedAnomalyFetcher.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.alert.fetcher;
-
-import org.apache.pinot.thirdeye.alert.commons.AnomalyFetcherConfig;
-import org.apache.pinot.thirdeye.alert.commons.AnomalySource;
-import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertSnapshotDTO;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.util.Collection;
-import java.util.Properties;
-import org.joda.time.DateTime;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class TestUnnotifiedAnomalyFetcher {
- private static final String TEST = "test";
- private MergedAnomalyResultManager mergedAnomalyResultDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private DAOTestBase testDAOProvider;
- @BeforeClass
- public void beforeClass(){
- testDAOProvider = DAOTestBase.getInstance();
- DAORegistry daoRegistry = DAORegistry.getInstance();
- mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
-
-
- AnomalyFunctionDTO anomalyFunction = DaoTestUtils.getTestFunctionSpec(TEST, TEST);
- anomalyFunction.setFilters("dimension=test;");
- long functionId = anomalyFunctionDAO.save(anomalyFunction);
-
- // Add mock anomalies
- MergedAnomalyResultDTO anomaly = DaoTestUtils.getTestMergedAnomalyResult(1l, 2l, TEST, TEST,
- -0.1, functionId, 1l);
- mergedAnomalyResultDAO.save(anomaly);
-
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(3l, 4l, TEST, TEST,-0.2, functionId,
- 3l);
- mergedAnomalyResultDAO.save(anomaly);
- }
-
- @AfterClass(alwaysRun = true)
- void afterClass() {
- testDAOProvider.cleanup();
- }
-
- @Test
- public void testGetAlertCandidates(){
- AlertSnapshotDTO alertSnapshot = DaoTestUtils.getTestAlertSnapshot();
- AnomalyFetcherConfig anomalyFetcherConfig = DaoTestUtils.getTestAnomalyFetcherConfig();
-
- AnomalyFetcher anomalyFetcher = new UnnotifiedAnomalyFetcher();
- anomalyFetcher.init(anomalyFetcherConfig);
- Collection<MergedAnomalyResultDTO>
- alertCandidates = anomalyFetcher.getAlertCandidates(new DateTime(2l), alertSnapshot);
- Assert.assertEquals(alertCandidates.size(), 1);
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnBoardJobRunnerTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnBoardJobRunnerTest.java
deleted file mode 100644
index 31dcd43..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnBoardJobRunnerTest.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.BaseDetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnBoardJobRunner;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJob;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobStatus;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTaskStatus;
-import org.apache.pinot.thirdeye.anomaly.task.TaskConstants;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.MapConfiguration;
-import org.apache.commons.configuration2.StrictConfigurationComparator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-
-public class DetectionOnBoardJobRunnerTest {
- private static final Logger LOG = LoggerFactory.getLogger(DetectionOnBoardJobRunnerTest.class);
-
- private final static int timeOutSize = 1;
- private final static TimeUnit timeOutUnit = TimeUnit.SECONDS;
-
- @Test
- public void testNormalRun() {
- final int jobId = 1;
- final String jobName = "normalJob";
-
- DetectionOnboardJob onboardJob = new NormalDetectionOnboardJob(jobName, Collections.<String, String>emptyMap());
- List<DetectionOnboardTask> tasks = onboardJob.getTasks();
- Configuration configuration = onboardJob.getTaskConfiguration();
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
-
- DetectionOnboardJobStatus jobStatus =
- new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
-
- // Submit the job to executor
- DetectionOnBoardJobRunner jobRunner = new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus);
- jobRunner.run();
- Assert.assertEquals(jobStatus.getJobStatus(), JobConstants.JobStatus.COMPLETED);
- Assert.assertEquals(jobStatus.getTaskStatuses().size(), 2);
- for (DetectionOnboardTaskStatus taskStatus : jobStatus.getTaskStatuses()) {
- Assert.assertEquals(taskStatus.getTaskStatus(), TaskConstants.TaskStatus.COMPLETED);
- }
-
- // Check execution context
- DetectionOnboardExecutionContext executionContext = jobContext.getExecutionContext();
- String task1Result = (String) executionContext.getExecutionResult(NormalDetectionOnboardJob.TASK1_NAME);
- Assert.assertEquals(task1Result, NormalDetectionOnboardJob.TASK1_NAME + NormalDetectionOnboardTask.VALUE_POSTFIX);
-
- String task2Result = (String) executionContext.getExecutionResult(NormalDetectionOnboardJob.TASK2_NAME);
- Assert.assertEquals(task2Result, NormalDetectionOnboardJob.TASK2_NAME + NormalDetectionOnboardTask.VALUE_POSTFIX);
- }
-
- @Test
- public void testTaskConfig() {
- final int jobId = 1;
- final String jobName = "normalJob";
-
- Map<String, String> properties = new HashMap<>();
- properties.put("task1.property1", "value11");
- properties.put("task1.property2", "value12");
- properties.put("task2.property1", "value21");
-
- DetectionOnboardJob onboardJob = new LogConfigDetectionOnboardJob(jobName, properties);
- List<DetectionOnboardTask> tasks = onboardJob.getTasks();
- Configuration configuration = onboardJob.getTaskConfiguration();
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
-
- DetectionOnboardJobStatus jobStatus =
- new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
-
- // Submit the job to executor
- DetectionOnBoardJobRunner jobRunner = new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus);
- jobRunner.run();
- Assert.assertEquals(jobStatus.getJobStatus(), JobConstants.JobStatus.COMPLETED);
-
- // Check execution context
- DetectionOnboardExecutionContext executionContext = jobContext.getExecutionContext();
- {
- Configuration task1Result =
- (Configuration) executionContext.getExecutionResult(LogConfigDetectionOnboardJob.TASK1_NAME);
- Map<String, String> expectedTask1Property = new HashMap<>();
- expectedTask1Property.put("property1", "value11");
- expectedTask1Property.put("property2", "value12");
- Configuration expectedTask1Config = new MapConfiguration(expectedTask1Property);
- Assert.assertTrue(new StrictConfigurationComparator().compare(task1Result, expectedTask1Config));
- }
- {
- Configuration task2Result =
- (Configuration) executionContext.getExecutionResult(LogConfigDetectionOnboardJob.TASK2_NAME);
- Map<String, String> expectedTask2Property = new HashMap<>();
- expectedTask2Property.put("property1", "value21");
- Configuration expectedTask1Config = new MapConfiguration(expectedTask2Property);
- Assert.assertTrue(new StrictConfigurationComparator().compare(task2Result, expectedTask1Config));
- }
- }
-
- @Test
- public void testAbortAtTimeOut() {
- final int jobId = 1;
- final String jobName = "abortAtTimeOutJob";
-
- DetectionOnboardJob onboardJob = new TimeOutDetectionOnboardJob(jobName, Collections.<String, String>emptyMap());
- List<DetectionOnboardTask> tasks = onboardJob.getTasks();
- Configuration configuration = onboardJob.getTaskConfiguration();
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
-
- DetectionOnboardJobStatus jobStatus =
- new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
-
- // Submit the job to executor
- DetectionOnBoardJobRunner jobRunner =
- new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus, timeOutSize, timeOutUnit);
- jobRunner.run();
-
- Assert.assertEquals(jobStatus.getJobStatus(), JobConstants.JobStatus.FAILED);
- // There should be 1 task status because the second task will not be executed.
- Assert.assertEquals(jobStatus.getTaskStatuses().size(), 1);
- for (DetectionOnboardTaskStatus taskStatus : jobStatus.getTaskStatuses()) {
- Assert.assertEquals(taskStatus.getTaskStatus(), TaskConstants.TaskStatus.TIMEOUT);
- }
- }
-
- @Test
- public void testAbortOnFailure() {
- final int jobId = 1;
- final String jobName = "abortOnFailureJob";
-
- DetectionOnboardJob onboardJob = new HasFailureDetectionOnboardJob(jobName, Collections.<String, String>emptyMap());
- List<DetectionOnboardTask> tasks = onboardJob.getTasks();
- Configuration configuration = onboardJob.getTaskConfiguration();
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
-
- DetectionOnboardJobStatus jobStatus =
- new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
-
- // Submit the job to executor
- DetectionOnBoardJobRunner jobRunner =
- new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus, timeOutSize, timeOutUnit);
- jobRunner.run();
-
- Assert.assertEquals(jobStatus.getJobStatus(), JobConstants.JobStatus.FAILED);
- // There should be 1 task status because the second task will not be executed.
- Assert.assertEquals(jobStatus.getTaskStatuses().size(), 1);
- for (DetectionOnboardTaskStatus taskStatus : jobStatus.getTaskStatuses()) {
- Assert.assertEquals(taskStatus.getTaskStatus(), TaskConstants.TaskStatus.FAILED);
- }
- }
-
- @Test
- public void testContinueOnFailure() {
- final int jobId = 1;
- final String jobName = "continueOnFailureJob";
-
- Map<String, String> properties = new HashMap<>();
- properties.put("faultyTask." + DetectionOnBoardJobRunner.ABORT_ON_FAILURE, "false");
-
- DetectionOnboardJob onboardJob = new HasFailureDetectionOnboardJob(jobName, properties);
- List<DetectionOnboardTask> tasks = onboardJob.getTasks();
- Configuration configuration = onboardJob.getTaskConfiguration();
- DetectionOnboardJobContext jobContext = new DetectionOnboardJobContext(jobId, jobName, configuration);
-
- DetectionOnboardJobStatus jobStatus =
- new DetectionOnboardJobStatus(jobId, jobName, JobConstants.JobStatus.SCHEDULED, "");
-
- // Submit the job to executor
- DetectionOnBoardJobRunner jobRunner =
- new DetectionOnBoardJobRunner(jobContext, tasks, jobStatus, timeOutSize, timeOutUnit);
- jobRunner.run();
-
- Assert.assertEquals(jobStatus.getJobStatus(), JobConstants.JobStatus.COMPLETED);
- List<DetectionOnboardTaskStatus> taskStatuses = jobStatus.getTaskStatuses();
- Assert.assertEquals(taskStatuses.size(), 2);
- Assert.assertEquals(taskStatuses.get(0).getTaskStatus(), TaskConstants.TaskStatus.FAILED);
- Assert.assertNotNull(taskStatuses.get(0).getMessage());
- Assert.assertTrue(!taskStatuses.get(0).getMessage().isEmpty());
- Assert.assertEquals(taskStatuses.get(1).getTaskStatus(), TaskConstants.TaskStatus.COMPLETED);
- }
-
- static class TimeOutDetectionOnboardJob extends BaseDetectionOnboardJob {
- public TimeOutDetectionOnboardJob(String jobName, Map<String, String> properties) {
- super(jobName, properties);
- }
-
- @Override
- public Configuration getTaskConfiguration() {
- return new MapConfiguration(properties);
- }
-
- @Override
- public List<DetectionOnboardTask> getTasks() {
- List<DetectionOnboardTask> taskList = new ArrayList<>();
- taskList.add(new TimeOutDetectionOnboardTask("timeOutTask"));
- taskList.add(new NormalDetectionOnboardTask("someNormalTask"));
- return taskList;
- }
- }
-
- static class NormalDetectionOnboardJob extends BaseDetectionOnboardJob {
- static final String TASK1_NAME = "task1";
- static final String TASK2_NAME = "task2";
-
- public NormalDetectionOnboardJob(String jobName, Map<String, String> properties) {
- super(jobName, properties);
- }
-
- @Override
- public Configuration getTaskConfiguration() {
- return new MapConfiguration(properties);
- }
-
- @Override
- public List<DetectionOnboardTask> getTasks() {
- List<DetectionOnboardTask> taskList = new ArrayList<>();
- taskList.add(new NormalDetectionOnboardTask(TASK1_NAME));
- taskList.add(new NormalDetectionOnboardTask(TASK2_NAME));
- return taskList;
- }
- }
-
- static class NormalDetectionOnboardTask extends BaseDetectionOnboardTask {
- static final String VALUE_POSTFIX = "Value";
-
- public NormalDetectionOnboardTask(String taskName) {
- super(taskName);
- }
-
- @Override
- public void run() {
- String taskName = getTaskName();
- DetectionOnboardExecutionContext executionContext = this.getTaskContext().getExecutionContext();
- executionContext.setExecutionResult(taskName, taskName + VALUE_POSTFIX);
- }
- }
-
- static class LogConfigDetectionOnboardJob extends BaseDetectionOnboardJob {
- static final String TASK1_NAME = "task1";
- static final String TASK2_NAME = "task2";
-
- public LogConfigDetectionOnboardJob(String jobName, Map<String, String> properties) {
- super(jobName, properties);
- }
-
- @Override
- public Configuration getTaskConfiguration() {
- return new MapConfiguration(properties);
- }
-
- @Override
- public List<DetectionOnboardTask> getTasks() {
- List<DetectionOnboardTask> taskList = new ArrayList<>();
- taskList.add(new LogConfigDetectionOnboardTask(TASK1_NAME));
- taskList.add(new LogConfigDetectionOnboardTask(TASK2_NAME));
- return taskList;
- }
- }
-
- static class LogConfigDetectionOnboardTask extends BaseDetectionOnboardTask {
- public LogConfigDetectionOnboardTask(String taskName) {
- super(taskName);
- }
-
- @Override
- public void run() {
- Configuration configuration = getTaskContext().getConfiguration();
- getTaskContext().getExecutionContext().setExecutionResult(getTaskName(), configuration);
- }
- }
-
- static class TimeOutDetectionOnboardTask extends BaseDetectionOnboardTask {
- public TimeOutDetectionOnboardTask(String taskName) {
- super(taskName);
- }
-
- @Override
- public void run() {
- try {
- Thread.sleep(timeOutUnit.toMillis(timeOutSize + 2));
- } catch (InterruptedException e) {
- // Do nothing
- }
- }
- }
-
- static class HasFailureDetectionOnboardJob extends BaseDetectionOnboardJob {
- public HasFailureDetectionOnboardJob(String jobName, Map<String, String> properties) {
- super(jobName, properties);
- }
-
- @Override
- public Configuration getTaskConfiguration() {
- return new MapConfiguration(properties);
- }
-
- @Override
- public List<DetectionOnboardTask> getTasks() {
- List<DetectionOnboardTask> taskList = new ArrayList<>();
- taskList.add(new AlwaysFailDetectionOnboardTask("faultyTask"));
- taskList.add(new NormalDetectionOnboardTask("someNormalTask"));
- return taskList;
- }
- }
-
- static class AlwaysFailDetectionOnboardTask extends BaseDetectionOnboardTask {
- public AlwaysFailDetectionOnboardTask(String taskName) {
- super(taskName);
- }
-
- @Override
- public void run() {
- LOG.info("Triggering NullPointerException for TESTING purpose.");
- List<String> nullList = null;
- nullList.get(100);
- }
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResourceTest.java
deleted file mode 100644
index 9596873..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/DetectionOnboardResourceTest.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.apache.pinot.thirdeye.anomaly.job.JobConstants;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardJobStatus;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class DetectionOnboardResourceTest {
- private static final Logger LOG = LoggerFactory.getLogger(DetectionOnboardResourceTest.class);
- public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
-
- private DetectionOnboardResource detectionOnboardResource;
- private DAOTestBase daoTestBase;
-
- @BeforeClass
- public void initResource() {
- daoTestBase = DAOTestBase.getInstance();
- DAORegistry daoRegistry = DAORegistry.getInstance();
- detectionOnboardResource = new DetectionOnboardResource(daoRegistry.getTaskDAO(), daoRegistry.getAnomalyFunctionDAO());
- }
-
- @AfterClass
- public void shutdownResource() {
- daoTestBase.cleanup();
- }
-
- @Test
- public void testCreateJob() throws Exception {
- Map<String, String> properties = OnboardingTaskTestUtils.getJobProperties();
-
- String propertiesJson = OBJECT_MAPPER.writeValueAsString(properties);
- String normalJobStatusJson = detectionOnboardResource.createDetectionOnboardingJob("NormalJob", propertiesJson);
-
- DetectionOnboardJobStatus onboardJobStatus =
- OBJECT_MAPPER.readValue(normalJobStatusJson, DetectionOnboardJobStatus.class);
- JobConstants.JobStatus jobStatus = onboardJobStatus.getJobStatus();
- Assert.assertTrue(
- JobConstants.JobStatus.COMPLETED.equals(jobStatus) || JobConstants.JobStatus.SCHEDULED.equals(jobStatus));
- long jobId = onboardJobStatus.getJobId();
-
- DetectionOnboardJobStatus onboardJobStatusGet =
- OBJECT_MAPPER.readValue(detectionOnboardResource.getDetectionOnboardingJobStatus(jobId), DetectionOnboardJobStatus.class);
- JobConstants.JobStatus jobStatusGet = onboardJobStatusGet.getJobStatus();
- LOG.info("Job Status: {}", jobStatusGet);
- Assert.assertTrue(
- JobConstants.JobStatus.COMPLETED.equals(jobStatusGet) || JobConstants.JobStatus.SCHEDULED.equals(jobStatusGet));
- }
-
- @Test(dependsOnMethods = "testCreateJob")
- public void testFailedJobCreation() throws Exception {
- // Trigger error of duplicate job names
- Map<String, String> properties = Collections.emptyMap();
-
- String propertiesJson = OBJECT_MAPPER.writeValueAsString(properties);
- String normalJobStatusJson = detectionOnboardResource.createDetectionOnboardingJob("NormalJob", propertiesJson);
- DetectionOnboardJobStatus onboardJobStatus =
- OBJECT_MAPPER.readValue(normalJobStatusJson, DetectionOnboardJobStatus.class);
- Assert.assertEquals(onboardJobStatus.getJobStatus(), JobConstants.JobStatus.FAILED);
- Assert.assertNotNull(onboardJobStatus.getMessage());
- }
-
- @Test
- public void testNonExistingJobId() throws IOException {
- DetectionOnboardJobStatus onboardJobStatus = OBJECT_MAPPER
- .readValue(detectionOnboardResource.getDetectionOnboardingJobStatus(-1L), DetectionOnboardJobStatus.class);
- Assert.assertEquals(onboardJobStatus.getJobStatus(), JobConstants.JobStatus.UNKNOWN);
- Assert.assertNotNull(onboardJobStatus.getMessage());
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/OnboardingTaskTestUtils.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/OnboardingTaskTestUtils.java
deleted file mode 100644
index d7e06ec..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/OnboardingTaskTestUtils.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard;
-
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTaskContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.tasks.DefaultDetectionOnboardJob;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.commons.configuration2.Configuration;
-import org.apache.commons.configuration2.MapConfiguration;
-
-
-public class OnboardingTaskTestUtils {
- public static String TEST_COLLECTION = "test_dataset";
- public static String TEST_METRIC = "test_metric";
-
- /**
- * Generate a default job properties for all onboarding tests
- * @return a job properties
- */
- public static Map<String, String> getJobProperties(){
- Map<String, String> properties = new HashMap<>();
- properties.put(
- DefaultDetectionOnboardJob.FUNCTION_FACTORY_CONFIG_PATH, ClassLoader.getSystemResource("sample-functions.properties").getPath());
- properties.put(DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY_CONFIG_PATH, ClassLoader.getSystemResource("sample-alertfilter.properties").getPath());
- properties.put(DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY_CONFIG_PATH, ClassLoader.getSystemResource("sample-alertfilter-autotune.properties").getPath());
- properties.put(DefaultDetectionOnboardJob.FUNCTION_NAME, "Normal Function");
- properties.put(DefaultDetectionOnboardJob.ALERT_APPLICATION, "test");
- properties.put(DefaultDetectionOnboardJob.COLLECTION_NAME, TEST_COLLECTION);
- properties.put(DefaultDetectionOnboardJob.METRIC_NAME, TEST_METRIC);
- properties.put(DefaultDetectionOnboardJob.WINDOW_SIZE, "1");
- properties.put(DefaultDetectionOnboardJob.WINDOW_UNIT, "DAYS");
- properties.put(DefaultDetectionOnboardJob.CRON_EXPRESSION, "0 0 0 1/1 * ? *");
- properties.put(DefaultDetectionOnboardJob.FUNCTION_PROPERTIES, "metricTimezone=America/Los_Angeles;");
- properties.put(DefaultDetectionOnboardJob.ALERT_NAME, "Normal Alert");
- properties.put(DefaultDetectionOnboardJob.ALERT_TO, "test@test.com");
- properties.put(DefaultDetectionOnboardJob.SMTP_HOST, "test.com");
- properties.put(DefaultDetectionOnboardJob.SMTP_PORT, "25");
- properties.put(DefaultDetectionOnboardJob.DEFAULT_ALERT_RECEIVER_ADDRESS, "test@test.com");
- properties.put(DefaultDetectionOnboardJob.DEFAULT_ALERT_SENDER_ADDRESS, "test@test.com");
- properties.put(DefaultDetectionOnboardJob.THIRDEYE_DASHBOARD_HOST, "test.com");
- properties.put(DefaultDetectionOnboardJob.PHANTON_JS_PATH, "/");
- properties.put(DefaultDetectionOnboardJob.ROOT_DIR, "/");
-
- return properties;
- }
-
- /**
- * Return a default task configuration for all onboarding tasks
- * @return a task context
- */
- public static DetectionOnboardTaskContext getDetectionTaskContext() {
- Configuration configuration = new MapConfiguration(getJobProperties());
- DetectionOnboardTaskContext detectionOnboardTaskContext = new DetectionOnboardTaskContext();
- detectionOnboardTaskContext.setConfiguration(configuration);
- return detectionOnboardTaskContext;
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/TestOnboardingTasks.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/TestOnboardingTasks.java
deleted file mode 100644
index 454e89f..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/anomaly/onboard/tasks/TestOnboardingTasks.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.anomaly.onboard.tasks;
-
-import org.apache.pinot.thirdeye.anomaly.onboard.OnboardingTaskTestUtils;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardExecutionContext;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTask;
-import org.apache.pinot.thirdeye.anomaly.onboard.framework.DetectionOnboardTaskContext;
-import org.apache.pinot.thirdeye.constant.MetricAggFunction;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.JobManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.util.concurrent.TimeUnit;
-import org.junit.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-
-public class TestOnboardingTasks {
- private DAOTestBase daoTestBase;
- private DetectionOnboardTaskContext context;
- private DatasetConfigManager datasetConfigDAO;
- private MetricConfigManager metricConfigDAO;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private AlertConfigManager alertConfigDAO;
- private JobManager jobDAO;
-
- @BeforeClass
- public void beforeClass(){
- daoTestBase = DAOTestBase.getInstance();
- DAORegistry daoRegistry = DAORegistry.getInstance();
- datasetConfigDAO = daoRegistry.getDatasetConfigDAO();
- metricConfigDAO = daoRegistry.getMetricConfigDAO();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
- alertConfigDAO = daoRegistry.getAlertConfigDAO();
- jobDAO = daoRegistry.getJobDAO();
- context = OnboardingTaskTestUtils.getDetectionTaskContext();
- initDataset();
- initMetric();
- }
-
- public void initDataset(){
- // Prepare for data
- DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
- datasetConfig.setDataset(OnboardingTaskTestUtils.TEST_COLLECTION);
- datasetConfig.setTimeColumn("Date");
- datasetConfig.setTimeUnit(TimeUnit.DAYS);
- datasetConfig.setTimeDuration(1);
- datasetConfig.setTimeFormat("SIMPLE_DATE_FORMAT:yyyyMMdd");
- datasetConfig.setTimezone("US/Pacific");
- datasetConfigDAO.save(datasetConfig);
- Assert.assertNotNull(datasetConfigDAO.findByDataset(OnboardingTaskTestUtils.TEST_COLLECTION));
- }
-
- public void initMetric(){
- // Prepare for data
- MetricConfigDTO metricConfigDTO = new MetricConfigDTO();
- metricConfigDTO.setDataset(OnboardingTaskTestUtils.TEST_COLLECTION);
- metricConfigDTO.setName(OnboardingTaskTestUtils.TEST_METRIC);
- metricConfigDTO.setAlias(OnboardingTaskTestUtils.TEST_COLLECTION + "::" + OnboardingTaskTestUtils.TEST_METRIC);
- metricConfigDTO.setActive(true);
- metricConfigDTO.setDefaultAggFunction(MetricAggFunction.SUM);
- metricConfigDAO.save(metricConfigDTO);
- Assert.assertNotNull(metricConfigDAO.findByMetricName(OnboardingTaskTestUtils.TEST_METRIC));
- }
-
- @AfterClass(alwaysRun = true)
- public void afterClass(){
- daoTestBase.cleanup();
- }
-
- @Test
- public void testOnboardingTasks() throws Exception{
- AnomalyFunctionDTO dummyFunction = new AnomalyFunctionDTO();
- dummyFunction.setFunctionName(context.getConfiguration().getString(DefaultDetectionOnboardJob.FUNCTION_NAME));
- dummyFunction.setMetricId(-1);
- dummyFunction.setIsActive(false);
- anomalyFunctionDAO.save(dummyFunction);
-
- DetectionOnboardTask task = new DataPreparationOnboardingTask();
- task.setTaskContext(context);
- task.run();
-
- DetectionOnboardExecutionContext executionContext = context.getExecutionContext();
- Assert.assertNotNull(executionContext.getExecutionResult(DefaultDetectionOnboardJob.FUNCTION_FACTORY));
- Assert.assertNotNull(executionContext.getExecutionResult(DefaultDetectionOnboardJob.ALERT_FILTER_FACTORY));
- Assert.assertNotNull(executionContext.getExecutionResult(DefaultDetectionOnboardJob.ALERT_FILTER_AUTOTUNE_FACTORY));
-
- task = new FunctionCreationOnboardingTask();
- task.setTaskContext(context);
- task.run();
-
- Assert.assertEquals(1, anomalyFunctionDAO.findAll().size());
- Assert.assertEquals(1, alertConfigDAO.findAll().size());
-
- FunctionReplayOnboardingTask DetectionOnboardTask = new FunctionReplayOnboardingTask();
- DetectionOnboardTask.setTaskContext(context);
- DetectionOnboardTask.initDetectionJob();
-
- Assert.assertEquals(1, jobDAO.findAll().size());
-
- task = new AlertFilterAutoTuneOnboardingTask();
- task.setTaskContext(context);
- task.run();
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/OnboardResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/OnboardResourceTest.java
deleted file mode 100644
index eda3917..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resource/OnboardResourceTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-package org.apache.pinot.thirdeye.dashboard.resource;
-
-import org.apache.pinot.thirdeye.dashboard.ThirdEyeDashboardConfiguration;
-import org.apache.pinot.thirdeye.dashboard.resources.OnboardResource;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import javax.ws.rs.core.Response;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-
-public class OnboardResourceTest {
-
- private DAOTestBase testDAOProvider;
- private DAORegistry daoRegistry;
-
- @BeforeMethod
- public void beforeClass() {
- // Prepare database
- testDAOProvider = DAOTestBase.getInstance();
- daoRegistry = DAORegistry.getInstance();
-
- DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
- datasetConfigDTO.setDataset("test_dataset");
- datasetConfigDTO.setOwners(new HashSet<String>(Arrays.asList("user_1", "user_2")));
- daoRegistry.getDatasetConfigDAO().save(datasetConfigDTO);
-
- MetricConfigDTO metricConfigDTO = new MetricConfigDTO();
- metricConfigDTO.setName("test_metric");
- metricConfigDTO.setDataset("test_dataset");
- metricConfigDTO.setAlias("test_alias");
- metricConfigDTO.setTags(new HashSet<String>(Arrays.asList("test_tag", "random_tag")));
- daoRegistry.getMetricConfigDAO().save(metricConfigDTO);
- }
-
- @AfterMethod(alwaysRun = true)
- void afterClass() {
- testDAOProvider.cleanup();
- }
-
- @Test
- public void testBulkOnboard() throws Exception {
- ThirdEyeDashboardConfiguration config = new ThirdEyeDashboardConfiguration();
- config.setFailureFromAddress("thirdeye@test");
- OnboardResource onboardResource = new OnboardResource(config);
- Response response = onboardResource.bulkOnboardAlert("test_tag", null, "test_prefix_", true, null, null, null, null);
-
- // Check if the alert group is automatically created
- List<AlertConfigDTO> alertConfigDTOList = this.daoRegistry.getAlertConfigDAO().findAll();
- Assert.assertEquals(alertConfigDTOList.size(), 1);
- Assert.assertEquals(alertConfigDTOList.get(0).getName(), "auto_onboard_dataset_testDataset_alert");
- Assert.assertEquals(alertConfigDTOList.get(0).getApplication(), "others");
- Assert.assertEquals(alertConfigDTOList.get(0).getCronExpression(), "0 0/5 * * * ? *");
-
- // Check if anomaly function is created
- List<AnomalyFunctionDTO> anomalyFunctionDTOList = this.daoRegistry.getAnomalyFunctionDAO().findAll();
- Assert.assertEquals(anomalyFunctionDTOList.size(), 1);
- Assert.assertEquals(anomalyFunctionDTOList.get(0).getFunctionName(), "test_prefix_testMetric_testDataset");
-
- // Check if alert group has subscribed to the function
- Assert.assertEquals(alertConfigDTOList.get(0).getEmailConfig().getFunctionIds().size(), 1);
- Assert.assertEquals(alertConfigDTOList.get(0).getEmailConfig().getFunctionIds().get(0), anomalyFunctionDTOList.get(0).getId());
-
- // Verify response
- Assert.assertEquals(response.getStatus(), 200);
- Assert.assertEquals(((Map<String, String>)response.getEntity()).get("metric test_metric"),
- "success! onboarded and added function id " + anomalyFunctionDTOList.get(0).getId()
- + " to subscription alertGroup = auto_onboard_dataset_testDataset_alert");
- Assert.assertEquals(((Map<String, String>)response.getEntity()).get("message"),
- "successfully onboarded 1 metrics with function ids [" + anomalyFunctionDTOList.get(0).getId() + "]");
- }
-
- @Test
- public void testBulkOnboardWithInvalidAlertGroup() throws Exception {
- OnboardResource onboardResource = new OnboardResource(null);
- Response response = onboardResource.bulkOnboardAlert("test_tag", null, null, true, "group_name", null, null, null);
- Assert.assertEquals(response.getStatus(), 400);
- Map<String, String> responseMap = (Map<String, String>) response.getEntity();
- Assert.assertEquals(responseMap.get("message"), "cannot find an alert group with name group_name.");
- }
-
- @Test
- public void testSoftBulkOnboardWithNoAlertGroup() throws Exception {
- OnboardResource onboardResource = new OnboardResource(null);
- Response response = onboardResource.bulkOnboardAlert("test_tag", null, null, false, null, null, null, null);
- Assert.assertEquals(response.getStatus(), 400);
- Map<String, String> responseMap = (Map<String, String>) response.getEntity();
- Assert.assertEquals(responseMap.get("message"), "cannot find an alert group for metric test_metric.");
- }
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
index 43d0fb5..53544e0 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
@@ -20,6 +20,8 @@ import com.google.common.collect.HashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
import java.io.IOException;
+import java.util.Collections;
+import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.apache.pinot.thirdeye.alert.commons.AnomalyFeedConfig;
import org.apache.pinot.thirdeye.alert.commons.AnomalyFetcherConfig;
@@ -72,6 +74,8 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.joda.time.DateTime;
+import static org.apache.pinot.thirdeye.detection.alert.StatefulDetectionAlertFilter.*;
+
public class DaoTestUtils {
@@ -138,6 +142,28 @@ public class DaoTestUtils {
return functionSpec;
}
+ public static DetectionAlertConfigDTO getTestNotificationConfig(String name) {
+ DetectionAlertConfigDTO notificationConfigDTO = new DetectionAlertConfigDTO();
+ notificationConfigDTO.setName(name);
+ notificationConfigDTO.setActive(true);
+ notificationConfigDTO.setApplication("test");
+ notificationConfigDTO.setFrom("te@linkedin.com");
+ notificationConfigDTO.setCronExpression("0/10 * * * * ?");
+
+ Map<String, Object> properties = new HashMap<>();
+ Map<String, Set<String>> recipients = new HashMap<>();
+ recipients.put(PROP_TO, Collections.singleton("anomaly-to@linedin.com"));
+ recipients.put(PROP_CC, Collections.singleton("anomaly-cc@linedin.com"));
+ recipients.put(PROP_BCC, Collections.singleton("anomaly-bcc@linedin.com"));
+ properties.put(PROP_RECIPIENTS, recipients);
+ notificationConfigDTO.setProperties(properties);
+
+ Map<Long, Long> vectorClocks = new HashMap<>();
+ notificationConfigDTO.setVectorClocks(vectorClocks);
+
+ return notificationConfigDTO;
+ }
+
public static AlertConfigDTO getTestAlertConfiguration(String name) {
AlertConfigDTO alertConfigDTO = new AlertConfigDTO();
alertConfigDTO.setName(name);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java
index 87d8ed7..ebba5b9 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/DataProviderTest.java
@@ -34,11 +34,13 @@ import org.apache.pinot.thirdeye.common.dimension.DimensionMap;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager;
import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.EventDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
@@ -60,7 +62,6 @@ import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
public class DataProviderTest {
- private static final double EPSILON_MEAN = 20.0;
private DAOTestBase testBase;
private EventManager eventDAO;
@@ -68,6 +69,7 @@ public class DataProviderTest {
private MetricConfigManager metricDAO;
private DatasetConfigManager datasetDAO;
private EvaluationManager evaluationDAO;
+ private DetectionConfigManager detectionDAO;
private QueryCache cache;
private TimeSeriesLoader timeseriesLoader;
@@ -79,6 +81,7 @@ public class DataProviderTest {
private List<Long> anomalyIds;
private List<Long> metricIds;
private List<Long> datasetIds;
+ private List<Long> detectionIds;
@BeforeMethod
public void beforeMethod() throws Exception {
@@ -90,6 +93,7 @@ public class DataProviderTest {
this.metricDAO = reg.getMetricConfigDAO();
this.datasetDAO = reg.getDatasetConfigDAO();
this.evaluationDAO = reg.getEvaluationManager();
+ this.detectionDAO = reg.getDetectionConfigManager();
// events
this.eventIds = new ArrayList<>();
this.eventIds.add(this.eventDAO.save(makeEvent(3600000L, 7200000L)));
@@ -98,12 +102,22 @@ public class DataProviderTest {
this.eventIds.add(this.eventDAO.save(makeEvent(604800000L, 1209600000L, Arrays.asList("b=2", "c=3"))));
this.eventIds.add(this.eventDAO.save(makeEvent(1209800000L, 1210600000L, Collections.singleton("b=4"))));
+ // detections
+ this.detectionIds = new ArrayList<>();
+ DetectionConfigDTO detectionConfig = new DetectionConfigDTO();
+ detectionConfig.setName("test_detection_1");
+ detectionConfig.setDescription("test_description_1");
+ this.detectionIds.add(this.detectionDAO.save(detectionConfig));
+ detectionConfig.setName("test_detection_2");
+ detectionConfig.setDescription("test_description_2");
+ this.detectionIds.add(this.detectionDAO.save(detectionConfig));
+
// anomalies
this.anomalyIds = new ArrayList<>();
- this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, 100L, 4000000L, 8000000L, Arrays.asList("a=1", "c=3", "b=2"))));
- this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, 100L, 8000000L, 12000000L, Arrays.asList("a=1", "c=4"))));
- this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, 200L, 604800000L, 1209600000L, Collections.<String>emptyList())));
- this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, 200L, 14400000L, 18000000L, Arrays.asList("a=1", "c=3"))));
+ this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, detectionIds.get(0), 4000000L, 8000000L, Arrays.asList("a=1", "c=3", "b=2"))));
+ this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, detectionIds.get(0), 8000000L, 12000000L, Arrays.asList("a=1", "c=4"))));
+ this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, detectionIds.get(1), 604800000L, 1209600000L, Collections.<String>emptyList())));
+ this.anomalyIds.add(this.anomalyDAO.save(makeAnomaly(null, detectionIds.get(1), 14400000L, 18000000L, Arrays.asList("a=1", "c=3"))));
// metrics
this.metricIds = new ArrayList<>();
@@ -251,7 +265,7 @@ public class DataProviderTest {
Collection<MergedAnomalyResultDTO> anomalies = this.provider.fetchAnomalies(Collections.singleton(slice), -1).get(slice);
Assert.assertEquals(anomalies.size(), 1);
- Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(2), 200L, 604800000L, 1209600000L, Collections.<String>emptyList())));
+ Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(2), detectionIds.get(1), 604800000L, 1209600000L, Collections.<String>emptyList())));
}
@Test
@@ -261,9 +275,9 @@ public class DataProviderTest {
Collection<MergedAnomalyResultDTO> anomalies = this.provider.fetchAnomalies(Collections.singleton(slice), -1).get(slice);
Assert.assertEquals(anomalies.size(), 3);
- Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(0), 100L, 4000000L, 8000000L, Arrays.asList("a=1", "c=3", "b=2"))));
- Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(2), 200L, 604800000L, 1209600000L, Collections.<String>emptyList())));
- Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(3), 200L, 14400000L, 18000000L, Arrays.asList("a=1", "c=3"))));
+ Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(0), detectionIds.get(0), 4000000L, 8000000L, Arrays.asList("a=1", "c=3", "b=2"))));
+ Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(2), detectionIds.get(1), 604800000L, 1209600000L, Collections.<String>emptyList())));
+ Assert.assertTrue(anomalies.contains(makeAnomaly(this.anomalyIds.get(3), detectionIds.get(1), 14400000L, 18000000L, Arrays.asList("a=1", "c=3"))));
}
//
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestEntityGroupKeyContent.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestEntityGroupKeyContent.java
index 3bc96cd..de38796 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestEntityGroupKeyContent.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestEntityGroupKeyContent.java
@@ -161,7 +161,7 @@ public class TestEntityGroupKeyContent {
EmailContentFormatter
contentFormatter = new EmailContentFormatter(new EntityGroupKeyContent(), thirdeyeAnomalyConfig);
ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAlertConfig(DaoTestUtils.getTestAlertConfiguration("Test Config"));
+ context.setNotificationConfig(DaoTestUtils.getTestNotificationConfig("Test Config"));
EmailEntity emailEntity = contentFormatter.getEmailEntity(
new DetectionAlertFilterRecipients(EmailUtils.getValidEmailAddresses("a@b.com")), TEST, anomalies, context);
String htmlPath = ClassLoader.getSystemResource("test-entity-groupby-email-content-formatter.html").getPath();
@@ -279,7 +279,7 @@ public class TestEntityGroupKeyContent {
EmailContentFormatter
contentFormatter = new EmailContentFormatter(new EntityGroupKeyContent(), props, thirdeyeAnomalyConfig);
ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAlertConfig(DaoTestUtils.getTestAlertConfiguration("Test Config"));
+ context.setNotificationConfig(DaoTestUtils.getTestNotificationConfig("Test Config"));
EmailEntity emailEntity = contentFormatter.getEmailEntity(
new DetectionAlertFilterRecipients(EmailUtils.getValidEmailAddresses("a@b.com")), TEST, anomalies, context);
String htmlPath = ClassLoader.getSystemResource("test-entity-groupby-with-whitelist-email-content-formatter.html").getPath();
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestHierarchicalAnomaliesContent.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestHierarchicalAnomaliesContent.java
index fc46f65..e2ec1e7 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestHierarchicalAnomaliesContent.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestHierarchicalAnomaliesContent.java
@@ -16,6 +16,7 @@
package org.apache.pinot.thirdeye.notification.content.templates;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
import org.apache.pinot.thirdeye.notification.commons.EmailEntity;
import org.apache.pinot.thirdeye.notification.formatter.ADContentFormatterContext;
import org.apache.pinot.thirdeye.notification.formatter.channels.EmailContentFormatter;
@@ -146,12 +147,12 @@ public class TestHierarchicalAnomaliesContent {
anomalies.add(anomaly);
mergedAnomalyResultDAO.save(anomaly);
- AlertConfigDTO alertConfigDTO = DaoTestUtils.getTestAlertConfiguration("Test Config");
+ DetectionAlertConfigDTO notificationConfigDTO = DaoTestUtils.getTestNotificationConfig("Test Config");
EmailContentFormatter
contentFormatter = new EmailContentFormatter(new HierarchicalAnomaliesContent(), thirdeyeAnomalyConfig);
ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAlertConfig(alertConfigDTO);
+ context.setNotificationConfig(notificationConfigDTO);
DetectionAlertFilterRecipients recipients = new DetectionAlertFilterRecipients(
EmailUtils.getValidEmailAddresses("a@b.com"));
EmailEntity emailEntity = contentFormatter.getEmailEntity(recipients, TEST, anomalies, context);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestMetricAnomaliesContent.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestMetricAnomaliesContent.java
index bef4d0d..701d133 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestMetricAnomaliesContent.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestMetricAnomaliesContent.java
@@ -16,6 +16,20 @@
package org.apache.pinot.thirdeye.notification.content.templates;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.EvaluationManager;
+import org.apache.pinot.thirdeye.datalayer.bao.EventManager;
+import org.apache.pinot.thirdeye.datalayer.bao.TaskManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datasource.ThirdEyeCacheRegistry;
+import org.apache.pinot.thirdeye.datasource.loader.AggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultAggregationLoader;
+import org.apache.pinot.thirdeye.datasource.loader.DefaultTimeSeriesLoader;
+import org.apache.pinot.thirdeye.datasource.loader.TimeSeriesLoader;
+import org.apache.pinot.thirdeye.detection.DataProvider;
+import org.apache.pinot.thirdeye.detection.DefaultDataProvider;
+import org.apache.pinot.thirdeye.detection.DetectionPipelineLoader;
import org.apache.pinot.thirdeye.notification.commons.EmailEntity;
import org.apache.pinot.thirdeye.notification.formatter.ADContentFormatterContext;
import org.apache.pinot.thirdeye.notification.formatter.channels.EmailContentFormatter;
@@ -27,12 +41,9 @@ import org.apache.pinot.thirdeye.anomaly.utils.EmailUtils;
import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
@@ -50,24 +61,48 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import static org.apache.pinot.thirdeye.anomaly.SmtpConfiguration.*;
+import static org.apache.pinot.thirdeye.datalayer.DaoTestUtils.*;
public class TestMetricAnomaliesContent {
private static final String TEST = "test";
private int id = 0;
private String dashboardHost = "http://localhost:8080/dashboard";
+ private String detectionConfigFile = "/sample-detection-config.yml";
private DAOTestBase testDAOProvider;
- private AnomalyFunctionManager anomalyFunctionDAO;
+ private DetectionConfigManager detectionConfigDAO;
private MergedAnomalyResultManager mergedAnomalyResultDAO;
private MetricConfigManager metricDAO;
+ private DatasetConfigManager datasetDAO;
+ private EventManager eventDAO;
+ private MergedAnomalyResultManager anomalyDAO;
+ private TaskManager taskDAO;
+ private EvaluationManager evaluationDAO;
+ private DetectionPipelineLoader detectionPipelineLoader;
+ private DataProvider provider;
@BeforeClass
public void beforeClass(){
testDAOProvider = DAOTestBase.getInstance();
DAORegistry daoRegistry = DAORegistry.getInstance();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
+ detectionConfigDAO = daoRegistry.getDetectionConfigManager();
mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
metricDAO = daoRegistry.getMetricConfigDAO();
+ datasetDAO = daoRegistry.getDatasetConfigDAO();
+ eventDAO = daoRegistry.getEventDAO();
+ taskDAO = daoRegistry.getTaskDAO();
+ anomalyDAO = daoRegistry.getMergedAnomalyResultDAO();
+ evaluationDAO = daoRegistry.getEvaluationManager();
+ detectionPipelineLoader = new DetectionPipelineLoader();
+
+ TimeSeriesLoader timeseriesLoader =
+ new DefaultTimeSeriesLoader(daoRegistry.getMetricConfigDAO(), datasetDAO, null);
+ AggregationLoader aggregationLoader =
+ new DefaultAggregationLoader(metricDAO, datasetDAO, ThirdEyeCacheRegistry.getInstance().getQueryCache(),
+ ThirdEyeCacheRegistry.getInstance().getDatasetMaxDataTimeCache());
+
+ provider = new DefaultDataProvider(metricDAO, datasetDAO, eventDAO, anomalyDAO, evaluationDAO,
+ timeseriesLoader, aggregationLoader, detectionPipelineLoader);
}
@AfterClass(alwaysRun = true)
@@ -98,15 +133,18 @@ public class TestMetricAnomaliesContent {
alerters.put("smtpConfiguration", smtpProps);
thirdeyeAnomalyConfig.setAlerterConfiguration(alerters);
+ // create test dataset config
+ datasetDAO.save(getTestDatasetConfig("test-collection"));
+ metricDAO.save(getTestMetricConfig("test-collection", "cost", null));
List<AnomalyResult> anomalies = new ArrayList<>();
- AnomalyFunctionDTO anomalyFunction = DaoTestUtils.getTestFunctionSpec(TEST, TEST);
- anomalyFunctionDAO.save(anomalyFunction);
+ DetectionConfigDTO detectionConfigDTO = DaoTestUtils.getTestDetectionConfig(provider, detectionConfigFile);
+ detectionConfigDAO.save(detectionConfigDTO);
MergedAnomalyResultDTO anomaly = DaoTestUtils.getTestMergedAnomalyResult(
new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis(),
new DateTime(2017, 11, 6, 13, 0, dateTimeZone).getMillis(),
TEST, TEST, 0.1, 1l, new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis());
- anomaly.setFunction(anomalyFunction);
+ anomaly.setDetectionConfigId(detectionConfigDTO.getId());
anomaly.setAvgCurrentVal(1.1);
anomaly.setAvgBaselineVal(1.0);
mergedAnomalyResultDAO.save(anomaly);
@@ -115,7 +153,7 @@ public class TestMetricAnomaliesContent {
new DateTime(2017, 11, 7, 10, 0, dateTimeZone).getMillis(),
new DateTime(2017, 11, 7, 17, 0, dateTimeZone).getMillis(),
TEST, TEST, 0.1, 1l, new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis());
- anomaly.setFunction(anomalyFunction);
+ anomaly.setDetectionConfigId(detectionConfigDTO.getId());
anomaly.setAvgCurrentVal(0.9);
anomaly.setAvgBaselineVal(1.0);
mergedAnomalyResultDAO.save(anomaly);
@@ -127,12 +165,10 @@ public class TestMetricAnomaliesContent {
metric.setAlias(TEST + "::" + TEST);
metricDAO.save(metric);
- AlertConfigDTO alertConfigDTO = DaoTestUtils.getTestAlertConfiguration("Test Config");
-
EmailContentFormatter
contentFormatter = new EmailContentFormatter(new MetricAnomaliesContent(), thirdeyeAnomalyConfig);
ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAlertConfig(alertConfigDTO);
+ context.setNotificationConfig(DaoTestUtils.getTestNotificationConfig("Test Config"));
DetectionAlertFilterRecipients recipients = new DetectionAlertFilterRecipients(
EmailUtils.getValidEmailAddresses("a@b.com"));
EmailEntity emailEntity = contentFormatter.getEmailEntity(recipients, TEST, anomalies, context);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestOnboardingNotificationContent.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestOnboardingNotificationContent.java
deleted file mode 100644
index 13864ab..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/notification/content/templates/TestOnboardingNotificationContent.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.notification.content.templates;
-
-import org.apache.pinot.thirdeye.notification.commons.EmailEntity;
-import org.apache.pinot.thirdeye.notification.formatter.ADContentFormatterContext;
-import org.apache.pinot.thirdeye.notification.formatter.channels.EmailContentFormatter;
-import org.apache.pinot.thirdeye.notification.ContentFormatterUtils;
-import org.apache.pinot.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
-import org.apache.pinot.thirdeye.anomaly.monitor.MonitorConfiguration;
-import org.apache.pinot.thirdeye.anomaly.task.TaskDriverConfiguration;
-import org.apache.pinot.thirdeye.anomaly.utils.EmailUtils;
-import org.apache.pinot.thirdeye.anomalydetection.context.AnomalyResult;
-import org.apache.pinot.thirdeye.common.time.TimeGranularity;
-import org.apache.pinot.thirdeye.datalayer.DaoTestUtils;
-import org.apache.pinot.thirdeye.datalayer.bao.AlertConfigManager;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AlertConfigDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
-import org.apache.pinot.thirdeye.datasource.DAORegistry;
-import org.apache.pinot.thirdeye.detection.alert.DetectionAlertFilterRecipients;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-import static org.apache.pinot.thirdeye.anomaly.SmtpConfiguration.*;
-
-
-public class TestOnboardingNotificationContent {
- private static final String TEST = "test";
- private int id = 0;
- private String dashboardHost = "http://localhost:8080/dashboard";
- private DAOTestBase testDAOProvider;
- private AnomalyFunctionManager anomalyFunctionDAO;
- private MergedAnomalyResultManager mergedAnomalyResultDAO;
- private AlertConfigManager alertConfigDAO;
- @BeforeClass
- public void beforeClass(){
- testDAOProvider = DAOTestBase.getInstance();
- DAORegistry daoRegistry = DAORegistry.getInstance();
- anomalyFunctionDAO = daoRegistry.getAnomalyFunctionDAO();
- mergedAnomalyResultDAO = daoRegistry.getMergedAnomalyResultDAO();
- alertConfigDAO = daoRegistry.getAlertConfigDAO();
- }
-
- @AfterClass(alwaysRun = true)
- void afterClass() {
- testDAOProvider.cleanup();
- }
- @Test
- public void testGetEmailEntity() throws Exception {
- DateTimeZone dateTimeZone = DateTimeZone.forID("America/Los_Angeles");
- ThirdEyeAnomalyConfiguration thirdeyeAnomalyConfig = new ThirdEyeAnomalyConfiguration();
- thirdeyeAnomalyConfig.setId(id);
- thirdeyeAnomalyConfig.setDashboardHost(dashboardHost);
- MonitorConfiguration monitorConfiguration = new MonitorConfiguration();
- monitorConfiguration.setMonitorFrequency(new TimeGranularity(3, TimeUnit.SECONDS));
- thirdeyeAnomalyConfig.setMonitorConfiguration(monitorConfiguration);
- TaskDriverConfiguration taskDriverConfiguration = new TaskDriverConfiguration();
- taskDriverConfiguration.setNoTaskDelayInMillis(1000);
- taskDriverConfiguration.setRandomDelayCapInMillis(200);
- taskDriverConfiguration.setTaskFailureDelayInMillis(500);
- taskDriverConfiguration.setMaxParallelTasks(2);
- thirdeyeAnomalyConfig.setTaskDriverConfiguration(taskDriverConfiguration);
- thirdeyeAnomalyConfig.setRootDir(System.getProperty("dw.rootDir", "NOT_SET(dw.rootDir)"));
- Map<String, Map<String, Object>> alerters = new HashMap<>();
- Map<String, Object> smtpProps = new HashMap<>();
- smtpProps.put(SMTP_HOST_KEY, "host");
- smtpProps.put(SMTP_PORT_KEY, "9000");
- alerters.put("smtpConfiguration", smtpProps);
- thirdeyeAnomalyConfig.setAlerterConfiguration(alerters);
-
- List<AnomalyResult> anomalies = new ArrayList<>();
- AnomalyFunctionDTO anomalyFunction = DaoTestUtils.getTestFunctionSpec(TEST, TEST);
- anomalyFunctionDAO.save(anomalyFunction);
- MergedAnomalyResultDTO anomaly = DaoTestUtils.getTestMergedAnomalyResult(
- new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis(),
- new DateTime(2017, 11, 6, 13, 0, dateTimeZone).getMillis(),
- TEST, TEST, 0.1, 1l, new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis());
- anomaly.setFunction(anomalyFunction);
- anomaly.setAvgCurrentVal(1.1);
- anomaly.setAvgBaselineVal(1.0);
- mergedAnomalyResultDAO.save(anomaly);
- anomalies.add(anomaly);
- anomaly = DaoTestUtils.getTestMergedAnomalyResult(
- new DateTime(2017, 11, 7, 10, 0, dateTimeZone).getMillis(),
- new DateTime(2017, 11, 7, 17, 0, dateTimeZone).getMillis(),
- TEST, TEST, 0.1, 1l, new DateTime(2017, 11, 6, 10, 0, dateTimeZone).getMillis());
- anomaly.setFunction(anomalyFunction);
- anomaly.setAvgCurrentVal(0.9);
- anomaly.setAvgBaselineVal(1.0);
- mergedAnomalyResultDAO.save(anomaly);
- anomalies.add(anomaly);
-
- AlertConfigDTO alertConfigDTO = DaoTestUtils.getTestAlertConfiguration("Test Config");
- alertConfigDAO.save(alertConfigDTO);
-
- ADContentFormatterContext context = new ADContentFormatterContext();
- context.setAnomalyFunctionSpec(anomalyFunction);
- context.setAlertConfig(alertConfigDTO);
- EmailContentFormatter
- contentFormatter = new EmailContentFormatter(new OnboardingNotificationContent(), thirdeyeAnomalyConfig);
- DetectionAlertFilterRecipients recipients = new DetectionAlertFilterRecipients(
- EmailUtils.getValidEmailAddresses("a@b.com"));
- EmailEntity emailEntity = contentFormatter.getEmailEntity(recipients, TEST, anomalies, context);
-
- String htmlPath = ClassLoader.getSystemResource("test-onboard-notification-email-content-formatter.html").getPath();
- Assert.assertEquals(
- ContentFormatterUtils.getEmailHtml(emailEntity).replaceAll("\\s", ""),
- ContentFormatterUtils.getHtmlContent(htmlPath).replaceAll("\\s", ""));
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/CleanupAndRegenerateAnomaliesTool.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/CleanupAndRegenerateAnomaliesTool.java
deleted file mode 100644
index 5149e46..0000000
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/tools/CleanupAndRegenerateAnomaliesTool.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/**
- * Copyright (C) 2014-2018 LinkedIn Corp. (pinot-core@linkedin.com)
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.pinot.thirdeye.tools;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
-import org.apache.pinot.thirdeye.anomaly.utils.DetectionResourceHttpUtils;
-import org.apache.pinot.thirdeye.dashboard.resources.OnboardResource;
-import org.apache.pinot.thirdeye.datalayer.bao.AnomalyFunctionManager;
-import org.apache.pinot.thirdeye.datalayer.bao.MergedAnomalyResultManager;
-import org.apache.pinot.thirdeye.datalayer.dto.AnomalyFunctionDTO;
-import org.apache.pinot.thirdeye.datalayer.util.DaoProviderUtil;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.client.ClientProtocolException;
-import org.joda.time.DateTime;
-import org.joda.time.format.ISODateTimeFormat;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.collections.Lists;
-
-/**
- * Utility class to cleanup all anomalies for input datasets,
- * and regenerate anomalies for time range specified in the input
- * Inputs:
- * config file for config class CleanupAndRegenerateAnomaliesConfig
- */
-public class CleanupAndRegenerateAnomaliesTool {
-
- private static final Logger LOG = LoggerFactory.getLogger(CleanupAndRegenerateAnomaliesTool.class);
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory());
- private enum Mode {
- DELETE,
- GENERATE_FOR_RANGE, BACKFILL_FOR_RANGE
- }
- private String monitoringWindowStartTime;
- private String monitoringWindowEndTime;
- private List<Long> functionIds;
-
- private AnomalyFunctionManager anomalyFunctionDAO;
- private MergedAnomalyResultManager mergedResultDAO;
- private DetectionResourceHttpUtils detectionResourceHttpUtils;
-
- public CleanupAndRegenerateAnomaliesTool(String startTime, String endTime, String datasets, String functionIds,
- File persistenceFile, String detectionHost, int detectionPort, String token)
- throws Exception {
- init(persistenceFile);
- this.monitoringWindowStartTime = startTime;
- this.monitoringWindowEndTime = endTime;
- this.functionIds = getFunctionIds(datasets, functionIds);
- detectionResourceHttpUtils = new DetectionResourceHttpUtils(detectionHost, detectionPort, token);
- }
-
- public void init(File persistenceFile) throws Exception {
- DaoProviderUtil.init(persistenceFile);
- anomalyFunctionDAO = DaoProviderUtil
- .getInstance(org.apache.pinot.thirdeye.datalayer.bao.jdbc.AnomalyFunctionManagerImpl.class);
- mergedResultDAO = DaoProviderUtil
- .getInstance(org.apache.pinot.thirdeye.datalayer.bao.jdbc.MergedAnomalyResultManagerImpl.class);
- }
-
- private List<Long> getFunctionIds(String datasets, String functionIds) {
- List<Long> functionIdsList = new ArrayList<>();
- if (StringUtils.isNotBlank(functionIds)) {
- String[] tokens = functionIds.split(",");
- for (String token : tokens) {
- functionIdsList.add(Long.valueOf(token));
- }
- } else if (StringUtils.isNotBlank(datasets)) {
- List<String> datasetsList = Lists.newArrayList(datasets.split(","));
- for (String dataset : datasetsList) {
- List<AnomalyFunctionDTO> anomalyFunctions = anomalyFunctionDAO.findAllByCollection(dataset);
- for (AnomalyFunctionDTO anomalyFunction : anomalyFunctions) {
- functionIdsList.add(anomalyFunction.getId());
- }
- }
- }
- return functionIdsList;
- }
-
- /**
- * Delete raw or merged anomalies whose start time is located in the given time ranges, except
- * the following two cases:
- *
- * 1. If a raw anomaly belongs to a merged anomaly whose start time is not located in the given
- * time ranges, then the raw anomaly will not be deleted.
- *
- * 2. If a raw anomaly belongs to a merged anomaly whose start time is located in the given
- * time ranges, then it is deleted regardless its start time.
- *
- * If monitoringWindowStartTime is not given, then start time is set to 0.
- * If monitoringWindowEndTime is not given, then end time is set to Long.MAX_VALUE.
- */
- private void deleteExistingAnomalies() {
- long startTime = 0;
- long endTime = Long.MAX_VALUE;
- if (StringUtils.isNotBlank(monitoringWindowStartTime)) {
- startTime =
- ISODateTimeFormat.dateTimeParser().parseDateTime(monitoringWindowStartTime).getMillis();
- }
- if (StringUtils.isNotBlank(monitoringWindowEndTime)) {
- endTime =
- ISODateTimeFormat.dateTimeParser().parseDateTime(monitoringWindowEndTime).getMillis();
- }
- LOG.info("Deleting anomalies in the time range: {} -- {}", new DateTime(startTime), new
- DateTime(endTime));
-
- for (Long functionId : functionIds) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if(anomalyFunction == null){
- LOG.info("Requested functionId {} doesn't exist", functionId);
- continue;
- }
-
- LOG.info("Beginning cleanup of functionId {} collection {} metric {}",
- functionId, anomalyFunction.getCollection(), anomalyFunction.getMetric());
-
- // Clean up merged and raw anomaly of functionID
- OnboardResource onboardResource = new OnboardResource(anomalyFunctionDAO, mergedResultDAO);
- onboardResource.deleteExistingAnomalies(functionId, startTime, endTime);
- }
- }
-
-
- /**
- * Regenerates anomalies for the whole given range as one monitoring window
- * @throws Exception
- */
- @Deprecated
- private void regenerateAnomaliesInRange() throws Exception {
- LOG.info("Begin regenerate anomalies for entire range...");
- for (Long functionId : functionIds) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- boolean isActive = anomalyFunction.getIsActive();
- if (!isActive) {
- LOG.info("Skipping function {}", functionId);
- continue;
- }
- runAdhocFunctionForWindow(functionId, monitoringWindowStartTime, monitoringWindowEndTime);
- }
- }
-
- /**
- * Breaks down the given range into consecutive monitoring windows as per function definition
- * Regenerates anomalies for each window separately
- * @throws Exception
- */
- private void regenerateAnomaliesForBucketsInRange(boolean forceBackfill) throws Exception {
- for (Long functionId : functionIds) {
- AnomalyFunctionDTO anomalyFunction = anomalyFunctionDAO.findById(functionId);
- if (!anomalyFunction.getIsActive()) {
- LOG.info("Skipping deactivated function {}", functionId);
- continue;
- }
-
- LOG.info("Sending backfill function {} for range {} to {}", functionId, monitoringWindowStartTime, monitoringWindowEndTime);
-
- String response =
- detectionResourceHttpUtils.runBackfillAnomalyFunction(String.valueOf(functionId), monitoringWindowStartTime,
- monitoringWindowEndTime, forceBackfill);
- LOG.info("Response {}", response);
- }
- }
-
- private void runAdhocFunctionForWindow(Long functionId, String monitoringWindowStart, String monitoringWindowEnd)
- throws ClientProtocolException, IOException {
- LOG.info("Running adhoc function {} for range {} to {}", functionId, monitoringWindowStart, monitoringWindowEnd);
- String response = detectionResourceHttpUtils.runAdhocAnomalyFunction(String.valueOf(functionId),
- monitoringWindowStart, monitoringWindowEnd);
- LOG.info("Response {}", response);
- }
-
- public static void main(String[] args) throws Exception {
-
- if (args.length != 2) {
- System.err.println("USAGE CleanupAndRegenerateAnomaliesTool <config_yml_file> <mode> \n "
- + "Please take note: \nDELETE mode will delete all anomalies for that functionId/dataset, "
- + "\nGENERATE mode will generate anomalies in time range you specify");
- System.exit(1);
- }
- File configFile = new File(args[0]);
- CleanupAndRegenerateAnomaliesConfig config =
- OBJECT_MAPPER.readValue(configFile, CleanupAndRegenerateAnomaliesConfig.class);
-
- String mode = args[1];
-
- File persistenceFile = new File(config.getPersistenceFile());
- if (!persistenceFile.exists()) {
- System.err.println("Missing file:" + persistenceFile);
- System.exit(1);
- }
- String detectorHost = config.getDetectorHost();
- int detectorPort = config.getDetectorPort();
- if (StringUtils.isBlank(detectorHost)) {
- LOG.error("Detector host and port must be provided");
- System.exit(1);
- }
-
- String startTimeIso = config.getStartTimeIso();
- String endTimeIso = config.getEndTimeIso();
- Mode runMode = Mode.valueOf(mode);
- if ((runMode.equals(Mode.GENERATE_FOR_RANGE) || runMode.equals(Mode.BACKFILL_FOR_RANGE))
- && (StringUtils.isBlank(startTimeIso) || StringUtils.isBlank(endTimeIso))) {
- LOG.error("StarteTime and endTime must be provided in generate mode");
- System.exit(1);
- }
-
- String datasets = config.getDatasets();
- String functionIds = config.getFunctionIds();
- if (StringUtils.isBlank(datasets) && StringUtils.isBlank(functionIds)) {
- LOG.error("Must provide one of datasets or functionIds");
- System.exit(1);
- }
-
- boolean doForceBackfill = false;
- String forceBackfill = config.getForceBackfill();
- if (StringUtils.isNotBlank(forceBackfill)) {
- doForceBackfill = Boolean.parseBoolean(forceBackfill);
- }
-
- String authToken = "";
-
- CleanupAndRegenerateAnomaliesTool tool = new CleanupAndRegenerateAnomaliesTool(startTimeIso,
- endTimeIso, datasets, functionIds, persistenceFile, detectorHost, detectorPort, authToken);
-
- if (runMode.equals(Mode.DELETE)) {
- // DELETE mode deletes *ALL* anomalies for all functions in functionIds or datasets
- tool.deleteExistingAnomalies();
- } else if (runMode.equals(Mode.GENERATE_FOR_RANGE)) {
- // GENERATE_FOR_RANGE mode regenerates anomalies for all active functions in functionIds or datasets
- tool.regenerateAnomaliesInRange();
- } else if (runMode.equals(Mode.BACKFILL_FOR_RANGE)) {
- // BACKFILL_FOR_RANGE mode regenerates anomalies for all active functions in functionIds or datasets
- // It will honor the monitoring window size of the function, and run for all consecutive windows, one by one,
- // to cover the entire range provided as input
- tool.regenerateAnomaliesForBucketsInRange(doForceBackfill);
- } else {
- LOG.error("Incorrect mode {}", mode);
- System.exit(1);
- }
- // Added this because database connection gets stuck at times and program never terminates
- System.exit(0);
- }
-
-}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/sample-detection-config.yml b/thirdeye/thirdeye-pinot/src/test/resources/sample-detection-config.yml
index 442d7e6..888f983 100644
--- a/thirdeye/thirdeye-pinot/src/test/resources/sample-detection-config.yml
+++ b/thirdeye/thirdeye-pinot/src/test/resources/sample-detection-config.yml
@@ -14,6 +14,6 @@ rules:
- name: detection_rule_1
type: THRESHOLD
params:
- max: 500
- min: NaN
+ max: 900
+ min: 100
monitoringGranularity: 1_HOURS
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/test-metric-anomalies-template.html b/thirdeye/thirdeye-pinot/src/test/resources/test-metric-anomalies-template.html
index 06e0e09..65fe591 100644
--- a/thirdeye/thirdeye-pinot/src/test/resources/test-metric-anomalies-template.html
+++ b/thirdeye/thirdeye-pinot/src/test/resources/test-metric-anomalies-template.html
@@ -20,7 +20,7 @@
between <strong>Nov 06, 10:00</strong> and <strong>Nov 07, 17:00</strong> (PDT)
</p>
<p>
- <a style="padding: 6px 12px; border-radius: 2px; border: 1px solid #FFF; font-size: 16px; font-weight: bold; color: white; text-decoration: none; line-height: 32px;" href="http://localhost:8080/dashboard/app/#/anomalies?anomalyIds=2,3">View all anomalies on ThirdEye</a>
+ <a style="padding: 6px 12px; border-radius: 2px; border: 1px solid #FFF; font-size: 16px; font-weight: bold; color: white; text-decoration: none; line-height: 32px;" href="http://localhost:8080/dashboard/app/#/anomalies?anomalyIds=4,5">View all anomalies on ThirdEye</a>
</p>
</td>
</tr>
@@ -38,11 +38,11 @@
<!-- List down all the alerts for the given metric -->
<p>
<span style="color: #1D1D1D; font-size: 16px; font-weight: bold; display:inline-block; vertical-align: middle;">Alert: </span>
- <span style="color: #606060; font-size: 16px; text-decoration: none; display:inline-block; vertical-align: middle; width: 77%; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">integration test function 1</span>
- <a href="http://localhost:8080/dashboard/app/#/manage/explore/1" target="blank" style="text-decoration: none; color: #0B5EA1; display:inline-block; vertical-align: middle;">(Edit Settings)</a>
+ <span style="color: #606060; font-size: 16px; text-decoration: none; display:inline-block; vertical-align: middle; width: 77%; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;">sample_detection</span>
+ <a href="http://localhost:8080/dashboard/app/#/manage/explore/3" target="blank" style="text-decoration: none; color: #0B5EA1; display:inline-block; vertical-align: middle;">(Edit Settings)</a>
</p>
<p>
- <span style="color: #606060; font-size: 13px; text-decoration: none; display:inline-block; vertical-align: middle; width: 77%; white-space: wrap;"></span>
+ <span style="color: #606060; font-size: 13px; text-decoration: none; display:inline-block; vertical-align: middle; width: 77%; white-space: wrap;">If this alert fires then it means so-and-so and check so-and-so for irregularities</span>
</p>
<!-- List all the anomalies under this detection -->
<table border="0" width="100%" align="center" style="width:100%; padding:0; margin:0; border-collapse: collapse;text-align:left;">
@@ -56,7 +56,7 @@
<td style="padding: 6px 12px;white-space: nowrap;">
<div style="color: rgba(0,0,0,0.9); font-size:14px; line-height:20px;">Nov 06, 10:00 PDT</div>
<span style="color: rgba(0,0,0,0.6); font-size:12px; line-height:16px;">3 hours</span>
- <a style="font-weight: bold; text-decoration: none; font-size:14px; line-height:20px; color: #0073B1;" href="http://localhost:8080/dashboard/app/#/rootcause?anomalyId=2"
+ <a style="font-weight: bold; text-decoration: none; font-size:14px; line-height:20px; color: #0073B1;" href="http://localhost:8080/dashboard/app/#/rootcause?anomalyId=4"
target="_blank">(view)</a>
</td>
<td style="word-break: break-all; width: 200px; padding-right:4px 20px 4px 0">
@@ -71,7 +71,7 @@
<td style="padding: 6px 12px;white-space: nowrap;">
<div style="color: rgba(0,0,0,0.9); font-size:14px; line-height:20px;">Nov 07, 10:00 PDT</div>
<span style="color: rgba(0,0,0,0.6); font-size:12px; line-height:16px;">7 hours</span>
- <a style="font-weight: bold; text-decoration: none; font-size:14px; line-height:20px; color: #0073B1;" href="http://localhost:8080/dashboard/app/#/rootcause?anomalyId=3"
+ <a style="font-weight: bold; text-decoration: none; font-size:14px; line-height:20px; color: #0073B1;" href="http://localhost:8080/dashboard/app/#/rootcause?anomalyId=5"
target="_blank">(view)</a>
</td>
<td style="word-break: break-all; width: 200px; padding-right:4px 20px 4px 0">
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org