You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ji...@apache.org on 2020/05/20 20:29:05 UTC
[incubator-pinot] branch master updated: [TE] alerts search and
pagination endpoint (#5413)
This is an automated email from the ASF dual-hosted git repository.
jihao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 775d578 [TE] alerts search and pagination endpoint (#5413)
775d578 is described below
commit 775d57827b38f293cf62748b118d940617b459ed
Author: Jihao Zhang <ji...@linkedin.com>
AuthorDate: Wed May 20 13:28:52 2020 -0700
[TE] alerts search and pagination endpoint (#5413)
This commit adds the endpoint for alert search and pagination. Specifically,
- The endpoint to search and paginate the alerts.
- The Alert searcher to retrieve the alerts in the database based on the search filter.
- Add the capability of counting, pagination, and find by JSON value predicate to the generic DAO.
- Unit tests
---
.../dashboard/ThirdEyeDashboardApplication.java | 2 +
.../resources/v2/alerts/AlertResource.java | 79 +++++++
.../resources/v2/alerts/AlertSearchFilter.java | 157 +++++++++++++
.../resources/v2/alerts/AlertSearcher.java | 259 +++++++++++++++++++++
.../thirdeye/datalayer/bao/AbstractManager.java | 29 ++-
.../datalayer/bao/jdbc/AbstractManagerImpl.java | 15 ++
.../thirdeye/datalayer/dao/GenericPojoDao.java | 82 +++++++
.../datalayer/entity/DetectionConfigIndex.java | 18 ++
.../thirdeye/datalayer/util/SqlQueryBuilder.java | 40 ++++
.../thirdeye/detection/DetectionResource.java | 1 -
.../src/main/resources/schema/create-schema.sql | 4 +
.../resources/v2/alerts/AlertSearcherTest.java | 75 ++++++
12 files changed, 759 insertions(+), 2 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
index 1ac0973..ccd70d3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/ThirdEyeDashboardApplication.java
@@ -44,6 +44,7 @@ import org.apache.pinot.thirdeye.dashboard.resources.CacheResource;
import org.apache.pinot.thirdeye.dashboard.resources.CustomizedEventResource;
import org.apache.pinot.thirdeye.dashboard.resources.DashboardResource;
import org.apache.pinot.thirdeye.dashboard.resources.DatasetConfigResource;
+import org.apache.pinot.thirdeye.dashboard.resources.v2.alerts.AlertResource;
import org.apache.pinot.thirdeye.dashboard.resources.DetectionJobResource;
import org.apache.pinot.thirdeye.dashboard.resources.EmailResource;
import org.apache.pinot.thirdeye.dashboard.resources.EntityManagerResource;
@@ -194,6 +195,7 @@ public class ThirdEyeDashboardApplication
env.jersey().register(new YamlResource(config.getAlerterConfiguration(), config.getDetectionPreviewConfig(),
config.getAlertOnboardingPermitPerSecond()));
env.jersey().register(new SqlDataSourceResource());
+ env.jersey().register(new AlertResource());
TimeSeriesLoader timeSeriesLoader = new DefaultTimeSeriesLoader(
DAO_REGISTRY.getMetricConfigDAO(), DAO_REGISTRY.getDatasetConfigDAO(),
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertResource.java
new file mode 100644
index 0000000..1bc4f93
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertResource.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.dashboard.resources.v2.alerts;
+
+import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
+import java.util.List;
+import javax.ws.rs.DefaultValue;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.pinot.thirdeye.api.Constants;
+
+
+/**
+ * The Alert resource.
+ */
+@Path(value = "/alerts")
+@Produces(MediaType.APPLICATION_JSON)
+@Api(tags = {Constants.DETECTION_TAG})
+public class AlertResource {
+ private final AlertSearcher alertSearcher;
+
+ /**
+ * Instantiates a new Alert resource.
+ */
+ public AlertResource() {
+ this.alertSearcher = new AlertSearcher();
+ }
+
+ /**
+ * Search the alerts with result pagination. It will return record from No.(offset+1) to record No.(offset + limit).
+ *
+ * @param limit the returned result limit
+ * @param offset the offset of the start position
+ * @param applications the applications for the alerts
+ * @param subscriptionGroups the subscription groups for the alerts
+ * @param names the names for the alerts
+ * @param createdBy the owners for the alerts
+ * @param ruleTypes the rule types for the alerts
+ * @param metrics the metrics for the alerts
+ * @param datasets the datasets for the alerts
+ * @param active if the alert is active
+ * @return the response
+ */
+ @GET
+ @Produces(MediaType.APPLICATION_JSON)
+ @ApiOperation("Search and paginate alerts according to the parameters")
+ public Response findAlerts(@QueryParam("limit") @DefaultValue("10") long limit,
+ @QueryParam("offset") @DefaultValue("0") long offset, @QueryParam("application") List<String> applications,
+ @QueryParam("subscriptionGroup") List<String> subscriptionGroups, @QueryParam("names") List<String> names,
+ @QueryParam("createdBy") List<String> createdBy, @QueryParam("ruleType") List<String> ruleTypes,
+ @QueryParam("metric") List<String> metrics, @QueryParam("dataset") List<String> datasets,
+ @QueryParam("active") Boolean active) {
+ AlertSearchFilter searchFilter = new AlertSearchFilter(applications, subscriptionGroups, names, createdBy, ruleTypes, metrics, datasets, active);
+ return Response.ok().entity(this.alertSearcher.search(searchFilter, limit, offset)).build();
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearchFilter.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearchFilter.java
new file mode 100644
index 0000000..0c488ff
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearchFilter.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.dashboard.resources.v2.alerts;
+
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * The type Alert search filter.
+ */
+public class AlertSearchFilter {
+ private final List<String> applications;
+ private final List<String> subscriptionGroups;
+ private final List<String> createdBy;
+ private final List<String> ruleTypes;
+ private final List<String> metrics;
+ private final List<String> datasets;
+ private final List<String> names;
+ private final Boolean active;
+
+ public AlertSearchFilter() {
+ this.applications = Collections.emptyList();
+ this.subscriptionGroups = Collections.emptyList();
+ this.createdBy = Collections.emptyList();
+ this.ruleTypes = Collections.emptyList();
+ this.datasets = Collections.emptyList();
+ this.metrics = Collections.emptyList();
+ this.names = Collections.emptyList();
+ this.active = null;
+ }
+
+ /**
+ * Instantiates a new Alert search filter.
+ *
+ * @param applications the applications
+ * @param subscriptionGroups the subscription groups
+ * @param names the names
+ * @param createdBy the createdBy
+ * @param ruleTypes the rule types
+ * @param metrics the metrics
+ * @param datasets the datasets
+ * @param active the active
+ */
+ public AlertSearchFilter(List<String> applications, List<String> subscriptionGroups, List<String> names,
+ List<String> createdBy, List<String> ruleTypes, List<String> metrics, List<String> datasets, Boolean active) {
+ this.applications = applications;
+ this.subscriptionGroups = subscriptionGroups;
+ this.names = names;
+ this.createdBy = createdBy;
+ this.ruleTypes = ruleTypes;
+ this.metrics = metrics;
+ this.datasets = datasets;
+ this.active = active;
+ }
+
+ /**
+ * Gets applications.
+ *
+ * @return the applications
+ */
+ public List<String> getApplications() {
+ return applications;
+ }
+
+ /**
+ * Gets subscription groups.
+ *
+ * @return the subscription groups
+ */
+ public List<String> getSubscriptionGroups() {
+ return subscriptionGroups;
+ }
+
+ /**
+ * Gets createdBy.
+ *
+ * @return the owners
+ */
+ public List<String> getCreatedBy() {
+ return createdBy;
+ }
+
+ /**
+ * Gets rule types.
+ *
+ * @return the rule types
+ */
+ public List<String> getRuleTypes() {
+ return ruleTypes;
+ }
+
+ /**
+ * Gets metrics.
+ *
+ * @return the metrics
+ */
+ public List<String> getMetrics() {
+ return metrics;
+ }
+
+ /**
+ * Gets datasets.
+ *
+ * @return the datasets
+ */
+ public List<String> getDatasets() {
+ return datasets;
+ }
+
+ /**
+ * Gets names.
+ *
+ * @return the names
+ */
+ public List<String> getNames() {
+ return names;
+ }
+
+ /**
+ * Gets active.
+ *
+ * @return the active
+ */
+ public Boolean getActive() {
+ return active;
+ }
+
+ /**
+ * If all the search filters are empty.
+ *
+ * @return the boolean value of the result
+ */
+ public boolean isEmpty() {
+ return this.applications.isEmpty() && this.subscriptionGroups.isEmpty() && this.names.isEmpty()
+ && this.createdBy.isEmpty() && this.ruleTypes.isEmpty() && this.metrics.isEmpty() && this.datasets.isEmpty()
+ && active == null;
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcher.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcher.java
new file mode 100644
index 0000000..78fe3a1
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcher.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.dashboard.resources.v2.alerts;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Multimap;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.pinot.thirdeye.datalayer.bao.DatasetConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionAlertConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.bao.MetricConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionAlertConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datalayer.util.Predicate;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.formatter.DetectionConfigFormatter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * The Alert searcher.
+ */
+public class AlertSearcher {
+ private static final Logger LOG = LoggerFactory.getLogger(AlertSearcher.class.getName());
+ private final DetectionConfigManager detectionConfigDAO;
+ private final DetectionAlertConfigManager detectionAlertConfigDAO;
+ private final MetricConfigManager metricDAO;
+ private final DatasetConfigManager datasetDAO;
+ private final DetectionConfigFormatter detectionConfigFormatter;
+
+ /**
+ * The Alert search query.
+ */
+ static class AlertSearchQuery {
+ /**
+ * The Search filter.
+ */
+ final AlertSearchFilter searchFilter;
+ /**
+ * The Limit.
+ */
+ final long limit;
+ /**
+ * The Offset.
+ */
+ final long offset;
+
+ /**
+ * Instantiates a new Alert search query.
+ *
+ * @param searchFilter the search filter
+ * @param limit the limit
+ * @param offset the offset
+ */
+ public AlertSearchQuery(AlertSearchFilter searchFilter, long limit, long offset) {
+ this.searchFilter = searchFilter;
+ this.limit = limit;
+ this.offset = offset;
+ }
+ }
+
+ /**
+ * Instantiates a new Alert searcher.
+ */
+ public AlertSearcher() {
+ this.detectionConfigDAO = DAORegistry.getInstance().getDetectionConfigManager();
+ this.detectionAlertConfigDAO = DAORegistry.getInstance().getDetectionAlertConfigManager();
+ this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
+ this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
+ this.detectionConfigFormatter = new DetectionConfigFormatter(metricDAO, datasetDAO);
+ }
+
+ /**
+ * Search and retrive all the alerts matching to the search filter and limits.
+ *
+ * @param searchFilter the search filter
+ * @param limit the limit
+ * @param offset the offset
+ * @return the result
+ */
+ public Map<String, Object> search(AlertSearchFilter searchFilter, long limit, long offset) {
+ AlertSearchQuery searchQuery = new AlertSearchQuery(searchFilter, limit, offset);
+ List<DetectionAlertConfigDTO> subscriptionGroups = findRelatedSubscriptionGroups(searchQuery);
+ List<DetectionConfigDTO> detectionConfigs = findDetectionConfig(searchQuery, subscriptionGroups);
+ return getResult(searchQuery, subscriptionGroups, detectionConfigs);
+ }
+
+ private List<DetectionAlertConfigDTO> findRelatedSubscriptionGroups(AlertSearchQuery searchQuery) {
+ AlertSearchFilter searchFilter = searchQuery.searchFilter;
+ List<Predicate> predicates = new ArrayList<>();
+ if (!searchFilter.getApplications().isEmpty()) {
+ predicates.add(Predicate.IN("application", searchFilter.getApplications().toArray()));
+ }
+ if (!searchFilter.getSubscriptionGroups().isEmpty()) {
+ predicates.add(Predicate.IN("name", searchFilter.getSubscriptionGroups().toArray()));
+ }
+ if (!predicates.isEmpty()) {
+ return this.detectionAlertConfigDAO.findByPredicate(Predicate.AND(predicates.toArray(new Predicate[0])));
+ } else {
+ return this.detectionAlertConfigDAO.findAll();
+ }
+ }
+
+ private List<DetectionConfigDTO> findDetectionConfig(AlertSearchQuery searchQuery,
+ List<DetectionAlertConfigDTO> subscriptionGroups) {
+ AlertSearchFilter searchFilter = searchQuery.searchFilter;
+ if (searchFilter.isEmpty()) {
+ // if no search filter is applied, by default, retrieve the paginated result from db
+ return this.detectionConfigDAO.list(searchQuery.limit, searchQuery.offset);
+ }
+
+ // look up and run the search filters on the detection config index
+ List<DetectionConfigDTO> indexedResult = new ArrayList<>();
+ List<Predicate> indexPredicates = new ArrayList<>();
+ if (!searchFilter.getApplications().isEmpty() || !searchFilter.getSubscriptionGroups().isEmpty()) {
+ Set<Long> detectionConfigIds = new TreeSet<>();
+ for (DetectionAlertConfigDTO subscriptionGroup : subscriptionGroups) {
+ detectionConfigIds.addAll(subscriptionGroup.getVectorClocks().keySet());
+ }
+ indexPredicates.add(Predicate.IN("baseId", detectionConfigIds.toArray()));
+ }
+ if (!searchFilter.getCreatedBy().isEmpty()) {
+ indexPredicates.add(Predicate.IN("createdBy", searchFilter.getCreatedBy().toArray()));
+ }
+ if (!searchFilter.getNames().isEmpty()) {
+ indexPredicates.add(Predicate.IN("name", searchFilter.getNames().toArray()));
+ }
+ if (searchFilter.getActive() != null) {
+ indexPredicates.add(Predicate.EQ("active", searchFilter.getActive() ? 1 : 0));
+ }
+ if (!indexPredicates.isEmpty()) {
+ indexedResult = this.detectionConfigDAO.findByPredicate(Predicate.AND(indexPredicates.toArray(new Predicate[0])));
+ }
+
+ // for metrics, datasets, rule types filters, run the search filters in the generic table
+ List<DetectionConfigDTO> jsonValResult = new ArrayList<>();
+ List<Predicate> jsonValPredicates = new ArrayList<>();
+ if (!searchFilter.getRuleTypes().isEmpty()) {
+ List<Predicate> ruleTypePredicates = new ArrayList<>();
+ for (String ruleType : searchFilter.getRuleTypes()) {
+ ruleTypePredicates.add(Predicate.LIKE("jsonVal", "%componentSpecs%:" + ruleType + "\"%"));
+ }
+ jsonValPredicates.add(Predicate.OR(ruleTypePredicates.toArray(new Predicate[0])));
+ }
+
+ Set<Long> metricIds = new HashSet<>();
+ if (!searchFilter.getMetrics().isEmpty()) {
+ for (String metric : searchFilter.getMetrics()) {
+ metricIds =
+ this.metricDAO.findByMetricName(metric).stream().map(AbstractDTO::getId).collect(Collectors.toSet());
+ }
+ }
+
+ if (!searchFilter.getDatasets().isEmpty()) {
+ for (String dataset : searchFilter.getDatasets()) {
+ metricIds.retainAll(
+ this.metricDAO.findByDataset(dataset).stream().map(AbstractDTO::getId).collect(Collectors.toSet()));
+ }
+ }
+
+ if (!metricIds.isEmpty()) {
+ List<Predicate> metricUrnPredicates = new ArrayList<>();
+ for (Long id : metricIds) {
+ metricUrnPredicates.add(Predicate.LIKE("jsonVal", "%thirdeye:metric:" + id + "%"));
+ }
+ jsonValPredicates.add(Predicate.OR(metricUrnPredicates.toArray(new Predicate[0])));
+ }
+
+ if (!jsonValPredicates.isEmpty()) {
+ jsonValResult =
+ this.detectionConfigDAO.findByPredicateJsonVal(Predicate.AND(jsonValPredicates.toArray(new Predicate[0])));
+ }
+
+ List<DetectionConfigDTO> result;
+ if (!jsonValPredicates.isEmpty() && !indexPredicates.isEmpty()) {
+ // merge the result from both tables
+ result = jsonValResult.stream().filter(indexedResult::contains).collect(Collectors.toList());
+ } else {
+ jsonValResult.addAll(indexedResult);
+ result = jsonValResult;
+ }
+ return result;
+ }
+
+ /**
+ * Format and generate the final search result
+ */
+ private Map<String, Object> getResult(AlertSearchQuery searchQuery, List<DetectionAlertConfigDTO> subscriptionGroups,
+ List<DetectionConfigDTO> detectionConfigs) {
+ long count;
+ if (searchQuery.searchFilter.isEmpty()) {
+ // if not filter is applied, execute count query
+ count = this.detectionConfigDAO.count();
+ } else {
+ // count and limit the filtered results
+ count = detectionConfigs.size();
+ if (searchQuery.offset >= count) {
+ // requested page is out of bound
+ detectionConfigs.clear();
+ } else {
+ detectionConfigs = detectionConfigs.subList((int) searchQuery.offset,
+ (int) Math.min(searchQuery.offset + searchQuery.limit, count));
+ }
+ }
+
+ // format the results
+ List<Map<String, Object>> alerts = detectionConfigs.parallelStream().map(config -> {
+ try {
+ return this.detectionConfigFormatter.format(config);
+ } catch (Exception e) {
+ LOG.warn("formatting detection config failed {}", config.getId(), e);
+ return null;
+ }
+ }).filter(Objects::nonNull).collect(Collectors.toList());
+
+ // join detections with subscription groups
+ Multimap<Long, String> detectionIdToSubscriptionGroups = ArrayListMultimap.create();
+ for (DetectionAlertConfigDTO subscriptionGroup : subscriptionGroups) {
+ for (long detectionConfigId : subscriptionGroup.getVectorClocks().keySet()) {
+ detectionIdToSubscriptionGroups.put(detectionConfigId, subscriptionGroup.getName());
+ }
+ }
+ for (Map<String, Object> alert : alerts) {
+ alert.put("subscriptionGroup", detectionIdToSubscriptionGroups.get(MapUtils.getLong(alert, "id")));
+ }
+
+ return ImmutableMap.of("count", count, "limit", searchQuery.limit, "offset", searchQuery.offset, "elements",
+ alerts);
+ }
+}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/AbstractManager.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/AbstractManager.java
index 69e6354..9faa14d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/AbstractManager.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/AbstractManager.java
@@ -21,7 +21,7 @@ package org.apache.pinot.thirdeye.datalayer.bao;
import java.util.List;
import java.util.Map;
-
+import org.apache.commons.lang3.NotImplementedException;
import org.apache.pinot.thirdeye.datalayer.dto.AbstractDTO;
import org.apache.pinot.thirdeye.datalayer.util.Predicate;
@@ -57,4 +57,31 @@ public interface AbstractManager<E extends AbstractDTO> {
List<Long> findIdsByPredicate(Predicate predicate);
int update(E entity, Predicate predicate);
+
+ /**
+ * Find the entities based on the JSON value predicate
+ * @param predicate the predicate
+ * @return the list of entities that match with the predicate
+ */
+ default List<E> findByPredicateJsonVal(Predicate predicate) {
+ throw new NotImplementedException("Not Implemented");
+ }
+
+ /**
+ * List the entities with pagination
+ * @param limit the limit for the number of elements returned
+ * @param offset the offset position
+ * @return the list of entities ordered by id in descending order
+ */
+ default List<E> list(long limit, long offset) {
+ throw new NotImplementedException("Not Implemented");
+ }
+
+ /**
+ * Count how many entities are there in the table
+ * @return the number of total entities
+ */
+ default long count() {
+ throw new NotImplementedException("Not Implemented");
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/AbstractManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/AbstractManagerImpl.java
index c13cea9..81a8535 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/AbstractManagerImpl.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/bao/jdbc/AbstractManagerImpl.java
@@ -171,6 +171,21 @@ public abstract class AbstractManagerImpl<E extends AbstractDTO> implements Abst
return genericPojoDao.getIdsByPredicate(predicate, beanClass);
}
+ @Override
+ public List<E> list(long limit, long offset) {
+ return convertBeanListToDTOList(genericPojoDao.list(beanClass, limit, offset));
+ }
+
+ @Override
+ public List<E> findByPredicateJsonVal(Predicate predicate) {
+ return convertBeanListToDTOList(genericPojoDao.getByPredicateJsonVal(predicate, beanClass));
+ }
+
+ @Override
+ public long count() {
+ return genericPojoDao.count(beanClass);
+ }
+
protected List<E> convertBeanListToDTOList(List<? extends AbstractBean> beans) {
List<E> result = new ArrayList<>();
for (AbstractBean bean : beans) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java
index 8e983c5..227c9b1 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/dao/GenericPojoDao.java
@@ -442,6 +442,88 @@ public class GenericPojoDao {
}
}
+ public <E extends AbstractBean> List<E> list(final Class<E> beanClass, long limit, long offset) {
+ long tStart = System.nanoTime();
+ try {
+ return runTask(connection -> {
+ List<GenericJsonEntity> entities;
+ Predicate predicate = Predicate.EQ("beanClass", beanClass.getName());
+ try (PreparedStatement selectStatement =
+ sqlQueryBuilder.createfindByParamsStatementWithLimit(connection, GenericJsonEntity.class, predicate, limit, offset)) {
+ try (ResultSet resultSet = selectStatement.executeQuery()) {
+ entities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class);
+ }
+ }
+ List<E> result = new ArrayList<>();
+ if (entities != null) {
+ for (GenericJsonEntity entity : entities) {
+ ThirdeyeMetricsUtil.dbReadByteCounter.inc(entity.getJsonVal().length());
+ E e = OBJECT_MAPPER.readValue(entity.getJsonVal(), beanClass);
+ e.setId(entity.getId());
+ e.setUpdateTime(entity.getUpdateTime());
+ result.add(e);
+ }
+ }
+ return result;
+ }, Collections.emptyList());
+ } finally {
+ ThirdeyeMetricsUtil.dbReadCallCounter.inc();
+ ThirdeyeMetricsUtil.dbReadDurationCounter.inc(System.nanoTime() - tStart);
+ }
+ }
+
+ public <E extends AbstractBean> List<E> getByPredicateJsonVal(Predicate predicate, final Class<E> beanClass) {
+ long tStart = System.nanoTime();
+ try {
+ return runTask(connection -> {
+ List<GenericJsonEntity> entities;
+ Predicate p = Predicate.AND(predicate, Predicate.EQ("beanClass", beanClass.getName()));
+ try (PreparedStatement selectStatement =
+ sqlQueryBuilder.createFindByParamsStatement(connection, GenericJsonEntity.class, p)) {
+ try (ResultSet resultSet = selectStatement.executeQuery()) {
+ entities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class);
+ }
+ }
+ List<E> result = new ArrayList<>();
+ if (entities != null) {
+ for (GenericJsonEntity entity : entities) {
+ ThirdeyeMetricsUtil.dbReadByteCounter.inc(entity.getJsonVal().length());
+ E e = OBJECT_MAPPER.readValue(entity.getJsonVal(), beanClass);
+ e.setId(entity.getId());
+ e.setUpdateTime(entity.getUpdateTime());
+ result.add(e);
+ }
+ }
+ return result;
+ }, Collections.emptyList());
+ } finally {
+ ThirdeyeMetricsUtil.dbReadCallCounter.inc();
+ ThirdeyeMetricsUtil.dbReadDurationCounter.inc(System.nanoTime() - tStart);
+ }
+ }
+
+ public <E extends AbstractBean> long count(final Class<E> beanClass) {
+ long tStart = System.nanoTime();
+ try {
+ return runTask(connection -> {
+ PojoInfo pojoInfo = pojoInfoMap.get(beanClass);
+ try (PreparedStatement selectStatement =
+ sqlQueryBuilder.createCountStatement(connection, pojoInfo.indexEntityClass)) {
+ try (ResultSet resultSet = selectStatement.executeQuery()) {
+ if (resultSet.next()) {
+ return resultSet.getInt(1);
+ } else {
+ throw new IllegalStateException("can't parse count query response");
+ }
+ }
+ }
+ }, -1);
+ } finally {
+ ThirdeyeMetricsUtil.dbReadCallCounter.inc();
+ ThirdeyeMetricsUtil.dbReadDurationCounter.inc(System.nanoTime() - tStart);
+ }
+ }
+
public <E extends AbstractBean> E get(final Long id, final Class<E> pojoClass) {
long tStart = System.nanoTime();
try {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/entity/DetectionConfigIndex.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/entity/DetectionConfigIndex.java
index 39e9d64..2c4011f 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/entity/DetectionConfigIndex.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/entity/DetectionConfigIndex.java
@@ -21,6 +21,8 @@ package org.apache.pinot.thirdeye.datalayer.entity;
public class DetectionConfigIndex extends AbstractIndexEntity {
String name;
+ boolean active;
+ String createdBy;
public String getName() {
return name;
@@ -29,4 +31,20 @@ public class DetectionConfigIndex extends AbstractIndexEntity {
public void setName(String name) {
this.name = name;
}
+
+ public boolean isActive() {
+ return active;
+ }
+
+ public void setActive(boolean active) {
+ this.active = active;
+ }
+
+ public String getCreatedBy() {
+ return createdBy;
+ }
+
+ public void setCreatedBy(String createdBy) {
+ this.createdBy = createdBy;
+ }
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/SqlQueryBuilder.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/SqlQueryBuilder.java
index 88df8de..6f4076c 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/SqlQueryBuilder.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datalayer/util/SqlQueryBuilder.java
@@ -338,6 +338,46 @@ public class SqlQueryBuilder {
return prepareStatement;
}
+ public PreparedStatement createfindByParamsStatementWithLimit(Connection connection,
+ Class<? extends AbstractEntity> entityClass, Predicate predicate, Long limit, Long offset) throws Exception {
+ String tableName = entityMappingHolder.tableToEntityNameMap.inverse().get(entityClass.getSimpleName());
+ BiMap<String, String> entityNameToDBNameMapping =
+ entityMappingHolder.columnMappingPerTable.get(tableName).inverse();
+ StringBuilder sqlBuilder = new StringBuilder("SELECT * FROM " + tableName);
+ StringBuilder whereClause = new StringBuilder(" WHERE ");
+ List<Pair<String, Object>> parametersList = new ArrayList<>();
+ generateWhereClause(entityNameToDBNameMapping, predicate, parametersList, whereClause);
+ sqlBuilder.append(whereClause.toString());
+ sqlBuilder.append(" ORDER BY id DESC");
+ if (limit != null) {
+ sqlBuilder.append(" LIMIT ").append(limit);
+ }
+ if (offset != null) {
+ sqlBuilder.append(" OFFSET ").append(offset);
+ }
+ PreparedStatement prepareStatement = connection.prepareStatement(sqlBuilder.toString());
+ int parameterIndex = 1;
+ LinkedHashMap<String, ColumnInfo> columnInfoMap =
+ entityMappingHolder.columnInfoPerTable.get(tableName);
+ for (Pair<String, Object> pair : parametersList) {
+ String dbFieldName = pair.getKey();
+ ColumnInfo info = columnInfoMap.get(dbFieldName);
+ Preconditions.checkNotNull(info, String.format("Found field '%s' but expected %s", dbFieldName, columnInfoMap.keySet()));
+ prepareStatement.setObject(parameterIndex++, pair.getValue(), info.sqlType);
+ LOG.debug("Setting {} to {}", pair.getKey(), pair.getValue());
+ }
+ return prepareStatement;
+ }
+
+ public PreparedStatement createCountStatement(Connection connection,
+ Class<? extends AbstractIndexEntity> indexEntityClass) throws Exception {
+ String tableName =
+ entityMappingHolder.tableToEntityNameMap.inverse().get(indexEntityClass.getSimpleName());
+ String sql = "Select count(*) from " + tableName;
+ PreparedStatement prepareStatement = connection.prepareStatement(sql);
+ return prepareStatement;
+ }
+
private void generateWhereClause(BiMap<String, String> entityNameToDBNameMapping,
Predicate predicate, List<Pair<String, Object>> parametersList, StringBuilder whereClause) {
String columnName = null;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
index 6c0637f..e861cec 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/DetectionResource.java
@@ -85,7 +85,6 @@ import org.apache.pinot.thirdeye.formatter.DetectionAlertConfigFormatter;
import org.apache.pinot.thirdeye.formatter.DetectionConfigFormatter;
import org.apache.pinot.thirdeye.rootcause.impl.MetricEntity;
import org.apache.pinot.thirdeye.util.AnomalyOffset;
-import org.h2.command.dml.Merge;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.Interval;
diff --git a/thirdeye/thirdeye-pinot/src/main/resources/schema/create-schema.sql b/thirdeye/thirdeye-pinot/src/main/resources/schema/create-schema.sql
index 00e6180..c93b922 100644
--- a/thirdeye/thirdeye-pinot/src/main/resources/schema/create-schema.sql
+++ b/thirdeye/thirdeye-pinot/src/main/resources/schema/create-schema.sql
@@ -380,6 +380,8 @@ create index session_principal_type_idx ON session_index(principal_type);
create table if not exists detection_config_index (
base_id bigint(20) not null,
`name` VARCHAR(256) not null,
+ active BOOLEAN,
+ created_by VARCHAR(256),
create_time timestamp,
update_time timestamp default current_timestamp,
version int(10)
@@ -387,6 +389,8 @@ create table if not exists detection_config_index (
ALTER TABLE `detection_config_index` ADD UNIQUE `detection_config_unique_index`(`name`);
create index detection_config_base_id_idx ON detection_config_index(base_id);
create index detection_config_name_idx ON detection_config_index(`name`);
+create index detection_config_active_idx ON detection_config_index(active);
+create index detection_config_created_by_index ON detection_config_index(created_by);
create table if not exists detection_alert_config_index (
base_id bigint(20) not null,
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcherTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcherTest.java
new file mode 100644
index 0000000..3e25461
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/dashboard/resources/v2/alerts/AlertSearcherTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.pinot.thirdeye.dashboard.resources.v2.alerts;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.pinot.thirdeye.datalayer.bao.DAOTestBase;
+import org.apache.pinot.thirdeye.datalayer.bao.DetectionConfigManager;
+import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
+import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+
+public class AlertSearcherTest {
+ private DAOTestBase testDAOProvider;
+
+ @BeforeMethod
+ public void setUp() {
+ testDAOProvider = DAOTestBase.getInstance();
+ DetectionConfigManager detectionDAO = DAORegistry.getInstance().getDetectionConfigManager();
+ DetectionConfigDTO detectionConfig1 = new DetectionConfigDTO();
+ detectionConfig1.setName("test_detection1");
+ DetectionConfigDTO detectionConfig2 = new DetectionConfigDTO();
+ detectionConfig2.setName("test_detection2");
+ DetectionConfigDTO detectionConfig3 = new DetectionConfigDTO();
+ detectionConfig3.setName("test_detection3");
+ detectionConfig3.setCreatedBy("test@example.com");
+ DetectionConfigDTO detectionConfig4 = new DetectionConfigDTO();
+ detectionConfig4.setActive(true);
+ detectionConfig4.setName("test_detection4");
+
+ detectionDAO.save(detectionConfig1);
+ detectionDAO.save(detectionConfig2);
+ detectionDAO.save(detectionConfig3);
+ detectionDAO.save(detectionConfig4);
+ }
+
+ @Test
+ public void testSearch() {
+ AlertSearcher searcher = new AlertSearcher();
+ Map<String, Object> result = searcher.search(new AlertSearchFilter(), 10 ,0);
+ Assert.assertEquals(result.get("count"), 4L);
+ Assert.assertEquals(result.get("limit"), 10L);
+ Assert.assertEquals(result.get("offset"), 0L);
+ }
+
+ @Test
+ public void testSearchActive() {
+ AlertSearcher searcher = new AlertSearcher();
+ Map<String, Object> result = searcher.search(new AlertSearchFilter(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), true), 10 ,0);
+ Assert.assertEquals(result.get("count"), 1L);
+ Assert.assertEquals(result.get("limit"), 10L);
+ Assert.assertEquals(result.get("offset"), 0L);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org