You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/07/28 00:36:09 UTC
[incubator-pinot] branch master updated: [TE] support optional
'name' field in data source config (#5750)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c9975c5 [TE] support optional 'name' field in data source config (#5750)
c9975c5 is described below
commit c9975c5149e3ee94751df062bed49562185ebd2b
Author: Alexander Pucher <ap...@apache.org>
AuthorDate: Mon Jul 27 17:35:50 2020 -0700
[TE] support optional 'name' field in data source config (#5750)
## Description
This PR adds an optional "name" property to data source configs in order to allow users to specify multiple datasources of the same type (e.g. two pinot clusters). If the "name" property isn't provided, the implementation mimics legacy behavior.
## Upgrade Notes
Does this PR prevent a zero down-time upgrade? (Assume upgrade order: Controller, Broker, Server, Minion)
**No**
Does this PR fix a zero-downtime upgrade introduced earlier?
**No**
Does this PR otherwise need attention when creating release notes? Things to consider:
**No** change not relevant to pinot
---
.../onboard/AutoOnboardPinotMetadataSource.java | 8 ++--
.../thirdeye/auto/onboard/ConfigGenerator.java | 4 +-
.../thirdeye/datasource/DataSourceConfig.java | 9 +++++
.../thirdeye/datasource/DataSourcesLoader.java | 3 +-
.../pinot/thirdeye/datasource/ThirdEyeRequest.java | 2 +-
.../datasource/csv/CSVThirdEyeDataSource.java | 10 ++++-
.../datasource/mock/AutoOnboardMockDataSource.java | 4 +-
.../datasource/mock/MockThirdEyeDataSource.java | 37 +++++++++---------
.../datasource/pinot/PinotThirdEyeDataSource.java | 7 +++-
.../pinot/PinotThirdEyeDataSourceConfig.java | 44 +++++++++++++++-------
.../pinot/PinotThirdeyeDataSourceProperties.java | 3 +-
.../pinot/resources/PinotDataSourceResource.java | 2 +-
.../datasource/sql/SqlThirdEyeDataSource.java | 6 ++-
.../yaml/translator/DetectionConfigTranslator.java | 2 +-
.../AutoOnboardPinotMetricsServiceTest.java | 3 +-
.../pinot/thirdeye/datalayer/DaoTestUtils.java | 2 +-
.../thirdeye/detection/yaml/YamlResourceTest.java | 2 +-
.../DetectionConfigSlaTranslatorTest.java | 2 +-
.../translator/DetectionConfigTranslatorTest.java | 2 +-
19 files changed, 98 insertions(+), 54 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
index 4a52d01..ada1cdb 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
@@ -77,6 +77,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
private final AlertConfigManager alertDAO;
private final DatasetConfigManager datasetDAO;
private final MetricConfigManager metricDAO;
+ private final String dataSourceName;
private AutoOnboardPinotMetricsUtils autoLoadPinotMetricsUtils;
@@ -92,6 +93,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
this.datasetDAO = DAO_REGISTRY.getDatasetConfigDAO();
this.metricDAO = DAO_REGISTRY.getMetricConfigDAO();
this.alertDAO = DAO_REGISTRY.getAlertConfigDAO();
+ this.dataSourceName = MapUtils.getString(metadataSourceConfig.getProperties(), "name", PinotThirdEyeDataSource.class.getSimpleName());
}
public AutoOnboardPinotMetadataSource(MetadataSourceConfig metadataSourceConfig, AutoOnboardPinotMetricsUtils utils) {
@@ -100,6 +102,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
this.datasetDAO = DAO_REGISTRY.getDatasetConfigDAO();
this.metricDAO = DAO_REGISTRY.getMetricConfigDAO();
this.alertDAO = DAO_REGISTRY.getAlertConfigDAO();
+ this.dataSourceName = MapUtils.getString(metadataSourceConfig.getProperties(), "name", PinotThirdEyeDataSource.class.getSimpleName());
}
public void run() {
@@ -132,7 +135,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
Collection<DatasetConfigDTO> filtered = Collections2.filter(allExistingDataset, new com.google.common.base.Predicate<DatasetConfigDTO>() {
@Override
public boolean apply(@Nullable DatasetConfigDTO datasetConfigDTO) {
- return datasetConfigDTO.getDataSource().equals(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ return datasetConfigDTO.getDataSource().equals(AutoOnboardPinotMetadataSource.this.dataSourceName);
}
});
@@ -185,8 +188,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
List<MetricFieldSpec> metricSpecs = schema.getMetricFieldSpecs();
// Create DatasetConfig
- DatasetConfigDTO datasetConfigDTO =
- ConfigGenerator.generateDatasetConfig(dataset, schema, timeColumnName, customConfigs);
+ DatasetConfigDTO datasetConfigDTO = ConfigGenerator.generateDatasetConfig(dataset, schema, timeColumnName, customConfigs, this.dataSourceName);
LOG.info("Creating dataset for {}", dataset);
this.datasetDAO.save(datasetConfigDTO);
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/ConfigGenerator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/ConfigGenerator.java
index 65f55b0..692d38a 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/ConfigGenerator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/auto/onboard/ConfigGenerator.java
@@ -75,7 +75,7 @@ public class ConfigGenerator {
}
public static DatasetConfigDTO generateDatasetConfig(String dataset, Schema schema, String timeColumnName,
- Map<String, String> customConfigs) {
+ Map<String, String> customConfigs, String dataSourceName) {
List<String> dimensions = schema.getDimensionNames();
DateTimeFieldSpec dateTimeFieldSpec = schema.getSpecForTimeColumn(timeColumnName);
// Create DatasetConfig
@@ -83,7 +83,7 @@ public class ConfigGenerator {
datasetConfigDTO.setDataset(dataset);
datasetConfigDTO.setDimensions(dimensions);
setDateTimeSpecs(datasetConfigDTO, dateTimeFieldSpec);
- datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ datasetConfigDTO.setDataSource(dataSourceName);
datasetConfigDTO.setProperties(customConfigs);
checkNonAdditive(datasetConfigDTO);
return datasetConfigDTO;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
index d7611cc..66552ed 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
@@ -32,10 +32,19 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
*/
public class DataSourceConfig {
+ private String name;
private String className;
private Map<String, Object> properties = new HashMap<>();
private List<MetadataSourceConfig> metadataSourceConfigs = new ArrayList<>();
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getClassName() {
return className;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
index 18cb04d..5d57d8e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
@@ -71,8 +71,7 @@ public class DataSourcesLoader {
// use class simple name as key, this enforces that there cannot be more than one data source of the same type
String name = thirdeyeDataSource.getName();
if (dataSourceMap.containsKey(name)) {
- throw new IllegalStateException("Data source " + name + " already exists. "
- + "There can be only ONE datasource of each type");
+ throw new IllegalStateException("Data source " + name + " already exists.");
}
dataSourceMap.put(name, thirdeyeDataSource);
} catch (Exception e) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/ThirdEyeRequest.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/ThirdEyeRequest.java
index 23e7e02..deff5d5 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/ThirdEyeRequest.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/ThirdEyeRequest.java
@@ -168,7 +168,7 @@ public class ThirdEyeRequest {
private String filterClause;
private final List<String> groupBy;
private TimeGranularity groupByTimeGranularity;
- private String dataSource = PinotThirdEyeDataSource.DATA_SOURCE_NAME;
+ private String dataSource;
private int limit;
public ThirdEyeRequestBuilder() {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
index 2f1b3ed..ec488f3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
@@ -22,6 +22,7 @@ package org.apache.pinot.thirdeye.datasource.csv;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.constant.MetricAggFunction;
@@ -75,6 +76,11 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
TranslateDelegator translator;
/**
+ * DataSource name
+ */
+ String name;
+
+ /**
* Factory method of CSVThirdEyeDataSource. Construct a CSVThirdEyeDataSource using data frames.
*
* @param dataSets the data sets
@@ -114,6 +120,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
CSVThirdEyeDataSource(Map<String, DataFrame> dataSets, Map<Long, String> metricNameMap) {
this.dataSets = dataSets;
this.translator = new StaticTranslator(metricNameMap);
+ this.name = CSVThirdEyeDataSource.class.getSimpleName();
}
/**
@@ -132,6 +139,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
this.dataSets = dataframes;
this.translator = new DAOTranslator();
+ this.name = MapUtils.getString(properties, "name", CSVThirdEyeDataSource.class.getSimpleName());
}
/**
@@ -140,7 +148,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
*/
@Override
public String getName() {
- return CSVThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
/**
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/AutoOnboardMockDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/AutoOnboardMockDataSource.java
index 561d1c8..3d02044 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/AutoOnboardMockDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/AutoOnboardMockDataSource.java
@@ -47,6 +47,7 @@ public class AutoOnboardMockDataSource extends AutoOnboard {
private final MetricConfigManager metricDAO;
private final DatasetConfigManager datasetDAO;
+ private final String dataSourceName;
/**
* Constructor for dependency injection
@@ -57,6 +58,7 @@ public class AutoOnboardMockDataSource extends AutoOnboard {
super(metadataSourceConfig);
this.metricDAO = DAORegistry.getInstance().getMetricConfigDAO();
this.datasetDAO = DAORegistry.getInstance().getDatasetConfigDAO();
+ this.dataSourceName = MapUtils.getString(metadataSourceConfig.getProperties(), "name", MockThirdEyeDataSource.class.getSimpleName());
}
@Override
@@ -80,7 +82,7 @@ public class AutoOnboardMockDataSource extends AutoOnboard {
DatasetConfigDTO datasetConfig = new DatasetConfigDTO();
datasetConfig.setDataset(datasetName);
- datasetConfig.setDataSource(MockThirdEyeDataSource.class.getSimpleName());
+ datasetConfig.setDataSource(this.dataSourceName);
datasetConfig.setDimensions(sortedDimensions);
datasetConfig.setTimezone(MapUtils.getString(dataset, "timezone", "America/Los_Angeles"));
datasetConfig.setTimeDuration(ThirdEyeUtils.getTimeDuration(granularity));
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
index 0731e84..a3abded 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
@@ -22,34 +22,21 @@ package org.apache.pinot.thirdeye.datasource.mock;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
-import java.io.File;
-import java.util.Scanner;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.StringSeries;
import org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
import org.apache.pinot.thirdeye.datasource.ThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeRequest;
-import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeResponse;
+import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.sql.SqlDataset;
import org.apache.pinot.thirdeye.datasource.sql.SqlUtils;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -60,7 +47,13 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_TIME;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_VALUE;
import static org.apache.pinot.thirdeye.datasource.sql.SqlResponseCacheLoader.*;
@@ -85,6 +78,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
final Map<String, DataFrame> datasetData;
final Map<Long, String> metricNameMap;
+ final String name;
+
final CSVThirdEyeDataSource delegate;
/**
@@ -94,6 +89,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
* @throws Exception if properties cannot be parsed
*/
public MockThirdEyeDataSource(Map<String, Object> properties) throws Exception {
+ this.name = MapUtils.getString(properties, "name", MockThirdEyeDataSource.class.getSimpleName());
+
// datasets
this.datasets = new HashMap<>();
Map<String, Object> config = ConfigUtils.getMap(properties.get(DATASETS));
@@ -292,7 +289,7 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return MockThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
index b3635f2..461e2ba 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.anomaly.utils.ThirdeyeMetricsUtil;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
@@ -57,8 +58,8 @@ import org.slf4j.LoggerFactory;
public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
private static final Logger LOG = LoggerFactory.getLogger(PinotThirdEyeDataSource.class);
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
- public static final String DATA_SOURCE_NAME = PinotThirdEyeDataSource.class.getSimpleName();
private static final String PINOT = "Pinot";
+ private String name;
private static final long CONNECTION_TIMEOUT = 60000;
@@ -81,6 +82,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = pinotThirdEyeDataSourceConfig.getName() != null ? pinotThirdEyeDataSourceConfig.getName() : PinotThirdEyeDataSource.class.getSimpleName();
}
@@ -98,6 +100,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = MapUtils.getString(properties, "name", PinotThirdEyeDataSource.class.getSimpleName());
}
/**
@@ -133,7 +136,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return DATA_SOURCE_NAME;
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
index c1db892..4c01d8c 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
@@ -23,17 +23,17 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
-import org.apache.pinot.thirdeye.datasource.DataSourceConfig;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* An immutable configurations for setting up {@link PinotThirdEyeDataSource}'s connection to Pinot.
@@ -51,6 +51,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public String getZookeeperUrl() {
return zookeeperUrl;
@@ -84,6 +85,14 @@ public class PinotThirdEyeDataSourceConfig {
this.tag = tag;
}
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getBrokerUrl() {
return brokerUrl;
}
@@ -128,14 +137,14 @@ public class PinotThirdEyeDataSourceConfig {
@Override
public int hashCode() {
return Objects.hash(getZookeeperUrl(), getControllerHost(), getControllerPort(), getControllerConnectionScheme(),
- getClusterName(), getBrokerUrl(), getTag());
+ getClusterName(), getBrokerUrl(), getTag(), getName());
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("zookeeperUrl", zookeeperUrl).add("controllerHost", controllerHost)
.add("controllerPort", controllerPort).add("controllerConnectionScheme", controllerConnectionScheme)
- .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).toString();
+ .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).add("name", name).toString();
}
public static Builder builder() {
@@ -150,6 +159,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public Builder setZookeeperUrl(String zookeeperUrl) {
this.zookeeperUrl = zookeeperUrl;
@@ -181,6 +191,11 @@ public class PinotThirdEyeDataSourceConfig {
return this;
}
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
public Builder setControllerConnectionScheme(String controllerConnectionScheme) {
this.controllerConnectionScheme = controllerConnectionScheme;
return this;
@@ -204,6 +219,7 @@ public class PinotThirdEyeDataSourceConfig {
config.setBrokerUrl(brokerUrl);
config.setTag(tag);
config.setControllerConnectionScheme(controllerConnectionScheme);
+ config.setName(name);
return config;
}
}
@@ -234,12 +250,10 @@ public class PinotThirdEyeDataSourceConfig {
* controller host and port, cluster name, and the URL to zoo keeper.
*/
static PinotThirdEyeDataSourceConfig createFromProperties(Map<String, Object> properties) {
+ String dataSourceName = MapUtils.getString(properties, PinotThirdeyeDataSourceProperties.NAME.getValue(), PinotThirdEyeDataSource.class.getSimpleName());
+
ImmutableMap<String, Object> processedProperties = processPropertyMap(properties);
- if (processedProperties == null) {
- throw new IllegalArgumentException(
- "Invalid properties for data source: " + PinotThirdEyeDataSource.DATA_SOURCE_NAME + ", properties="
- + properties);
- }
+ Preconditions.checkNotNull(processedProperties, "Invalid properties for data source: %s, properties=%s", dataSourceName, properties);
String controllerHost = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.CONTROLLER_HOST.getValue());
int controllerPort = MapUtils.getInteger(processedProperties, PinotThirdeyeDataSourceProperties.CONTROLLER_PORT.getValue());
@@ -247,9 +261,10 @@ public class PinotThirdEyeDataSourceConfig {
String zookeeperUrl = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.ZOOKEEPER_URL.getValue());
String clusterName = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.CLUSTER_NAME.getValue());
- // brokerUrl and tag are optional
+ // brokerUrl, tag, and name are optional
String brokerUrl = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.BROKER_URL.getValue());
String tag = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.TAG.getValue());
+ String name = MapUtils.getString(processedProperties, PinotThirdeyeDataSourceProperties.NAME.getValue());
Builder builder =
PinotThirdEyeDataSourceConfig.builder().setControllerHost(controllerHost).setControllerPort(controllerPort)
@@ -260,6 +275,9 @@ public class PinotThirdEyeDataSourceConfig {
if (StringUtils.isNotBlank(tag)) {
builder.setTag(tag);
}
+ if (StringUtils.isNotBlank(name)) {
+ builder.setName(name);
+ }
if (StringUtils.isNotBlank(controllerConnectionScheme)) {
builder.setControllerConnectionScheme(controllerConnectionScheme);
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdeyeDataSourceProperties.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdeyeDataSourceProperties.java
index 16ae2c8..3a61e22 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdeyeDataSourceProperties.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdeyeDataSourceProperties.java
@@ -26,7 +26,8 @@ public enum PinotThirdeyeDataSourceProperties {
CLUSTER_NAME("clusterName"),
ZOOKEEPER_URL("zookeeperUrl"),
TAG("tag"),
- BROKER_URL("brokerUrl");
+ BROKER_URL("brokerUrl"),
+ NAME("name");
private final String value;
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resources/PinotDataSourceResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resources/PinotDataSourceResource.java
index 7982d4b..bbcabd1 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resources/PinotDataSourceResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/resources/PinotDataSourceResource.java
@@ -95,7 +95,7 @@ public class PinotDataSourceResource {
Preconditions.checkNotNull(ThirdEyeCacheRegistry.getInstance(),
"Failed to get Pinot data source because ThirdEye cache registry is not initialized.");
pinotDataSource = (PinotThirdEyeDataSource) ThirdEyeCacheRegistry.getInstance().getQueryCache()
- .getDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ .getDataSource(PinotThirdEyeDataSource.class.getSimpleName());
Preconditions
.checkNotNull(pinotDataSource, "Failed to get Pinot data source because it is not initialized in ThirdEye.");
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
index 1c5d114..675bf0d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
@@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datasource.MetricFunction;
@@ -45,15 +47,17 @@ public class SqlThirdEyeDataSource implements ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
protected LoadingCache<RelationalQuery, ThirdEyeResultSetGroup> sqlResponseCache;
private SqlResponseCacheLoader sqlResponseCacheLoader;
+ private String name;
public SqlThirdEyeDataSource(Map<String, Object> properties) throws Exception {
sqlResponseCacheLoader = new SqlResponseCacheLoader(properties);
sqlResponseCache = ThirdEyeUtils.buildResponseCache(sqlResponseCacheLoader);
+ name = MapUtils.getString(properties, "name", SqlThirdEyeDataSource.class.getSimpleName());
}
@Override
public String getName() {
- return SqlThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
index 7304b21..948702f 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslator.java
@@ -194,7 +194,7 @@ public class DetectionConfigTranslator extends ConfigTranslator<DetectionConfigD
List<DatasetConfigDTO> datasetConfigs = this.metricAttributesMap.getAllDatasets();
if (MapUtils.getString(yamlConfigMap, PROP_CRON) == null
&& datasetConfigs.stream().allMatch(c -> c.bucketTimeGranularity().getUnit().equals(TimeUnit.DAYS))
- && datasetConfigs.stream().allMatch(c -> c.getDataSource().equals(PinotThirdEyeDataSource.DATA_SOURCE_NAME))) {
+ && datasetConfigs.stream().allMatch(c -> c.getDataSource().equals(PinotThirdEyeDataSource.class.getSimpleName()))) {
config.setDataAvailabilitySchedule(true);
}
return config;
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
index 938e043..f12339d 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
@@ -37,6 +37,7 @@ import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datalayer.dto.MetricConfigDTO;
import org.apache.pinot.thirdeye.datalayer.pojo.MetricConfigBean;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
+import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
import org.testng.Assert;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
@@ -60,7 +61,7 @@ public class AutoOnboardPinotMetricsServiceTest {
DAORegistry daoRegistry = DAORegistry.getInstance();
datasetConfigDAO = daoRegistry.getDatasetConfigDAO();
metricConfigDAO = daoRegistry.getMetricConfigDAO();
- testAutoLoadPinotMetricsService = new AutoOnboardPinotMetadataSource(null, null);
+ testAutoLoadPinotMetricsService = new AutoOnboardPinotMetadataSource(new MetadataSourceConfig(), null);
schema = Schema.fromInputSteam(ClassLoader.getSystemResourceAsStream("sample-pinot-schema.json"));
Map<String, String> pinotCustomConfigs = new HashMap<>();
pinotCustomConfigs.put("configKey1", "configValue1");
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
index d9b9e79..af965a6 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/datalayer/DaoTestUtils.java
@@ -211,7 +211,7 @@ public class DaoTestUtils {
datasetConfigDTO.setTimeDuration(1);
datasetConfigDTO.setTimeUnit(TimeUnit.HOURS);
datasetConfigDTO.setActive(true);
- datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.class.getSimpleName());
datasetConfigDTO.setLastRefreshTime(System.currentTimeMillis());
return datasetConfigDTO;
}
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
index 3bffc18..6f9b5c3 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
@@ -118,7 +118,7 @@ public class YamlResourceTest {
datasetConfigDTO.setDataset("test_dataset");
datasetConfigDTO.setTimeUnit(TimeUnit.DAYS);
datasetConfigDTO.setTimeDuration(1);
- datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.class.getSimpleName());
daoRegistry.getDatasetConfigDAO().save(datasetConfigDTO);
// Create a new detection
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigSlaTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigSlaTranslatorTest.java
index db9087b..e6aa8bd 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigSlaTranslatorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigSlaTranslatorTest.java
@@ -58,7 +58,7 @@ public class DetectionConfigSlaTranslatorTest {
datasetConfigDTO.setDataset("test_dataset");
datasetConfigDTO.setTimeUnit(TimeUnit.DAYS);
datasetConfigDTO.setTimeDuration(1);
- datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.class.getSimpleName());
daoRegistry.getDatasetConfigDAO().save(datasetConfigDTO);
DetectionRegistry.registerComponent(DataSlaQualityChecker.class.getName(), "DATA_SLA");
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
index f09c533..8b852a7 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/translator/DetectionConfigTranslatorTest.java
@@ -62,7 +62,7 @@ public class DetectionConfigTranslatorTest {
datasetConfigDTO.setDataset("test_dataset");
datasetConfigDTO.setTimeUnit(TimeUnit.DAYS);
datasetConfigDTO.setTimeDuration(1);
- datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
+ datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.class.getSimpleName());
daoRegistry.getDatasetConfigDAO().save(datasetConfigDTO);
this.yaml = new Yaml();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org