You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/07/24 03:40:24 UTC
[incubator-pinot] 01/01: support optional 'name' field in data
source config proeprties
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch support-multiple-datasources-per-type
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 02d758332d0e7b8650bb3b20670f3f374c13c4f9
Author: Alexander Pucher <al...@alexpucher.com>
AuthorDate: Thu Jul 23 20:40:03 2020 -0700
support optional 'name' field in data source config proeprties
---
.../thirdeye/datasource/DataSourceConfig.java | 9 ++++++
.../thirdeye/datasource/DataSourcesLoader.java | 3 +-
.../datasource/csv/CSVThirdEyeDataSource.java | 10 +++++-
.../datasource/mock/MockThirdEyeDataSource.java | 37 ++++++++++------------
.../datasource/pinot/PinotThirdEyeDataSource.java | 6 +++-
.../pinot/PinotThirdEyeDataSourceConfig.java | 30 ++++++++++++++----
.../datasource/sql/SqlThirdEyeDataSource.java | 6 +++-
7 files changed, 69 insertions(+), 32 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
index d7611cc..66552ed 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
@@ -32,10 +32,19 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
*/
public class DataSourceConfig {
+ private String name;
private String className;
private Map<String, Object> properties = new HashMap<>();
private List<MetadataSourceConfig> metadataSourceConfigs = new ArrayList<>();
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getClassName() {
return className;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
index 18cb04d..5d57d8e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
@@ -71,8 +71,7 @@ public class DataSourcesLoader {
// use class simple name as key, this enforces that there cannot be more than one data source of the same type
String name = thirdeyeDataSource.getName();
if (dataSourceMap.containsKey(name)) {
- throw new IllegalStateException("Data source " + name + " already exists. "
- + "There can be only ONE datasource of each type");
+ throw new IllegalStateException("Data source " + name + " already exists.");
}
dataSourceMap.put(name, thirdeyeDataSource);
} catch (Exception e) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
index 2f1b3ed..ec488f3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
@@ -22,6 +22,7 @@ package org.apache.pinot.thirdeye.datasource.csv;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.constant.MetricAggFunction;
@@ -75,6 +76,11 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
TranslateDelegator translator;
/**
+ * DataSource name
+ */
+ String name;
+
+ /**
* Factory method of CSVThirdEyeDataSource. Construct a CSVThirdEyeDataSource using data frames.
*
* @param dataSets the data sets
@@ -114,6 +120,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
CSVThirdEyeDataSource(Map<String, DataFrame> dataSets, Map<Long, String> metricNameMap) {
this.dataSets = dataSets;
this.translator = new StaticTranslator(metricNameMap);
+ this.name = CSVThirdEyeDataSource.class.getSimpleName();
}
/**
@@ -132,6 +139,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
this.dataSets = dataframes;
this.translator = new DAOTranslator();
+ this.name = MapUtils.getString(properties, "name", CSVThirdEyeDataSource.class.getSimpleName());
}
/**
@@ -140,7 +148,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
*/
@Override
public String getName() {
- return CSVThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
/**
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
index 0731e84..a3abded 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
@@ -22,34 +22,21 @@ package org.apache.pinot.thirdeye.datasource.mock;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
-import java.io.File;
-import java.util.Scanner;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.StringSeries;
import org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
import org.apache.pinot.thirdeye.datasource.ThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeRequest;
-import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeResponse;
+import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.sql.SqlDataset;
import org.apache.pinot.thirdeye.datasource.sql.SqlUtils;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -60,7 +47,13 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_TIME;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_VALUE;
import static org.apache.pinot.thirdeye.datasource.sql.SqlResponseCacheLoader.*;
@@ -85,6 +78,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
final Map<String, DataFrame> datasetData;
final Map<Long, String> metricNameMap;
+ final String name;
+
final CSVThirdEyeDataSource delegate;
/**
@@ -94,6 +89,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
* @throws Exception if properties cannot be parsed
*/
public MockThirdEyeDataSource(Map<String, Object> properties) throws Exception {
+ this.name = MapUtils.getString(properties, "name", MockThirdEyeDataSource.class.getSimpleName());
+
// datasets
this.datasets = new HashMap<>();
Map<String, Object> config = ConfigUtils.getMap(properties.get(DATASETS));
@@ -292,7 +289,7 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return MockThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
index b3635f2..3e02df2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.anomaly.utils.ThirdeyeMetricsUtil;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
@@ -59,6 +60,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
public static final String DATA_SOURCE_NAME = PinotThirdEyeDataSource.class.getSimpleName();
private static final String PINOT = "Pinot";
+ private String name;
private static final long CONNECTION_TIMEOUT = 60000;
@@ -81,6 +83,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = pinotThirdEyeDataSourceConfig.getName() != null ? pinotThirdEyeDataSourceConfig.getName() : DATA_SOURCE_NAME;
}
@@ -98,6 +101,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = MapUtils.getString(properties, "name", DATA_SOURCE_NAME);
}
/**
@@ -133,7 +137,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return DATA_SOURCE_NAME;
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
index c1db892..70a5109 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
@@ -23,17 +23,17 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
-import org.apache.pinot.thirdeye.datasource.DataSourceConfig;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* An immutable configurations for setting up {@link PinotThirdEyeDataSource}'s connection to Pinot.
@@ -51,6 +51,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public String getZookeeperUrl() {
return zookeeperUrl;
@@ -84,6 +85,14 @@ public class PinotThirdEyeDataSourceConfig {
this.tag = tag;
}
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getBrokerUrl() {
return brokerUrl;
}
@@ -128,14 +137,14 @@ public class PinotThirdEyeDataSourceConfig {
@Override
public int hashCode() {
return Objects.hash(getZookeeperUrl(), getControllerHost(), getControllerPort(), getControllerConnectionScheme(),
- getClusterName(), getBrokerUrl(), getTag());
+ getClusterName(), getBrokerUrl(), getTag(), getName());
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("zookeeperUrl", zookeeperUrl).add("controllerHost", controllerHost)
.add("controllerPort", controllerPort).add("controllerConnectionScheme", controllerConnectionScheme)
- .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).toString();
+ .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).add("name", name).toString();
}
public static Builder builder() {
@@ -150,6 +159,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public Builder setZookeeperUrl(String zookeeperUrl) {
this.zookeeperUrl = zookeeperUrl;
@@ -181,6 +191,11 @@ public class PinotThirdEyeDataSourceConfig {
return this;
}
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
public Builder setControllerConnectionScheme(String controllerConnectionScheme) {
this.controllerConnectionScheme = controllerConnectionScheme;
return this;
@@ -204,6 +219,7 @@ public class PinotThirdEyeDataSourceConfig {
config.setBrokerUrl(brokerUrl);
config.setTag(tag);
config.setControllerConnectionScheme(controllerConnectionScheme);
+ config.setName(name);
return config;
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
index 1c5d114..675bf0d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
@@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datasource.MetricFunction;
@@ -45,15 +47,17 @@ public class SqlThirdEyeDataSource implements ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
protected LoadingCache<RelationalQuery, ThirdEyeResultSetGroup> sqlResponseCache;
private SqlResponseCacheLoader sqlResponseCacheLoader;
+ private String name;
public SqlThirdEyeDataSource(Map<String, Object> properties) throws Exception {
sqlResponseCacheLoader = new SqlResponseCacheLoader(properties);
sqlResponseCache = ThirdEyeUtils.buildResponseCache(sqlResponseCacheLoader);
+ name = MapUtils.getString(properties, "name", SqlThirdEyeDataSource.class.getSimpleName());
}
@Override
public String getName() {
- return SqlThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org