You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2020/07/24 03:40:23 UTC
[incubator-pinot] branch support-multiple-datasources-per-type
created (now 02d7583)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a change to branch support-multiple-datasources-per-type
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 02d7583 support optional 'name' field in data source config proeprties
This branch includes the following new commits:
new 02d7583 support optional 'name' field in data source config proeprties
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: support optional 'name' field in data
source config proeprties
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch support-multiple-datasources-per-type
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 02d758332d0e7b8650bb3b20670f3f374c13c4f9
Author: Alexander Pucher <al...@alexpucher.com>
AuthorDate: Thu Jul 23 20:40:03 2020 -0700
support optional 'name' field in data source config proeprties
---
.../thirdeye/datasource/DataSourceConfig.java | 9 ++++++
.../thirdeye/datasource/DataSourcesLoader.java | 3 +-
.../datasource/csv/CSVThirdEyeDataSource.java | 10 +++++-
.../datasource/mock/MockThirdEyeDataSource.java | 37 ++++++++++------------
.../datasource/pinot/PinotThirdEyeDataSource.java | 6 +++-
.../pinot/PinotThirdEyeDataSourceConfig.java | 30 ++++++++++++++----
.../datasource/sql/SqlThirdEyeDataSource.java | 6 +++-
7 files changed, 69 insertions(+), 32 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
index d7611cc..66552ed 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourceConfig.java
@@ -32,10 +32,19 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
*/
public class DataSourceConfig {
+ private String name;
private String className;
private Map<String, Object> properties = new HashMap<>();
private List<MetadataSourceConfig> metadataSourceConfigs = new ArrayList<>();
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getClassName() {
return className;
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
index 18cb04d..5d57d8e 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/DataSourcesLoader.java
@@ -71,8 +71,7 @@ public class DataSourcesLoader {
// use class simple name as key, this enforces that there cannot be more than one data source of the same type
String name = thirdeyeDataSource.getName();
if (dataSourceMap.containsKey(name)) {
- throw new IllegalStateException("Data source " + name + " already exists. "
- + "There can be only ONE datasource of each type");
+ throw new IllegalStateException("Data source " + name + " already exists.");
}
dataSourceMap.put(name, thirdeyeDataSource);
} catch (Exception e) {
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
index 2f1b3ed..ec488f3 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/csv/CSVThirdEyeDataSource.java
@@ -22,6 +22,7 @@ package org.apache.pinot.thirdeye.datasource.csv;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeGranularity;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.constant.MetricAggFunction;
@@ -75,6 +76,11 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
TranslateDelegator translator;
/**
+ * DataSource name
+ */
+ String name;
+
+ /**
* Factory method of CSVThirdEyeDataSource. Construct a CSVThirdEyeDataSource using data frames.
*
* @param dataSets the data sets
@@ -114,6 +120,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
CSVThirdEyeDataSource(Map<String, DataFrame> dataSets, Map<Long, String> metricNameMap) {
this.dataSets = dataSets;
this.translator = new StaticTranslator(metricNameMap);
+ this.name = CSVThirdEyeDataSource.class.getSimpleName();
}
/**
@@ -132,6 +139,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
this.dataSets = dataframes;
this.translator = new DAOTranslator();
+ this.name = MapUtils.getString(properties, "name", CSVThirdEyeDataSource.class.getSimpleName());
}
/**
@@ -140,7 +148,7 @@ public class CSVThirdEyeDataSource implements ThirdEyeDataSource {
*/
@Override
public String getName() {
- return CSVThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
/**
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
index 0731e84..a3abded 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/mock/MockThirdEyeDataSource.java
@@ -22,34 +22,21 @@ package org.apache.pinot.thirdeye.datasource.mock;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Predicate;
import com.google.common.collect.Collections2;
-import java.io.File;
-import java.util.Scanner;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.pinot.thirdeye.dataframe.DataFrame;
import org.apache.pinot.thirdeye.dataframe.StringSeries;
import org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
import org.apache.pinot.thirdeye.datasource.ThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeRequest;
-import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.ThirdEyeResponse;
+import org.apache.pinot.thirdeye.datasource.csv.CSVThirdEyeDataSource;
import org.apache.pinot.thirdeye.datasource.sql.SqlDataset;
import org.apache.pinot.thirdeye.datasource.sql.SqlUtils;
import org.apache.pinot.thirdeye.detection.ConfigUtils;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.TimeUnit;
-import javax.annotation.Nullable;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.ArrayUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.math3.distribution.NormalDistribution;
import org.apache.tomcat.jdbc.pool.DataSource;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
@@ -60,7 +47,13 @@ import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.*;
+import javax.annotation.Nullable;
+import java.io.File;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_TIME;
+import static org.apache.pinot.thirdeye.dataframe.util.DataFrameUtils.COL_VALUE;
import static org.apache.pinot.thirdeye.datasource.sql.SqlResponseCacheLoader.*;
@@ -85,6 +78,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
final Map<String, DataFrame> datasetData;
final Map<Long, String> metricNameMap;
+ final String name;
+
final CSVThirdEyeDataSource delegate;
/**
@@ -94,6 +89,8 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
* @throws Exception if properties cannot be parsed
*/
public MockThirdEyeDataSource(Map<String, Object> properties) throws Exception {
+ this.name = MapUtils.getString(properties, "name", MockThirdEyeDataSource.class.getSimpleName());
+
// datasets
this.datasets = new HashMap<>();
Map<String, Object> config = ConfigUtils.getMap(properties.get(DATASETS));
@@ -292,7 +289,7 @@ public class MockThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return MockThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
index b3635f2..3e02df2 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSource.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections4.MapUtils;
import org.apache.pinot.thirdeye.anomaly.utils.ThirdeyeMetricsUtil;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
@@ -59,6 +60,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
public static final String DATA_SOURCE_NAME = PinotThirdEyeDataSource.class.getSimpleName();
private static final String PINOT = "Pinot";
+ private String name;
private static final long CONNECTION_TIMEOUT = 60000;
@@ -81,6 +83,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = pinotThirdEyeDataSourceConfig.getName() != null ? pinotThirdEyeDataSourceConfig.getName() : DATA_SOURCE_NAME;
}
@@ -98,6 +101,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
pinotDataSourceTimeQuery = new PinotDataSourceTimeQuery(this);
pinotDataSourceDimensionFilters = new PinotDataSourceDimensionFilters(this);
+ name = MapUtils.getString(properties, "name", DATA_SOURCE_NAME);
}
/**
@@ -133,7 +137,7 @@ public class PinotThirdEyeDataSource implements ThirdEyeDataSource {
@Override
public String getName() {
- return DATA_SOURCE_NAME;
+ return this.name;
}
@Override
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
index c1db892..70a5109 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/pinot/PinotThirdEyeDataSourceConfig.java
@@ -23,17 +23,17 @@ import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.thirdeye.auto.onboard.AutoOnboardPinotMetadataSource;
-import org.apache.pinot.thirdeye.datasource.DataSourceConfig;
import org.apache.pinot.thirdeye.datasource.MetadataSourceConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import org.apache.commons.collections4.MapUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* An immutable configurations for setting up {@link PinotThirdEyeDataSource}'s connection to Pinot.
@@ -51,6 +51,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public String getZookeeperUrl() {
return zookeeperUrl;
@@ -84,6 +85,14 @@ public class PinotThirdEyeDataSourceConfig {
this.tag = tag;
}
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
public String getBrokerUrl() {
return brokerUrl;
}
@@ -128,14 +137,14 @@ public class PinotThirdEyeDataSourceConfig {
@Override
public int hashCode() {
return Objects.hash(getZookeeperUrl(), getControllerHost(), getControllerPort(), getControllerConnectionScheme(),
- getClusterName(), getBrokerUrl(), getTag());
+ getClusterName(), getBrokerUrl(), getTag(), getName());
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this).add("zookeeperUrl", zookeeperUrl).add("controllerHost", controllerHost)
.add("controllerPort", controllerPort).add("controllerConnectionScheme", controllerConnectionScheme)
- .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).toString();
+ .add("clusterName", clusterName).add("brokerUrl", brokerUrl).add("tag", tag).add("name", name).toString();
}
public static Builder builder() {
@@ -150,6 +159,7 @@ public class PinotThirdEyeDataSourceConfig {
private String clusterName;
private String brokerUrl;
private String tag;
+ private String name;
public Builder setZookeeperUrl(String zookeeperUrl) {
this.zookeeperUrl = zookeeperUrl;
@@ -181,6 +191,11 @@ public class PinotThirdEyeDataSourceConfig {
return this;
}
+ public Builder setName(String name) {
+ this.name = name;
+ return this;
+ }
+
public Builder setControllerConnectionScheme(String controllerConnectionScheme) {
this.controllerConnectionScheme = controllerConnectionScheme;
return this;
@@ -204,6 +219,7 @@ public class PinotThirdEyeDataSourceConfig {
config.setBrokerUrl(brokerUrl);
config.setTag(tag);
config.setControllerConnectionScheme(controllerConnectionScheme);
+ config.setName(name);
return config;
}
}
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
index 1c5d114..675bf0d 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/datasource/sql/SqlThirdEyeDataSource.java
@@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.commons.collections.MapUtils;
import org.apache.pinot.thirdeye.common.time.TimeSpec;
import org.apache.pinot.thirdeye.datalayer.dto.DatasetConfigDTO;
import org.apache.pinot.thirdeye.datasource.MetricFunction;
@@ -45,15 +47,17 @@ public class SqlThirdEyeDataSource implements ThirdEyeDataSource {
private static final ThirdEyeCacheRegistry CACHE_REGISTRY_INSTANCE = ThirdEyeCacheRegistry.getInstance();
protected LoadingCache<RelationalQuery, ThirdEyeResultSetGroup> sqlResponseCache;
private SqlResponseCacheLoader sqlResponseCacheLoader;
+ private String name;
public SqlThirdEyeDataSource(Map<String, Object> properties) throws Exception {
sqlResponseCacheLoader = new SqlResponseCacheLoader(properties);
sqlResponseCache = ThirdEyeUtils.buildResponseCache(sqlResponseCacheLoader);
+ name = MapUtils.getString(properties, "name", SqlThirdEyeDataSource.class.getSimpleName());
}
@Override
public String getName() {
- return SqlThirdEyeDataSource.class.getSimpleName();
+ return this.name;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org