You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ap...@apache.org on 2018/11/13 23:11:23 UTC
[incubator-pinot] branch master updated: [TE] datasource - update
dataset if underline pinot table time field changed (#3456)
This is an automated email from the ASF dual-hosted git repository.
apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3bd8fec [TE] datasource - update dataset if underline pinot table time field changed (#3456)
3bd8fec is described below
commit 3bd8fec33be7f4473a7fbbcbb135c3ac5b963214
Author: Xiaohui Sun <xh...@linkedin.com>
AuthorDate: Tue Nov 13 15:11:19 2018 -0800
[TE] datasource - update dataset if underline pinot table time field changed (#3456)
---
.../auto/onboard/AutoOnboardPinotMetadataSource.java | 16 +++++++++++++++-
.../thirdeye/auto/onboard/ConfigGenerator.java | 20 ++++++++++++--------
.../onboard/AutoOnboardPinotMetricsServiceTest.java | 14 ++++++++++++++
3 files changed, 41 insertions(+), 9 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
index 7a71c8e..b915a03 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetadataSource.java
@@ -17,11 +17,11 @@
package com.linkedin.thirdeye.auto.onboard;
import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.CaseFormat;
import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.linkedin.pinot.common.data.MetricFieldSpec;
import com.linkedin.pinot.common.data.Schema;
+import com.linkedin.pinot.common.data.TimeGranularitySpec;
import com.linkedin.thirdeye.datalayer.bao.AlertConfigManager;
import com.linkedin.thirdeye.datalayer.bao.DatasetConfigManager;
import com.linkedin.thirdeye.datalayer.bao.MetricConfigManager;
@@ -211,6 +211,7 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
DatasetConfigDTO datasetConfig) throws Exception {
checkDimensionChanges(dataset, datasetConfig, schema);
checkMetricChanges(dataset, datasetConfig, schema);
+ checkTimeFieldChanges(datasetConfig, schema);
appendNewCustomConfigs(datasetConfig, customConfigs);
}
@@ -330,6 +331,19 @@ public class AutoOnboardPinotMetadataSource extends AutoOnboard {
}
+ private void checkTimeFieldChanges(DatasetConfigDTO datasetConfig, Schema schema) {
+ TimeGranularitySpec timeSpec = schema.getTimeFieldSpec().getOutgoingGranularitySpec();
+ if (!datasetConfig.getTimeColumn().equals(timeSpec.getName())
+ || !datasetConfig.getTimeFormat().equals(timeSpec.getTimeFormat())
+ || datasetConfig.bucketTimeGranularity().getUnit() != timeSpec.getTimeType()
+ || datasetConfig.bucketTimeGranularity().getSize() != timeSpec.getTimeUnitSize()) {
+ ConfigGenerator.setTimeSpecs(datasetConfig, timeSpec);
+ DAO_REGISTRY.getDatasetConfigDAO().update(datasetConfig);
+ LOG.info("Refreshed time field. name = {}, format = {}, type = {}, unit size = {}.",
+ timeSpec.getName(), timeSpec.getTimeType(), timeSpec.getTimeType(), timeSpec.getTimeUnitSize());
+ }
+ }
+
/**
* This method ensures that the given custom configs exist in the dataset config and their value are the same.
*
diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/ConfigGenerator.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/ConfigGenerator.java
index e5ef479..96bcaf8 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/ConfigGenerator.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/auto/onboard/ConfigGenerator.java
@@ -39,6 +39,17 @@ public class ConfigGenerator {
private static final String PDT_TIMEZONE = "US/Pacific";
private static final String BYTES_STRING = "BYTES";
+ public static void setTimeSpecs(DatasetConfigDTO datasetConfigDTO, TimeGranularitySpec timeSpec) {
+ datasetConfigDTO.setTimeColumn(timeSpec.getName());
+ datasetConfigDTO.setTimeDuration(timeSpec.getTimeUnitSize());
+ datasetConfigDTO.setTimeUnit(timeSpec.getTimeType());
+ datasetConfigDTO.setTimeFormat(timeSpec.getTimeFormat());
+ datasetConfigDTO.setExpectedDelay(getExpectedDelayFromTimeunit(timeSpec.getTimeType()));
+ if (timeSpec.getTimeFormat().startsWith(TimeFormat.SIMPLE_DATE_FORMAT.toString())) {
+ datasetConfigDTO.setTimezone(PDT_TIMEZONE);
+ }
+ }
+
public static DatasetConfigDTO generateDatasetConfig(String dataset, Schema schema,
Map<String, String> customConfigs) {
List<String> dimensions = schema.getDimensionNames();
@@ -48,14 +59,7 @@ public class ConfigGenerator {
DatasetConfigDTO datasetConfigDTO = new DatasetConfigDTO();
datasetConfigDTO.setDataset(dataset);
datasetConfigDTO.setDimensions(dimensions);
- datasetConfigDTO.setTimeColumn(timeSpec.getName());
- datasetConfigDTO.setTimeDuration(timeSpec.getTimeUnitSize());
- datasetConfigDTO.setTimeUnit(timeSpec.getTimeType());
- datasetConfigDTO.setTimeFormat(timeSpec.getTimeFormat());
- datasetConfigDTO.setExpectedDelay(getExpectedDelayFromTimeunit(timeSpec.getTimeType()));
- if (timeSpec.getTimeFormat().startsWith(TimeFormat.SIMPLE_DATE_FORMAT.toString())) {
- datasetConfigDTO.setTimezone(PDT_TIMEZONE);
- }
+ setTimeSpecs(datasetConfigDTO, timeSpec);
datasetConfigDTO.setDataSource(PinotThirdEyeDataSource.DATA_SOURCE_NAME);
datasetConfigDTO.setProperties(customConfigs);
return datasetConfigDTO;
diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
index 2ebacb9..4c28d7d 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/auto/onboard/AutoOnboardPinotMetricsServiceTest.java
@@ -21,6 +21,7 @@ import com.linkedin.pinot.common.data.DimensionFieldSpec;
import com.linkedin.pinot.common.data.FieldSpec.DataType;
import com.linkedin.pinot.common.data.MetricFieldSpec;
import com.linkedin.pinot.common.data.Schema;
+import com.linkedin.pinot.common.data.TimeFieldSpec;
import com.linkedin.pinot.common.data.TimeGranularitySpec;
import com.linkedin.thirdeye.api.MetricType;
import com.linkedin.thirdeye.datalayer.bao.DAOTestBase;
@@ -140,6 +141,19 @@ public class AutoOnboardPinotMetricsServiceTest {
Assert.assertTrue(datasetCustomConfigs.containsKey(configKey));
Assert.assertEquals(datasetCustomConfigs.get(configKey), configValue);
}
+
+ TimeFieldSpec timeFieldSpec = new TimeFieldSpec("timestampInEpoch", DataType.LONG, TimeUnit.MILLISECONDS);
+ schema.removeField(schema.getTimeColumnName());
+ schema.addField(timeFieldSpec);
+ testAutoLoadPinotMetricsService.addPinotDataset(dataset, schema, new HashMap<>(pinotCustomConfigs), newDatasetConfig1);
+ Assert.assertEquals(datasetConfigDAO.findAll().size(), 1);
+ datasetConfig = datasetConfigDAO.findByDataset(dataset);
+ TimeGranularitySpec timeGranularitySpec = schema.getTimeFieldSpec().getOutgoingGranularitySpec();
+ Assert.assertEquals(datasetConfig.bucketTimeGranularity().getUnit(), timeGranularitySpec.getTimeType());
+ Assert.assertEquals(datasetConfig.bucketTimeGranularity().getSize(), timeGranularitySpec.getTimeUnitSize());
+ Assert.assertEquals(datasetConfig.getTimeFormat(), timeGranularitySpec.getTimeFormat());
+ Assert.assertEquals(datasetConfig.getTimezone(), "UTC");
+ Assert.assertEquals(datasetConfig.getExpectedDelay().getUnit(), TimeUnit.HOURS);
}
@Test (dependsOnMethods={"testRefreshDataset"})
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org