[incubator-pinot] branch master updated: [TE] datasource - aggregate to one minute level for epoch timestamp (#3530)

[ap...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

apucher pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 77e394d   [TE] datasource - aggregate to one minute level for epoch timestamp (#3530)
77e394d is described below

commit 77e394dfdc39a15916f957ce4124eec41113218e
Author: Xiaohui Sun <xh...@linkedin.com>
AuthorDate: Tue Nov 20 14:02:54 2018 -0800

     [TE] datasource - aggregate to one minute level for epoch timestamp (#3530)
    
    Pinot uses UTC time which causes metrics shown inconsistently in ThirdEye which is PST/PDT by default. This aggregates to one minute level instead of the actual aggregation granularity.
---
 .../com/linkedin/thirdeye/datasource/pinot/PqlUtils.java    | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
index 39c1cf1..11281a0 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datasource/pinot/PqlUtils.java
@@ -348,11 +348,10 @@ public class PqlUtils {
     return AND.join(components);
   }
 
-  private static String convertEpochToAggGranularity(String timeColumnName, TimeSpec timeSpec, TimeGranularity aggregationGranularity) {
-    String groupByTimeColumnName = String.format("dateTimeConvert(%s,'%d:%s:%s','%d:%s:%s','%d:%s')", timeColumnName,
+  private static String convertEpochToMinuteAggGranularity(String timeColumnName, TimeSpec timeSpec) {
+    String groupByTimeColumnName = String.format("dateTimeConvert(%s,'%d:%s:%s','%d:%s:%s','1:MINUTES')", timeColumnName,
         timeSpec.getDataGranularity().getSize(), timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat(),
-        timeSpec.getDataGranularity().getSize(), timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat(),
-        aggregationGranularity.getSize(), aggregationGranularity.getUnit());
+        timeSpec.getDataGranularity().getSize(), timeSpec.getDataGranularity().getUnit(), timeSpec.getFormat());
     return groupByTimeColumnName;
   }
 
@@ -361,11 +360,11 @@ public class PqlUtils {
     String timeColumnName = timeSpec.getColumnName();
     List<String> groups = new LinkedList<>();
     if (aggregationGranularity != null && !groups.contains(timeColumnName)) {
-      // Convert the time column to aggregation granularity if it is epoch.
-      // E.g., dateTimeConvert(timestampInEpoch,'1:MILLISECONDS:EPOCH','1:MILLISECONDS:EPOCH','15:MINUTES')
+      // Convert the time column to 1 minute granularity if it is epoch.
+      // E.g., dateTimeConvert(timestampInEpoch,'1:MILLISECONDS:EPOCH','1:MILLISECONDS:EPOCH','1:MINUTES')
       if (timeSpec.getFormat().equals(TimeGranularitySpec.TimeFormat.EPOCH.toString())
           && !timeSpec.getDataGranularity().equals(aggregationGranularity)) {
-        String groupByTimeColumnName = convertEpochToAggGranularity(timeColumnName, timeSpec, aggregationGranularity);
+        String groupByTimeColumnName = convertEpochToMinuteAggGranularity(timeColumnName, timeSpec);
         groups.add(groupByTimeColumnName);
       } else {
         groups.add(timeColumnName);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org