You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ar...@apache.org on 2024/08/19 11:33:08 UTC
(superset) branch time_comparison_zeros updated (d0f1c6a49e -> fa06ed5177)
This is an automated email from the ASF dual-hosted git repository.
arivero pushed a change to branch time_comparison_zeros
in repository https://gitbox.apache.org/repos/asf/superset.git
discard d0f1c6a49e Time Comparison:
add a225f329c5 feat(docs): fix bug google chrome < 114 not found (#29941)
add 6e1ef193dd fix(sqllab): Add abort call on query refresh timeout (#29956)
add bac61fc24e chore: Allow auto pruning of the query table (#29936)
new fa06ed5177 Time Comparison:
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d0f1c6a49e)
\
N -- N -- N refs/heads/time_comparison_zeros (fa06ed5177)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/docs/configuration/alerts-reports.mdx | 13 ++-
.../SqlLab/components/QueryAutoRefresh/index.tsx | 17 +++-
superset/commands/sql_lab/query.py | 109 +++++++++++++++++++++
superset/config.py | 6 ++
superset/tasks/scheduler.py | 14 +++
5 files changed, 149 insertions(+), 10 deletions(-)
create mode 100644 superset/commands/sql_lab/query.py
(superset) 01/01: Time Comparison:
Posted by ar...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
arivero pushed a commit to branch time_comparison_zeros
in repository https://gitbox.apache.org/repos/asf/superset.git
commit fa06ed51773f959d3d3706995014dc30572ebff6
Author: Antonio Rivero <an...@gmail.com>
AuthorDate: Mon Aug 19 13:25:05 2024 +0200
Time Comparison:
- Support Human readable dates in the label and data call
- Handling inherit and dates as time_offsets in the backend
---
.../src/time-comparison/getTimeOffset.ts | 5 ++-
.../BigNumberPeriodOverPeriod/buildQuery.ts | 9 ++++-
.../BigNumberPeriodOverPeriod/transformProps.ts | 7 ++++
.../plugins/plugin-chart-table/src/buildQuery.ts | 10 +++++-
.../plugin-chart-table/src/transformProps.ts | 9 ++++-
.../components/controls/ComparisonRangeLabel.tsx | 20 ++++++++++-
.../components/controls/TimeOffsetControl.tsx | 16 ++++++++-
superset/common/query_context_processor.py | 42 +++++++++++++++++++++-
8 files changed, 111 insertions(+), 7 deletions(-)
diff --git a/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts b/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts
index 4d09d509a2..84047841e4 100644
--- a/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts
+++ b/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts
@@ -280,9 +280,12 @@ export const getTimeOffset = ({
const customShift =
customStartDateTime &&
+ filterStartDateTime &&
Math.round((filterStartDateTime - customStartDateTime) / DAY_IN_MS);
const inInheritShift =
isInherit &&
+ filterEndDateTime &&
+ filterStartDateTime &&
Math.round((filterEndDateTime - filterStartDateTime) / DAY_IN_MS);
const newShifts = ensureIsArray(shifts)
@@ -292,7 +295,7 @@ export const getTimeOffset = ({
if (includeFutureOffsets && customShift < 0) {
return `${customShift * -1} days after`;
}
- if (customShift >= 0) {
+ if (customShift >= 0 && filterStartDateTime) {
return `${customShift} days ago`;
}
}
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts
index ce75ebd13b..2c680b8310 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts
@@ -57,7 +57,7 @@ export default function buildQuery(formData: QueryFormData) {
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
}
- const timeOffsets = ensureIsArray(
+ let timeOffsets = ensureIsArray(
isTimeComparison(formData, baseQueryObject)
? getTimeOffset({
timeRangeFilter: {
@@ -74,6 +74,13 @@ export default function buildQuery(formData: QueryFormData) {
})
: [],
);
+ if (isEmpty(timeOffsets)) {
+ if (formData.time_compare && formData.time_compare === 'custom') {
+ timeOffsets = [formData.start_date_offset];
+ } else {
+ timeOffsets = ensureIsArray(formData.time_compare) || [];
+ }
+ }
return [
{
...baseQueryObject,
diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts
index fbac5f4303..3381e26a8a 100644
--- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts
+++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts
@@ -130,6 +130,13 @@ export default function transformProps(chartProps: ChartProps) {
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: startDateOffset,
});
+ if (isEmpty(dataOffset)) {
+ if (timeComparison && timeComparison === 'custom') {
+ dataOffset = [startDateOffset];
+ } else {
+ dataOffset = ensureIsArray(timeComparison) || [];
+ }
+ }
}
const { value1, value2 } = data.reduce(
diff --git a/superset-frontend/plugins/plugin-chart-table/src/buildQuery.ts b/superset-frontend/plugins/plugin-chart-table/src/buildQuery.ts
index 554914053c..a3c7c37937 100644
--- a/superset-frontend/plugins/plugin-chart-table/src/buildQuery.ts
+++ b/superset-frontend/plugins/plugin-chart-table/src/buildQuery.ts
@@ -107,7 +107,7 @@ const buildQuery: BuildQuery<TableChartFormData> = (
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
}
- const timeOffsets = ensureIsArray(
+ let timeOffsets = ensureIsArray(
isTimeComparison(formData, baseQueryObject)
? getTimeOffset({
timeRangeFilter: {
@@ -125,6 +125,14 @@ const buildQuery: BuildQuery<TableChartFormData> = (
: [],
);
+ if (isEmpty(timeOffsets)) {
+ if (formData.time_compare && formData.time_compare === 'custom') {
+ timeOffsets = [formData.start_date_offset];
+ } else {
+ timeOffsets = ensureIsArray(formData.time_compare) || [];
+ }
+ }
+
let temporalColumAdded = false;
let temporalColum = null;
diff --git a/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts b/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts
index 8c02d8293b..d11d7ed972 100644
--- a/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts
+++ b/superset-frontend/plugins/plugin-chart-table/src/transformProps.ts
@@ -615,7 +615,7 @@ const transformProps = (
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
}
- const timeOffsets = getTimeOffset({
+ let timeOffsets = getTimeOffset({
timeRangeFilter: {
...TimeRangeFilters[0],
comparator:
@@ -628,6 +628,13 @@ const transformProps = (
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: formData.start_date_offset,
});
+ if (isEmpty(timeOffsets)) {
+ if (formData.time_compare && formData.time_compare === 'custom') {
+ timeOffsets = [formData.start_date_offset];
+ } else {
+ timeOffsets = ensureIsArray(formData.time_compare) || [];
+ }
+ }
const comparisonSuffix = isUsingTimeComparison
? ensureIsArray(timeOffsets)[0]
: '';
diff --git a/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx b/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx
index 5df43256ad..41f08654f1 100644
--- a/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx
+++ b/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx
@@ -122,7 +122,25 @@ export const ComparisonRangeLabel = ({
ensureIsArray(newShifts),
);
}
- return Promise.resolve({ value: '' });
+ return fetchTimeRange(filter.comparator, filter.subject).then(res => {
+ const datePattern = /\d{4}-\d{2}-\d{2}/g;
+ const dates = res?.value?.match(datePattern);
+ const [startDate, endDate] = dates ?? [];
+ const postProcessedShifts = getTimeOffset({
+ timeRangeFilter: {
+ ...filter,
+ comparator: `${startDate} : ${endDate}`,
+ },
+ shifts: shiftsArray,
+ startDate: useStartDate,
+ includeFutureOffsets: false, // So we don't trigger requests for future dates
+ });
+ return fetchTimeRange(
+ filter.comparator,
+ filter.subject,
+ ensureIsArray(postProcessedShifts),
+ );
+ });
});
Promise.all(promises).then(res => {
// access the value property inside the res and set the labels with it in the state
diff --git a/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx b/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx
index 528e9dfac1..2cbf500e69 100644
--- a/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx
+++ b/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx
@@ -26,6 +26,7 @@ import {
css,
customTimeRangeDecode,
computeCustomDateTime,
+ fetchTimeRange,
} from '@superset-ui/core';
import { DatePicker } from 'antd';
import { RangePickerProps } from 'antd/lib/date-picker';
@@ -132,7 +133,20 @@ export default function TimeOffsetControls({
if (!isEmpty(currentTimeRangeFilters)) {
customTimeRange(currentTimeRangeFilters[0]?.comparator ?? '');
const date = currentTimeRangeFilters[0]?.comparator.split(' : ')[0];
- setFormatedFilterDate(moment(parseDttmToDate(date)));
+ const parsedDate = parseDttmToDate(date);
+ if (parsedDate) {
+ setFormatedFilterDate(moment(parseDttmToDate(date)));
+ } else {
+ fetchTimeRange(
+ currentTimeRangeFilters[0]?.comparator,
+ currentTimeRangeFilters[0]?.subject,
+ ).then(res => {
+ const datePattern = /\d{4}-\d{2}-\d{2}/g;
+ const dates = res?.value?.match(datePattern);
+ const [startDate, _] = dates ?? [];
+ setFormatedFilterDate(moment(parseDttmToDate(startDate)));
+ });
+ }
} else {
setCustomStartDateInFilter(undefined);
setFormatedFilterDate(moment(parseDttmToDate('')));
diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py
index 26935a4d96..ece73c348a 100644
--- a/superset/common/query_context_processor.py
+++ b/superset/common/query_context_processor.py
@@ -17,6 +17,7 @@
from __future__ import annotations
import copy
+from datetime import datetime
import logging
import re
from typing import Any, cast, ClassVar, TYPE_CHECKING, TypedDict
@@ -368,6 +369,38 @@ class QueryContextProcessor:
axis=1,
)
+ def is_valid_date(self, date_string: str):
+ try:
+ # Attempt to parse the string as a date in the format YYYY-MM-DD
+ datetime.strptime(date_string, "%Y-%m-%d")
+ return True
+ except ValueError:
+ # If parsing fails, it's not a valid date in the format YYYY-MM-DD
+ return False
+
+ def get_time_offset_for_custom_or_inherit(
+ self,
+ offset: str,
+ outer_from_dttm: datetime,
+ outer_to_dttm: datetime,
+ ) -> str:
+ """
+ Get the time offset for custom or inherit.
+
+ :param offset: The offset string.
+ :param outer_from_dttm: The outer from datetime.
+ :param outer_to_dttm: The outer to datetime.
+ :returns: The time offset.
+ """
+ if offset == "inherit":
+ # return the difference in days between the from and the to dttm formatted as a string with the " days ago" suffix
+ return f"{(outer_to_dttm - outer_from_dttm).days} days ago"
+ if self.is_valid_date(offset):
+ # return the offset as the difference in days between the outer from dttm and the offset date (which is a YYYY-MM-DD string) formatted as a string with the " days ago" suffix
+ offset_date = datetime.strptime(offset, "%Y-%m-%d")
+ return f"{(outer_from_dttm - offset_date).days} days ago"
+ return ""
+
def processing_time_offsets( # pylint: disable=too-many-locals,too-many-statements
self,
df: pd.DataFrame,
@@ -408,6 +441,13 @@ class QueryContextProcessor:
# time_offsets: ['1 year ago'],
# filters: [{col: 'dttm_col', op: 'TEMPORAL_RANGE', val: '2020 : 2021'}],
# }
+ original_offset = offset
+ if self.is_valid_date(offset) or offset == "inherit":
+ offset = self.get_time_offset_for_custom_or_inherit(
+ offset,
+ outer_from_dttm,
+ outer_to_dttm,
+ )
query_object_clone.from_dttm = get_past_or_future(
offset,
outer_from_dttm,
@@ -470,7 +510,7 @@ class QueryContextProcessor:
query_object_clone_dct = query_object_clone.to_dict()
# rename metrics: SUM(value) => SUM(value) 1 year ago
metrics_mapping = {
- metric: TIME_COMPARISON.join([metric, offset])
+ metric: TIME_COMPARISON.join([metric, original_offset])
for metric in metric_names
}