You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by vi...@apache.org on 2021/01/13 17:10:12 UTC
[superset] 01/09: refactor: from superset.utils.core break down
date_parser (#12408)
This is an automated email from the ASF dual-hosted git repository.
villebro pushed a commit to branch 1.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 35c15b8b3a3180a205982f7dc7e5871148f8784e
Author: Yongjie Zhao <yo...@gmail.com>
AuthorDate: Tue Jan 12 06:16:42 2021 +0800
refactor: from superset.utils.core break down date_parser (#12408)
---
superset/common/query_object.py | 8 +-
superset/connectors/druid/models.py | 9 +-
...1c4c6_migrate_num_period_compare_and_period_.py | 2 +-
superset/tasks/cache.py | 2 +-
superset/utils/core.py | 445 +------------------
superset/utils/date_parser.py | 469 +++++++++++++++++++++
superset/views/api.py | 2 +-
superset/viz.py | 9 +-
superset/viz_sip38.py | 9 +-
tests/utils/date_parser_tests.py | 263 ++++++++++++
tests/utils_tests.py | 239 -----------
11 files changed, 753 insertions(+), 704 deletions(-)
diff --git a/superset/common/query_object.py b/superset/common/query_object.py
index 7aa7ef7..43f7fee 100644
--- a/superset/common/query_object.py
+++ b/superset/common/query_object.py
@@ -28,12 +28,8 @@ from superset import app, is_feature_enabled
from superset.exceptions import QueryObjectValidationError
from superset.typing import Metric
from superset.utils import pandas_postprocessing
-from superset.utils.core import (
- DTTM_ALIAS,
- get_since_until,
- json_int_dttm_ser,
- parse_human_timedelta,
-)
+from superset.utils.core import DTTM_ALIAS, json_int_dttm_ser
+from superset.utils.date_parser import get_since_until, parse_human_timedelta
from superset.views.utils import get_time_range_endpoints
config = app.config
diff --git a/superset/connectors/druid/models.py b/superset/connectors/druid/models.py
index fe15b24..2a7e0f9 100644
--- a/superset/connectors/druid/models.py
+++ b/superset/connectors/druid/models.py
@@ -57,6 +57,7 @@ from superset.models.core import Database
from superset.models.helpers import AuditMixinNullable, ImportExportMixin, QueryResult
from superset.typing import FilterValues, Granularity, Metric, QueryObjectDict
from superset.utils import core as utils
+from superset.utils.date_parser import parse_human_datetime, parse_human_timedelta
try:
import requests
@@ -777,7 +778,7 @@ class DruidDatasource(Model, BaseDatasource):
granularity["timeZone"] = timezone
if origin:
- dttm = utils.parse_human_datetime(origin)
+ dttm = parse_human_datetime(origin)
assert dttm
granularity["origin"] = dttm.isoformat()
@@ -795,7 +796,7 @@ class DruidDatasource(Model, BaseDatasource):
else:
granularity["type"] = "duration"
granularity["duration"] = (
- utils.parse_human_timedelta(period_name).total_seconds() # type: ignore
+ parse_human_timedelta(period_name).total_seconds() # type: ignore
* 1000
)
return granularity
@@ -938,7 +939,7 @@ class DruidDatasource(Model, BaseDatasource):
)
# TODO: Use Lexicographic TopNMetricSpec once supported by PyDruid
if self.fetch_values_from:
- from_dttm = utils.parse_human_datetime(self.fetch_values_from)
+ from_dttm = parse_human_datetime(self.fetch_values_from)
assert from_dttm
else:
from_dttm = datetime(1970, 1, 1)
@@ -1426,7 +1427,7 @@ class DruidDatasource(Model, BaseDatasource):
time_offset = DruidDatasource.time_offset(query_obj["granularity"])
def increment_timestamp(ts: str) -> datetime:
- dt = utils.parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
+ dt = parse_human_datetime(ts).replace(tzinfo=DRUID_TZ)
return dt + timedelta(milliseconds=time_offset)
if DTTM_ALIAS in df.columns and time_offset:
diff --git a/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py b/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py
index ca4de4e..1d0d81f 100644
--- a/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py
+++ b/superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py
@@ -33,7 +33,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.ext.declarative import declarative_base
from superset import db
-from superset.utils.core import parse_human_timedelta
+from superset.utils.date_parser import parse_human_timedelta
revision = "3dda56f1c4c6"
down_revision = "bddc498dd179"
diff --git a/superset/tasks/cache.py b/superset/tasks/cache.py
index 953ed31..e32467d 100644
--- a/superset/tasks/cache.py
+++ b/superset/tasks/cache.py
@@ -31,7 +31,7 @@ from superset.models.core import Log
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from superset.models.tags import Tag, TaggedObject
-from superset.utils.core import parse_human_datetime
+from superset.utils.date_parser import parse_human_datetime
from superset.views.utils import build_extra_filters
logger = get_task_logger(__name__)
diff --git a/superset/utils/core.py b/superset/utils/core.py
index c500c19..7219317 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
"""Utility functions used across Superset"""
-import calendar
import decimal
import errno
import functools
@@ -39,7 +38,6 @@ from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import formatdate
from enum import Enum
-from time import struct_time
from timeit import default_timer
from types import TracebackType
from typing import (
@@ -65,29 +63,14 @@ import bleach
import markdown as md
import numpy as np
import pandas as pd
-import parsedatetime
import sqlalchemy as sa
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.backends.openssl.x509 import _Certificate
-from dateutil.parser import parse
-from dateutil.relativedelta import relativedelta
from flask import current_app, flash, g, Markup, render_template
from flask_appbuilder import SQLA
from flask_appbuilder.security.sqla.models import Role, User
-from flask_babel import gettext as __, lazy_gettext as _
-from holidays import CountryHoliday
-from pyparsing import (
- CaselessKeyword,
- Forward,
- Group,
- Optional as ppOptional,
- ParseException,
- ParseResults,
- pyparsing_common,
- quotedString,
- Suppress,
-)
+from flask_babel import gettext as __
from sqlalchemy import event, exc, select, Text
from sqlalchemy.dialects.mysql import MEDIUMTEXT
from sqlalchemy.engine import Connection, Engine
@@ -443,58 +426,6 @@ def list_minus(l: List[Any], minus: List[Any]) -> List[Any]:
return [o for o in l if o not in minus]
-def parse_human_datetime(human_readable: str) -> datetime:
- """
- Returns ``datetime.datetime`` from human readable strings
-
- >>> from datetime import date, timedelta
- >>> from dateutil.relativedelta import relativedelta
- >>> parse_human_datetime('2015-04-03')
- datetime.datetime(2015, 4, 3, 0, 0)
- >>> parse_human_datetime('2/3/1969')
- datetime.datetime(1969, 2, 3, 0, 0)
- >>> parse_human_datetime('now') <= datetime.now()
- True
- >>> parse_human_datetime('yesterday') <= datetime.now()
- True
- >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
- True
- >>> year_ago_1 = parse_human_datetime('one year ago').date()
- >>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
- >>> year_ago_1 == year_ago_2
- True
- >>> year_after_1 = parse_human_datetime('2 years after').date()
- >>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
- >>> year_after_1 == year_after_2
- True
- """
- try:
- dttm = parse(human_readable)
- except Exception: # pylint: disable=broad-except
- try:
- cal = parsedatetime.Calendar()
- parsed_dttm, parsed_flags = cal.parseDT(human_readable)
- # when time is not extracted, we 'reset to midnight'
- if parsed_flags & 2 == 0:
- parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
- dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
- except Exception as ex:
- logger.exception(ex)
- raise ValueError("Couldn't parse date string [{}]".format(human_readable))
- return dttm
-
-
-def dttm_from_timetuple(date_: struct_time) -> datetime:
- return datetime(
- date_.tm_year,
- date_.tm_mon,
- date_.tm_mday,
- date_.tm_hour,
- date_.tm_min,
- date_.tm_sec,
- )
-
-
def md5_hex(data: str) -> str:
return hashlib.md5(data.encode()).hexdigest()
@@ -516,39 +447,6 @@ class DashboardEncoder(json.JSONEncoder):
return json.JSONEncoder(sort_keys=True).default(o)
-def parse_human_timedelta(
- human_readable: Optional[str], source_time: Optional[datetime] = None,
-) -> timedelta:
- """
- Returns ``datetime.timedelta`` from natural language time deltas
-
- >>> parse_human_timedelta('1 day') == timedelta(days=1)
- True
- """
- cal = parsedatetime.Calendar()
- source_dttm = dttm_from_timetuple(
- source_time.timetuple() if source_time else datetime.now().timetuple()
- )
- modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
- return modified_dttm - source_dttm
-
-
-def parse_past_timedelta(
- delta_str: str, source_time: Optional[datetime] = None
-) -> timedelta:
- """
- Takes a delta like '1 year' and finds the timedelta for that period in
- the past, then represents that past timedelta in positive terms.
-
- parse_human_timedelta('1 year') find the timedelta 1 year in the future.
- parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
- or datetime.timedelta(365).
- """
- return -parse_human_timedelta(
- delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
- )
-
-
class JSONEncodedDict(TypeDecorator): # pylint: disable=abstract-method
"""Represents an immutable structure as a json-encoded string."""
@@ -1254,347 +1152,6 @@ def ensure_path_exists(path: str) -> None:
raise
-class EvalText: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[0]
-
- def eval(self) -> str:
- # strip quotes
- return self.value[1:-1]
-
-
-class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[1]
-
- def eval(self) -> datetime:
- return parse_human_datetime(self.value.eval())
-
-
-class EvalDateAddFunc: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[1]
-
- def eval(self) -> datetime:
- dttm_expression, delta, unit = self.value
- dttm = dttm_expression.eval()
- if unit.lower() == "quarter":
- delta = delta * 3
- unit = "month"
- return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
-
-
-class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[1]
-
- def eval(self) -> datetime:
- dttm_expression, unit = self.value
- dttm = dttm_expression.eval()
- if unit == "year":
- dttm = dttm.replace(
- month=1, day=1, hour=0, minute=0, second=0, microsecond=0
- )
- elif unit == "month":
- dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
- elif unit == "week":
- dttm = dttm - relativedelta(days=dttm.weekday())
- dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
- elif unit == "day":
- dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
- elif unit == "hour":
- dttm = dttm.replace(minute=0, second=0, microsecond=0)
- elif unit == "minute":
- dttm = dttm.replace(second=0, microsecond=0)
- else:
- dttm = dttm.replace(microsecond=0)
- return dttm
-
-
-class EvalLastDayFunc: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[1]
-
- def eval(self) -> datetime:
- dttm_expression, unit = self.value
- dttm = dttm_expression.eval()
- if unit == "year":
- return dttm.replace(
- month=12, day=31, hour=0, minute=0, second=0, microsecond=0
- )
- if unit == "month":
- return dttm.replace(
- day=calendar.monthrange(dttm.year, dttm.month)[1],
- hour=0,
- minute=0,
- second=0,
- microsecond=0,
- )
- # unit == "week":
- mon = dttm - relativedelta(days=dttm.weekday())
- mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
- return mon + relativedelta(days=6)
-
-
-class EvalHolidayFunc: # pylint: disable=too-few-public-methods
- def __init__(self, tokens: ParseResults) -> None:
- self.value = tokens[1]
-
- def eval(self) -> datetime:
- holiday = self.value[0].eval()
- dttm, country = [None, None]
- if len(self.value) >= 2:
- dttm = self.value[1].eval()
- if len(self.value) == 3:
- country = self.value[2]
- holiday_year = dttm.year if dttm else parse_human_datetime("today").year
- country = country.eval() if country else "US"
-
- holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
- searched_result = holiday_lookup.get_named(holiday)
- if len(searched_result) == 1:
- return dttm_from_timetuple(searched_result[0].timetuple())
- raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
-
-
-@memoized()
-def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
- ( # pylint: disable=invalid-name
- DATETIME,
- DATEADD,
- DATETRUNC,
- LASTDAY,
- HOLIDAY,
- YEAR,
- QUARTER,
- MONTH,
- WEEK,
- DAY,
- HOUR,
- MINUTE,
- SECOND,
- ) = map(
- CaselessKeyword,
- "datetime dateadd datetrunc lastday holiday "
- "year quarter month week day hour minute second".split(),
- )
- lparen, rparen, comma = map(Suppress, "(),")
- int_operand = pyparsing_common.signed_integer().setName("int_operand")
- text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
-
- # allow expression to be used recursively
- datetime_func = Forward().setName("datetime")
- dateadd_func = Forward().setName("dateadd")
- datetrunc_func = Forward().setName("datetrunc")
- lastday_func = Forward().setName("lastday")
- holiday_func = Forward().setName("holiday")
- date_expr = (
- datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
- )
-
- datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
- EvalDateTimeFunc
- )
- dateadd_func <<= (
- DATEADD
- + lparen
- + Group(
- date_expr
- + comma
- + int_operand
- + comma
- + (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
- + ppOptional(comma)
- )
- + rparen
- ).setParseAction(EvalDateAddFunc)
- datetrunc_func <<= (
- DATETRUNC
- + lparen
- + Group(
- date_expr
- + comma
- + (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
- + ppOptional(comma)
- )
- + rparen
- ).setParseAction(EvalDateTruncFunc)
- lastday_func <<= (
- LASTDAY
- + lparen
- + Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
- + rparen
- ).setParseAction(EvalLastDayFunc)
- holiday_func <<= (
- HOLIDAY
- + lparen
- + Group(
- text_operand
- + ppOptional(comma)
- + ppOptional(date_expr)
- + ppOptional(comma)
- + ppOptional(text_operand)
- + ppOptional(comma)
- )
- + rparen
- ).setParseAction(EvalHolidayFunc)
-
- return date_expr
-
-
-def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
- if datetime_expression:
- try:
- return datetime_parser().parseString(datetime_expression)[0].eval()
- except ParseException as error:
- raise ValueError(error)
- return None
-
-
-# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
-def get_since_until(
- time_range: Optional[str] = None,
- since: Optional[str] = None,
- until: Optional[str] = None,
- time_shift: Optional[str] = None,
- relative_start: Optional[str] = None,
- relative_end: Optional[str] = None,
-) -> Tuple[Optional[datetime], Optional[datetime]]:
- """Return `since` and `until` date time tuple from string representations of
- time_range, since, until and time_shift.
-
- This functiom supports both reading the keys separately (from `since` and
- `until`), as well as the new `time_range` key. Valid formats are:
-
- - ISO 8601
- - X days/years/hours/day/year/weeks
- - X days/years/hours/day/year/weeks ago
- - X days/years/hours/day/year/weeks from now
- - freeform
-
- Additionally, for `time_range` (these specify both `since` and `until`):
-
- - Last day
- - Last week
- - Last month
- - Last quarter
- - Last year
- - No filter
- - Last X seconds/minutes/hours/days/weeks/months/years
- - Next X seconds/minutes/hours/days/weeks/months/years
-
- """
- separator = " : "
- _relative_start = relative_start if relative_start else "today"
- _relative_end = relative_end if relative_end else "today"
-
- if time_range == "No filter":
- return None, None
-
- if time_range and time_range.startswith("Last") and separator not in time_range:
- time_range = time_range + separator + _relative_end
-
- if time_range and time_range.startswith("Next") and separator not in time_range:
- time_range = _relative_start + separator + time_range
-
- if (
- time_range
- and time_range.startswith("previous calendar week")
- and separator not in time_range
- ):
- time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
- if (
- time_range
- and time_range.startswith("previous calendar month")
- and separator not in time_range
- ):
- time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
- if (
- time_range
- and time_range.startswith("previous calendar year")
- and separator not in time_range
- ):
- time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
-
- if time_range and separator in time_range:
- time_range_lookup = [
- (
- r"^last\s+(day|week|month|quarter|year)$",
- lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
- ),
- (
- r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
- lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
- ),
- (
- r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
- lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
- ),
- (
- r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
- lambda text: text,
- ),
- ]
-
- since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
- since_and_until: List[Optional[str]] = []
- for part in since_and_until_partition:
- if not part:
- # if since or until is "", set as None
- since_and_until.append(None)
- continue
-
- # Is it possible to match to time_range_lookup
- matched = False
- for pattern, fn in time_range_lookup:
- result = re.search(pattern, part, re.IGNORECASE)
- if result:
- matched = True
- # converted matched time_range to "formal time expressions"
- since_and_until.append(fn(*result.groups())) # type: ignore
- if not matched:
- # default matched case
- since_and_until.append(f"DATETIME('{part}')")
-
- _since, _until = map(datetime_eval, since_and_until)
- else:
- since = since or ""
- if since:
- since = add_ago_to_since(since)
- _since = parse_human_datetime(since) if since else None
- _until = (
- parse_human_datetime(until)
- if until
- else parse_human_datetime(_relative_end)
- )
-
- if time_shift:
- time_delta = parse_past_timedelta(time_shift)
- _since = _since if _since is None else (_since - time_delta)
- _until = _until if _until is None else (_until - time_delta)
-
- if _since and _until and _since > _until:
- raise ValueError(_("From date cannot be larger than to date"))
-
- return _since, _until
-
-
-def add_ago_to_since(since: str) -> str:
- """
- Backwards compatibility hack. Without this slices with since: 7 days will
- be treated as 7 days in the future.
-
- :param str since:
- :returns: Since with ago added if necessary
- :rtype: str
- """
- since_words = since.split(" ")
- grains = ["days", "years", "hours", "day", "year", "weeks"]
- if len(since_words) == 2 and since_words[1] in grains:
- since += " ago"
- return since
-
-
def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name
form_data: FormData,
) -> None:
diff --git a/superset/utils/date_parser.py b/superset/utils/date_parser.py
new file mode 100644
index 0000000..aee2c83
--- /dev/null
+++ b/superset/utils/date_parser.py
@@ -0,0 +1,469 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import calendar
+import logging
+import re
+from datetime import datetime, timedelta
+from time import struct_time
+from typing import List, Optional, Tuple
+
+import parsedatetime
+from dateutil.parser import parse
+from dateutil.relativedelta import relativedelta
+from flask_babel import lazy_gettext as _
+from holidays import CountryHoliday
+from pyparsing import (
+ CaselessKeyword,
+ Forward,
+ Group,
+ Optional as ppOptional,
+ ParseException,
+ ParseResults,
+ pyparsing_common,
+ quotedString,
+ Suppress,
+)
+
+from .core import memoized
+
+logger = logging.getLogger(__name__)
+
+
+def parse_human_datetime(human_readable: str) -> datetime:
+ """
+ Returns ``datetime.datetime`` from human readable strings
+
+ >>> from datetime import date, timedelta
+ >>> from dateutil.relativedelta import relativedelta
+ >>> parse_human_datetime('2015-04-03')
+ datetime.datetime(2015, 4, 3, 0, 0)
+ >>> parse_human_datetime('2/3/1969')
+ datetime.datetime(1969, 2, 3, 0, 0)
+ >>> parse_human_datetime('now') <= datetime.now()
+ True
+ >>> parse_human_datetime('yesterday') <= datetime.now()
+ True
+ >>> date.today() - timedelta(1) == parse_human_datetime('yesterday').date()
+ True
+ >>> year_ago_1 = parse_human_datetime('one year ago').date()
+ >>> year_ago_2 = (datetime.now() - relativedelta(years=1)).date()
+ >>> year_ago_1 == year_ago_2
+ True
+ >>> year_after_1 = parse_human_datetime('2 years after').date()
+ >>> year_after_2 = (datetime.now() + relativedelta(years=2)).date()
+ >>> year_after_1 == year_after_2
+ True
+ """
+ try:
+ dttm = parse(human_readable)
+ except Exception: # pylint: disable=broad-except
+ try:
+ cal = parsedatetime.Calendar()
+ parsed_dttm, parsed_flags = cal.parseDT(human_readable)
+ # when time is not extracted, we 'reset to midnight'
+ if parsed_flags & 2 == 0:
+ parsed_dttm = parsed_dttm.replace(hour=0, minute=0, second=0)
+ dttm = dttm_from_timetuple(parsed_dttm.utctimetuple())
+ except Exception as ex:
+ logger.exception(ex)
+ raise ValueError("Couldn't parse date string [{}]".format(human_readable))
+ return dttm
+
+
+def dttm_from_timetuple(date_: struct_time) -> datetime:
+ return datetime(
+ date_.tm_year,
+ date_.tm_mon,
+ date_.tm_mday,
+ date_.tm_hour,
+ date_.tm_min,
+ date_.tm_sec,
+ )
+
+
+def parse_human_timedelta(
+ human_readable: Optional[str], source_time: Optional[datetime] = None,
+) -> timedelta:
+ """
+ Returns ``datetime.timedelta`` from natural language time deltas
+
+ >>> parse_human_timedelta('1 day') == timedelta(days=1)
+ True
+ """
+ cal = parsedatetime.Calendar()
+ source_dttm = dttm_from_timetuple(
+ source_time.timetuple() if source_time else datetime.now().timetuple()
+ )
+ modified_dttm = dttm_from_timetuple(cal.parse(human_readable or "", source_dttm)[0])
+ return modified_dttm - source_dttm
+
+
+def parse_past_timedelta(
+ delta_str: str, source_time: Optional[datetime] = None
+) -> timedelta:
+ """
+ Takes a delta like '1 year' and finds the timedelta for that period in
+ the past, then represents that past timedelta in positive terms.
+
+ parse_human_timedelta('1 year') find the timedelta 1 year in the future.
+ parse_past_timedelta('1 year') returns -datetime.timedelta(-365)
+ or datetime.timedelta(365).
+ """
+ return -parse_human_timedelta(
+ delta_str if delta_str.startswith("-") else f"-{delta_str}", source_time,
+ )
+
+
+# pylint: disable=too-many-arguments, too-many-locals, too-many-branches
+def get_since_until(
+ time_range: Optional[str] = None,
+ since: Optional[str] = None,
+ until: Optional[str] = None,
+ time_shift: Optional[str] = None,
+ relative_start: Optional[str] = None,
+ relative_end: Optional[str] = None,
+) -> Tuple[Optional[datetime], Optional[datetime]]:
+ """Return `since` and `until` date time tuple from string representations of
+ time_range, since, until and time_shift.
+
+ This functiom supports both reading the keys separately (from `since` and
+ `until`), as well as the new `time_range` key. Valid formats are:
+
+ - ISO 8601
+ - X days/years/hours/day/year/weeks
+ - X days/years/hours/day/year/weeks ago
+ - X days/years/hours/day/year/weeks from now
+ - freeform
+
+ Additionally, for `time_range` (these specify both `since` and `until`):
+
+ - Last day
+ - Last week
+ - Last month
+ - Last quarter
+ - Last year
+ - No filter
+ - Last X seconds/minutes/hours/days/weeks/months/years
+ - Next X seconds/minutes/hours/days/weeks/months/years
+
+ """
+ separator = " : "
+ _relative_start = relative_start if relative_start else "today"
+ _relative_end = relative_end if relative_end else "today"
+
+ if time_range == "No filter":
+ return None, None
+
+ if time_range and time_range.startswith("Last") and separator not in time_range:
+ time_range = time_range + separator + _relative_end
+
+ if time_range and time_range.startswith("Next") and separator not in time_range:
+ time_range = _relative_start + separator + time_range
+
+ if (
+ time_range
+ and time_range.startswith("previous calendar week")
+ and separator not in time_range
+ ):
+ time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, WEEK), WEEK) : DATETRUNC(DATETIME('today'), WEEK)" # pylint: disable=line-too-long
+ if (
+ time_range
+ and time_range.startswith("previous calendar month")
+ and separator not in time_range
+ ):
+ time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, MONTH), MONTH) : DATETRUNC(DATETIME('today'), MONTH)" # pylint: disable=line-too-long
+ if (
+ time_range
+ and time_range.startswith("previous calendar year")
+ and separator not in time_range
+ ):
+ time_range = "DATETRUNC(DATEADD(DATETIME('today'), -1, YEAR), YEAR) : DATETRUNC(DATETIME('today'), YEAR)" # pylint: disable=line-too-long
+
+ if time_range and separator in time_range:
+ time_range_lookup = [
+ (
+ r"^last\s+(day|week|month|quarter|year)$",
+ lambda unit: f"DATEADD(DATETIME('{_relative_start}'), -1, {unit})",
+ ),
+ (
+ r"^last\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
+ lambda delta, unit: f"DATEADD(DATETIME('{_relative_start}'), -{int(delta)}, {unit})", # pylint: disable=line-too-long
+ ),
+ (
+ r"^next\s+([0-9]+)\s+(second|minute|hour|day|week|month|year)s$",
+ lambda delta, unit: f"DATEADD(DATETIME('{_relative_end}'), {int(delta)}, {unit})", # pylint: disable=line-too-long
+ ),
+ (
+ r"^(DATETIME.*|DATEADD.*|DATETRUNC.*|LASTDAY.*|HOLIDAY.*)$",
+ lambda text: text,
+ ),
+ ]
+
+ since_and_until_partition = [_.strip() for _ in time_range.split(separator, 1)]
+ since_and_until: List[Optional[str]] = []
+ for part in since_and_until_partition:
+ if not part:
+ # if since or until is "", set as None
+ since_and_until.append(None)
+ continue
+
+ # Is it possible to match to time_range_lookup
+ matched = False
+ for pattern, fn in time_range_lookup:
+ result = re.search(pattern, part, re.IGNORECASE)
+ if result:
+ matched = True
+ # converted matched time_range to "formal time expressions"
+ since_and_until.append(fn(*result.groups())) # type: ignore
+ if not matched:
+ # default matched case
+ since_and_until.append(f"DATETIME('{part}')")
+
+ _since, _until = map(datetime_eval, since_and_until)
+ else:
+ since = since or ""
+ if since:
+ since = add_ago_to_since(since)
+ _since = parse_human_datetime(since) if since else None
+ _until = (
+ parse_human_datetime(until)
+ if until
+ else parse_human_datetime(_relative_end)
+ )
+
+ if time_shift:
+ time_delta = parse_past_timedelta(time_shift)
+ _since = _since if _since is None else (_since - time_delta)
+ _until = _until if _until is None else (_until - time_delta)
+
+ if _since and _until and _since > _until:
+ raise ValueError(_("From date cannot be larger than to date"))
+
+ return _since, _until
+
+
+def add_ago_to_since(since: str) -> str:
+ """
+ Backwards compatibility hack. Without this slices with since: 7 days will
+ be treated as 7 days in the future.
+
+ :param str since:
+ :returns: Since with ago added if necessary
+ :rtype: str
+ """
+ since_words = since.split(" ")
+ grains = ["days", "years", "hours", "day", "year", "weeks"]
+ if len(since_words) == 2 and since_words[1] in grains:
+ since += " ago"
+ return since
+
+
+class EvalText: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[0]
+
+ def eval(self) -> str:
+ # strip quotes
+ return self.value[1:-1]
+
+
+class EvalDateTimeFunc: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[1]
+
+ def eval(self) -> datetime:
+ return parse_human_datetime(self.value.eval())
+
+
+class EvalDateAddFunc: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[1]
+
+ def eval(self) -> datetime:
+ dttm_expression, delta, unit = self.value
+ dttm = dttm_expression.eval()
+ if unit.lower() == "quarter":
+ delta = delta * 3
+ unit = "month"
+ return dttm + parse_human_timedelta(f"{delta} {unit}s", dttm)
+
+
+class EvalDateTruncFunc: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[1]
+
+ def eval(self) -> datetime:
+ dttm_expression, unit = self.value
+ dttm = dttm_expression.eval()
+ if unit == "year":
+ dttm = dttm.replace(
+ month=1, day=1, hour=0, minute=0, second=0, microsecond=0
+ )
+ elif unit == "month":
+ dttm = dttm.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
+ elif unit == "week":
+ dttm = dttm - relativedelta(days=dttm.weekday())
+ dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
+ elif unit == "day":
+ dttm = dttm.replace(hour=0, minute=0, second=0, microsecond=0)
+ elif unit == "hour":
+ dttm = dttm.replace(minute=0, second=0, microsecond=0)
+ elif unit == "minute":
+ dttm = dttm.replace(second=0, microsecond=0)
+ else:
+ dttm = dttm.replace(microsecond=0)
+ return dttm
+
+
+class EvalLastDayFunc: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[1]
+
+ def eval(self) -> datetime:
+ dttm_expression, unit = self.value
+ dttm = dttm_expression.eval()
+ if unit == "year":
+ return dttm.replace(
+ month=12, day=31, hour=0, minute=0, second=0, microsecond=0
+ )
+ if unit == "month":
+ return dttm.replace(
+ day=calendar.monthrange(dttm.year, dttm.month)[1],
+ hour=0,
+ minute=0,
+ second=0,
+ microsecond=0,
+ )
+ # unit == "week":
+ mon = dttm - relativedelta(days=dttm.weekday())
+ mon = mon.replace(hour=0, minute=0, second=0, microsecond=0)
+ return mon + relativedelta(days=6)
+
+
+class EvalHolidayFunc: # pylint: disable=too-few-public-methods
+ def __init__(self, tokens: ParseResults) -> None:
+ self.value = tokens[1]
+
+ def eval(self) -> datetime:
+ holiday = self.value[0].eval()
+ dttm, country = [None, None]
+ if len(self.value) >= 2:
+ dttm = self.value[1].eval()
+ if len(self.value) == 3:
+ country = self.value[2]
+ holiday_year = dttm.year if dttm else parse_human_datetime("today").year
+ country = country.eval() if country else "US"
+
+ holiday_lookup = CountryHoliday(country, years=[holiday_year], observed=False)
+ searched_result = holiday_lookup.get_named(holiday)
+ if len(searched_result) == 1:
+ return dttm_from_timetuple(searched_result[0].timetuple())
+ raise ValueError(_("Unable to find such a holiday: [{}]").format(holiday))
+
+
+@memoized()
+def datetime_parser() -> ParseResults: # pylint: disable=too-many-locals
+ ( # pylint: disable=invalid-name
+ DATETIME,
+ DATEADD,
+ DATETRUNC,
+ LASTDAY,
+ HOLIDAY,
+ YEAR,
+ QUARTER,
+ MONTH,
+ WEEK,
+ DAY,
+ HOUR,
+ MINUTE,
+ SECOND,
+ ) = map(
+ CaselessKeyword,
+ "datetime dateadd datetrunc lastday holiday "
+ "year quarter month week day hour minute second".split(),
+ )
+ lparen, rparen, comma = map(Suppress, "(),")
+ int_operand = pyparsing_common.signed_integer().setName("int_operand")
+ text_operand = quotedString.setName("text_operand").setParseAction(EvalText)
+
+ # allow expression to be used recursively
+ datetime_func = Forward().setName("datetime")
+ dateadd_func = Forward().setName("dateadd")
+ datetrunc_func = Forward().setName("datetrunc")
+ lastday_func = Forward().setName("lastday")
+ holiday_func = Forward().setName("holiday")
+ date_expr = (
+ datetime_func | dateadd_func | datetrunc_func | lastday_func | holiday_func
+ )
+
+ datetime_func <<= (DATETIME + lparen + text_operand + rparen).setParseAction(
+ EvalDateTimeFunc
+ )
+ dateadd_func <<= (
+ DATEADD
+ + lparen
+ + Group(
+ date_expr
+ + comma
+ + int_operand
+ + comma
+ + (YEAR | QUARTER | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ + ppOptional(comma)
+ )
+ + rparen
+ ).setParseAction(EvalDateAddFunc)
+ datetrunc_func <<= (
+ DATETRUNC
+ + lparen
+ + Group(
+ date_expr
+ + comma
+ + (YEAR | MONTH | WEEK | DAY | HOUR | MINUTE | SECOND)
+ + ppOptional(comma)
+ )
+ + rparen
+ ).setParseAction(EvalDateTruncFunc)
+ lastday_func <<= (
+ LASTDAY
+ + lparen
+ + Group(date_expr + comma + (YEAR | MONTH | WEEK) + ppOptional(comma))
+ + rparen
+ ).setParseAction(EvalLastDayFunc)
+ holiday_func <<= (
+ HOLIDAY
+ + lparen
+ + Group(
+ text_operand
+ + ppOptional(comma)
+ + ppOptional(date_expr)
+ + ppOptional(comma)
+ + ppOptional(text_operand)
+ + ppOptional(comma)
+ )
+ + rparen
+ ).setParseAction(EvalHolidayFunc)
+
+ return date_expr
+
+
+def datetime_eval(datetime_expression: Optional[str] = None) -> Optional[datetime]:
+ if datetime_expression:
+ try:
+ return datetime_parser().parseString(datetime_expression)[0].eval()
+ except ParseException as error:
+ raise ValueError(error)
+ return None
diff --git a/superset/views/api.py b/superset/views/api.py
index de7d656..7df1c5e 100644
--- a/superset/views/api.py
+++ b/superset/views/api.py
@@ -29,7 +29,7 @@ from superset.legacy import update_time_range
from superset.models.slice import Slice
from superset.typing import FlaskResponse
from superset.utils import core as utils
-from superset.utils.core import get_since_until
+from superset.utils.date_parser import get_since_until
from superset.views.base import api, BaseSupersetView, handle_api_exception
get_time_range_schema = {"type": "string"}
diff --git a/superset/viz.py b/superset/viz.py
index 6baef6a..21fbdf8 100644
--- a/superset/viz.py
+++ b/superset/viz.py
@@ -75,6 +75,7 @@ from superset.utils.core import (
QueryMode,
to_adhoc,
)
+from superset.utils.date_parser import get_since_until, parse_past_timedelta
from superset.utils.dates import datetime_to_epoch
from superset.utils.hashing import md5_sha_from_str
@@ -356,7 +357,7 @@ class BaseViz:
order_desc = form_data.get("order_desc", True)
try:
- since, until = utils.get_since_until(
+ since, until = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
@@ -367,7 +368,7 @@ class BaseViz:
raise QueryObjectValidationError(str(ex))
time_shift = form_data.get("time_shift", "")
- self.time_shift = utils.parse_past_timedelta(time_shift)
+ self.time_shift = parse_past_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
to_dttm = None if until is None else (until - self.time_shift)
if from_dttm and to_dttm and from_dttm > to_dttm:
@@ -1004,7 +1005,7 @@ class CalHeatmapViz(BaseViz):
data[metric] = values
try:
- start, end = utils.get_since_until(
+ start, end = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
@@ -1318,7 +1319,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
for option in time_compare:
query_object = self.query_obj()
try:
- delta = utils.parse_past_timedelta(option)
+ delta = parse_past_timedelta(option)
except ValueError as ex:
raise QueryObjectValidationError(str(ex))
query_object["inner_from_dttm"] = query_object["from_dttm"]
diff --git a/superset/viz_sip38.py b/superset/viz_sip38.py
index 5a16639..9ec1752 100644
--- a/superset/viz_sip38.py
+++ b/superset/viz_sip38.py
@@ -63,6 +63,7 @@ from superset.utils.core import (
merge_extra_filters,
to_adhoc,
)
+from superset.utils.date_parser import get_since_until, parse_past_timedelta
import dataclasses # isort:skip
@@ -359,7 +360,7 @@ class BaseViz:
# default order direction
order_desc = form_data.get("order_desc", True)
- since, until = utils.get_since_until(
+ since, until = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
@@ -367,7 +368,7 @@ class BaseViz:
until=form_data.get("until"),
)
time_shift = form_data.get("time_shift", "")
- self.time_shift = utils.parse_past_timedelta(time_shift)
+ self.time_shift = parse_past_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
to_dttm = None if until is None else (until - self.time_shift)
if from_dttm and to_dttm and from_dttm > to_dttm:
@@ -883,7 +884,7 @@ class CalHeatmapViz(BaseViz):
values[str(v / 10 ** 9)] = obj.get(metric)
data[metric] = values
- start, end = utils.get_since_until(
+ start, end = get_since_until(
relative_start=relative_start,
relative_end=relative_end,
time_range=form_data.get("time_range"),
@@ -1265,7 +1266,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
for option in time_compare:
query_object = self.query_obj()
- delta = utils.parse_past_timedelta(option)
+ delta = parse_past_timedelta(option)
query_object["inner_from_dttm"] = query_object["from_dttm"]
query_object["inner_to_dttm"] = query_object["to_dttm"]
diff --git a/tests/utils/date_parser_tests.py b/tests/utils/date_parser_tests.py
new file mode 100644
index 0000000..fc592d3
--- /dev/null
+++ b/tests/utils/date_parser_tests.py
@@ -0,0 +1,263 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+from superset.utils.date_parser import (
+ datetime_eval,
+ get_since_until,
+ parse_human_timedelta,
+ parse_past_timedelta,
+)
+from tests.base_tests import SupersetTestCase
+
+
+def mock_parse_human_datetime(s):
+ if s == "now":
+ return datetime(2016, 11, 7, 9, 30, 10)
+ elif s == "today":
+ return datetime(2016, 11, 7)
+ elif s == "yesterday":
+ return datetime(2016, 11, 6)
+ elif s == "tomorrow":
+ return datetime(2016, 11, 8)
+ elif s == "Last year":
+ return datetime(2015, 11, 7)
+ elif s == "Last week":
+ return datetime(2015, 10, 31)
+ elif s == "Last 5 months":
+ return datetime(2016, 6, 7)
+ elif s == "Next 5 months":
+ return datetime(2017, 4, 7)
+ elif s in ["5 days", "5 days ago"]:
+ return datetime(2016, 11, 2)
+ elif s == "2018-01-01T00:00:00":
+ return datetime(2018, 1, 1)
+ elif s == "2018-12-31T23:59:59":
+ return datetime(2018, 12, 31, 23, 59, 59)
+
+
+class TestDateParser(SupersetTestCase):
+ @patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
+ def test_get_since_until(self):
+ result = get_since_until()
+ expected = None, datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until(" : now")
+ expected = None, datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("yesterday : tomorrow")
+ expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
+ expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last year")
+ expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last quarter")
+ expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last 5 months")
+ expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Next 5 months")
+ expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until(since="5 days")
+ expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until(since="5 days ago", until="tomorrow")
+ expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
+ self.assertEqual(result, expected)
+
+ result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
+ expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until(time_range="5 days : now")
+ expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last week", relative_end="now")
+ expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last week", relative_start="now")
+ expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("Last week", relative_start="now", relative_end="now")
+ expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("previous calendar week")
+ expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("previous calendar month")
+ expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = get_since_until("previous calendar year")
+ expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ with self.assertRaises(ValueError):
+ get_since_until(time_range="tomorrow : yesterday")
+
+ @patch("superset.utils.date_parser.parse_human_datetime", mock_parse_human_datetime)
+ def test_datetime_eval(self):
+ result = datetime_eval("datetime('now')")
+ expected = datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetime('today' )")
+ expected = datetime(2016, 11, 7)
+ self.assertEqual(result, expected)
+
+ # Parse compact arguments spelling
+ result = datetime_eval("dateadd(datetime('today'),1,year,)")
+ expected = datetime(2017, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('today'), -2, year)")
+ expected = datetime(2014, 11, 7)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
+ expected = datetime(2017, 5, 7)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('today'), 3, month)")
+ expected = datetime(2017, 2, 7)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('today'), -3, week)")
+ expected = datetime(2016, 10, 17)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('today'), 3, day)")
+ expected = datetime(2016, 11, 10)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('now'), 3, hour)")
+ expected = datetime(2016, 11, 7, 12, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('now'), 40, minute)")
+ expected = datetime(2016, 11, 7, 10, 10, 10)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("dateadd(datetime('now'), -11, second)")
+ expected = datetime(2016, 11, 7, 9, 29, 59)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), year)")
+ expected = datetime(2016, 1, 1, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), month)")
+ expected = datetime(2016, 11, 1, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), day)")
+ expected = datetime(2016, 11, 7, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), week)")
+ expected = datetime(2016, 11, 7, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), hour)")
+ expected = datetime(2016, 11, 7, 9, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), minute)")
+ expected = datetime(2016, 11, 7, 9, 30, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("datetrunc(datetime('now'), second)")
+ expected = datetime(2016, 11, 7, 9, 30, 10)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("lastday(datetime('now'), year)")
+ expected = datetime(2016, 12, 31, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("lastday(datetime('today'), month)")
+ expected = datetime(2016, 11, 30, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("holiday('Christmas')")
+ expected = datetime(2016, 12, 25, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
+ expected = datetime(2018, 9, 3, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval(
+ "holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
+ )
+ expected = datetime(2018, 12, 26, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ result = datetime_eval(
+ "lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
+ )
+ expected = datetime(2018, 2, 28, 0, 0, 0)
+ self.assertEqual(result, expected)
+
+ @patch("superset.utils.date_parser.datetime")
+ def test_parse_human_timedelta(self, mock_datetime):
+ mock_datetime.now.return_value = datetime(2019, 4, 1)
+ mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
+ self.assertEqual(parse_human_timedelta("now"), timedelta(0))
+ self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
+ self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
+ self.assertEqual(parse_human_timedelta(None), timedelta(0))
+ self.assertEqual(
+ parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
+ )
+ self.assertEqual(
+ parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
+ )
+ self.assertEqual(
+ parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
+ )
+ self.assertEqual(
+ parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
+ )
+
+ @patch("superset.utils.date_parser.datetime")
+ def test_parse_past_timedelta(self, mock_datetime):
+ mock_datetime.now.return_value = datetime(2019, 4, 1)
+ mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
+ self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
+ self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
+ self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
+ self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
diff --git a/tests/utils_tests.py b/tests/utils_tests.py
index 20c7897..b47f7d1 100644
--- a/tests/utils_tests.py
+++ b/tests/utils_tests.py
@@ -46,7 +46,6 @@ from superset.utils.core import (
get_iterable,
get_email_address_list,
get_or_create_db,
- get_since_until,
get_stacktrace,
json_int_dttm_ser,
json_iso_dttm_ser,
@@ -55,15 +54,12 @@ from superset.utils.core import (
merge_extra_filters,
merge_request_params,
parse_ssl_cert,
- parse_human_timedelta,
parse_js_uri_path_item,
- parse_past_timedelta,
split,
TimeRangeEndpoint,
validate_json,
zlib_compress,
zlib_decompress,
- datetime_eval,
)
from superset.utils import schema
from superset.views.utils import (
@@ -76,31 +72,6 @@ from tests.base_tests import SupersetTestCase
from .fixtures.certificates import ssl_certificate
-def mock_parse_human_datetime(s):
- if s == "now":
- return datetime(2016, 11, 7, 9, 30, 10)
- elif s == "today":
- return datetime(2016, 11, 7)
- elif s == "yesterday":
- return datetime(2016, 11, 6)
- elif s == "tomorrow":
- return datetime(2016, 11, 8)
- elif s == "Last year":
- return datetime(2015, 11, 7)
- elif s == "Last week":
- return datetime(2015, 10, 31)
- elif s == "Last 5 months":
- return datetime(2016, 6, 7)
- elif s == "Next 5 months":
- return datetime(2017, 4, 7)
- elif s in ["5 days", "5 days ago"]:
- return datetime(2016, 11, 2)
- elif s == "2018-01-01T00:00:00":
- return datetime(2018, 1, 1)
- elif s == "2018-12-31T23:59:59":
- return datetime(2018, 12, 31, 23, 59, 59)
-
-
def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"):
result = {"clause": clause.upper(), "expressionType": expressionType}
@@ -147,36 +118,6 @@ class TestUtils(SupersetTestCase):
assert isinstance(base_json_conv(uuid.uuid4()), str) is True
assert isinstance(base_json_conv(timedelta(0)), str) is True
- @patch("superset.utils.core.datetime")
- def test_parse_human_timedelta(self, mock_datetime):
- mock_datetime.now.return_value = datetime(2019, 4, 1)
- mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
- self.assertEqual(parse_human_timedelta("now"), timedelta(0))
- self.assertEqual(parse_human_timedelta("1 year"), timedelta(366))
- self.assertEqual(parse_human_timedelta("-1 year"), timedelta(-365))
- self.assertEqual(parse_human_timedelta(None), timedelta(0))
- self.assertEqual(
- parse_human_timedelta("1 month", datetime(2019, 4, 1)), timedelta(30),
- )
- self.assertEqual(
- parse_human_timedelta("1 month", datetime(2019, 5, 1)), timedelta(31),
- )
- self.assertEqual(
- parse_human_timedelta("1 month", datetime(2019, 2, 1)), timedelta(28),
- )
- self.assertEqual(
- parse_human_timedelta("-1 month", datetime(2019, 2, 1)), timedelta(-31),
- )
-
- @patch("superset.utils.core.datetime")
- def test_parse_past_timedelta(self, mock_datetime):
- mock_datetime.now.return_value = datetime(2019, 4, 1)
- mock_datetime.side_effect = lambda *args, **kw: datetime(*args, **kw)
- self.assertEqual(parse_past_timedelta("1 year"), timedelta(365))
- self.assertEqual(parse_past_timedelta("-1 year"), timedelta(365))
- self.assertEqual(parse_past_timedelta("52 weeks"), timedelta(364))
- self.assertEqual(parse_past_timedelta("1 month"), timedelta(31))
-
def test_zlib_compression(self):
json_str = '{"test": 1}'
blob = zlib_compress(json_str)
@@ -699,186 +640,6 @@ class TestUtils(SupersetTestCase):
self.assertEqual(instance.watcher, 4)
self.assertEqual(result1, result8)
- @patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
- def test_get_since_until(self):
- result = get_since_until()
- expected = None, datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until(" : now")
- expected = None, datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = get_since_until("yesterday : tomorrow")
- expected = datetime(2016, 11, 6), datetime(2016, 11, 8)
- self.assertEqual(result, expected)
-
- result = get_since_until("2018-01-01T00:00:00 : 2018-12-31T23:59:59")
- expected = datetime(2018, 1, 1), datetime(2018, 12, 31, 23, 59, 59)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last year")
- expected = datetime(2015, 11, 7), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last quarter")
- expected = datetime(2016, 8, 7), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last 5 months")
- expected = datetime(2016, 6, 7), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until("Next 5 months")
- expected = datetime(2016, 11, 7), datetime(2017, 4, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until(since="5 days")
- expected = datetime(2016, 11, 2), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until(since="5 days ago", until="tomorrow")
- expected = datetime(2016, 11, 2), datetime(2016, 11, 8)
- self.assertEqual(result, expected)
-
- result = get_since_until(time_range="yesterday : tomorrow", time_shift="1 day")
- expected = datetime(2016, 11, 5), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until(time_range="5 days : now")
- expected = datetime(2016, 11, 2), datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last week", relative_end="now")
- expected = datetime(2016, 10, 31), datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last week", relative_start="now")
- expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- result = get_since_until("Last week", relative_start="now", relative_end="now")
- expected = datetime(2016, 10, 31, 9, 30, 10), datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = get_since_until("previous calendar week")
- expected = datetime(2016, 10, 31, 0, 0, 0), datetime(2016, 11, 7, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = get_since_until("previous calendar month")
- expected = datetime(2016, 10, 1, 0, 0, 0), datetime(2016, 11, 1, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = get_since_until("previous calendar year")
- expected = datetime(2015, 1, 1, 0, 0, 0), datetime(2016, 1, 1, 0, 0, 0)
- self.assertEqual(result, expected)
-
- with self.assertRaises(ValueError):
- get_since_until(time_range="tomorrow : yesterday")
-
- @patch("superset.utils.core.parse_human_datetime", mock_parse_human_datetime)
- def test_datetime_eval(self):
- result = datetime_eval("datetime('now')")
- expected = datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetime('today' )")
- expected = datetime(2016, 11, 7)
- self.assertEqual(result, expected)
-
- # Parse compact arguments spelling
- result = datetime_eval("dateadd(datetime('today'),1,year,)")
- expected = datetime(2017, 11, 7)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('today'), -2, year)")
- expected = datetime(2014, 11, 7)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('today'), 2, quarter)")
- expected = datetime(2017, 5, 7)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('today'), 3, month)")
- expected = datetime(2017, 2, 7)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('today'), -3, week)")
- expected = datetime(2016, 10, 17)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('today'), 3, day)")
- expected = datetime(2016, 11, 10)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('now'), 3, hour)")
- expected = datetime(2016, 11, 7, 12, 30, 10)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('now'), 40, minute)")
- expected = datetime(2016, 11, 7, 10, 10, 10)
- self.assertEqual(result, expected)
-
- result = datetime_eval("dateadd(datetime('now'), -11, second)")
- expected = datetime(2016, 11, 7, 9, 29, 59)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), year)")
- expected = datetime(2016, 1, 1, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), month)")
- expected = datetime(2016, 11, 1, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), day)")
- expected = datetime(2016, 11, 7, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), week)")
- expected = datetime(2016, 11, 7, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), hour)")
- expected = datetime(2016, 11, 7, 9, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), minute)")
- expected = datetime(2016, 11, 7, 9, 30, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("datetrunc(datetime('now'), second)")
- expected = datetime(2016, 11, 7, 9, 30, 10)
- self.assertEqual(result, expected)
-
- result = datetime_eval("lastday(datetime('now'), year)")
- expected = datetime(2016, 12, 31, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("lastday(datetime('today'), month)")
- expected = datetime(2016, 11, 30, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("holiday('Christmas')")
- expected = datetime(2016, 12, 25, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval("holiday('Labor day', datetime('2018-01-01T00:00:00'))")
- expected = datetime(2018, 9, 3, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval(
- "holiday('Boxing day', datetime('2018-01-01T00:00:00'), 'UK')"
- )
- expected = datetime(2018, 12, 26, 0, 0, 0)
- self.assertEqual(result, expected)
-
- result = datetime_eval(
- "lastday(dateadd(datetime('2018-01-01T00:00:00'), 1, month), month)"
- )
- expected = datetime(2018, 2, 28, 0, 0, 0)
- self.assertEqual(result, expected)
-
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
def test_convert_legacy_filters_into_adhoc_where(self):
form_data = {"where": "a = 1"}