You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/09/29 02:48:21 UTC
(superset) 01/01: refactor(db_engine_specs): derive epoch method dispatch from EPOCH_FORMATS
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch chore/epoch-format-dispatch-dry
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 02bae63d34b378ad133fba84a5db9a8c4b2c1b63
Author: Evan Rusackas <ev...@rusackas.com>
AuthorDate: Mon Sep 28 19:47:41 2026 -0700
refactor(db_engine_specs): derive epoch method dispatch from EPOCH_FORMATS
Follow-up to #44500, flagged by bito during that review: get_timestamp_expr
hardcoded a {"epoch_s": ..., "epoch_ms": ..., "epoch_us": ...} dict re-listing
the same keys EPOCH_FORMATS already declares in superset/constants.py. A
future EPOCH_FORMATS entry (e.g. epoch_ns) would pass the `pdf in
EPOCH_FORMATS` guard but KeyError here instead of failing cleanly, since
nothing kept the two collections in sync.
Every epoch method follows f"{pdf}_to_dttm" except epoch_s, whose method
predates epoch_ms/epoch_us and kept the shorter legacy name epoch_to_dttm.
Resolving the method name via getattr(), with a one-entry alias dict for that
sole exception, removes the duplicated key list entirely: a new EPOCH_FORMATS
entry following the naming convention needs no second edit here.
Also collapses two triplicated epoch_s/epoch_ms/epoch_us test bodies
(helpers_test.py, test_core.py) into parametrized tests -- same pattern
already used by the other epoch tests #44500 added.
Looked at whether a lint rule could catch "these two collections must stay
in sync" generally: not realistically buildable without heavy false-positive
risk (no syntactic signal ties two arbitrary literals together), and this
specific fix removes the duplication rather than just detecting it, so
there's nothing left here to lint for. A plain sync-assertion test (the
pattern test_dataset_dao.py's EPOCH_FORMATS-parametrized test already uses)
is the standard, zero-false-positive way this codebase already reaches for
if this class of bug recurs elsewhere.
Co-Authored-By: Evan Rusackas <ev...@preset.io>
Co-Authored-By: Claude Sonnet 5 <no...@anthropic.com>
---
superset/db_engine_specs/base.py | 18 ++++++---
tests/unit_tests/models/helpers_test.py | 70 ++++++++-------------------------
tests/unit_tests/utils/test_core.py | 67 ++++++-------------------------
3 files changed, 41 insertions(+), 114 deletions(-)
diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py
index 5deaf1e4d00..1215f8058a7 100644
--- a/superset/db_engine_specs/base.py
+++ b/superset/db_engine_specs/base.py
@@ -322,6 +322,13 @@ AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [
}
]
+# EPOCH_FORMATS keys resolve to a f"{pdf}_to_dttm" classmethod, except
+# "epoch_s": its method predates "epoch_ms"/"epoch_us" and kept the shorter
+# legacy name `epoch_to_dttm`. Only exceptions to the naming convention
+# belong here; a new EPOCH_FORMATS entry that follows the convention needs
+# no matching entry in this dict.
+_EPOCH_METHOD_ALIASES = {"epoch_s": "epoch_to_dttm"}
+
class DBEngineSpecMetadata(TypedDict, total=False):
"""
@@ -1358,12 +1365,11 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
# if epoch, translate to DATE using db specific conf
if pdf in EPOCH_FORMATS:
- epoch_to_dttm = {
- "epoch_s": cls.epoch_to_dttm,
- "epoch_ms": cls.epoch_ms_to_dttm,
- "epoch_us": cls.epoch_us_to_dttm,
- }[pdf]
- time_expr = time_expr.replace("{col}", epoch_to_dttm())
+ # "epoch_s" predates "epoch_ms"/"epoch_us" and kept the shorter
+ # legacy method name; every other format follows f"{pdf}_to_dttm",
+ # so adding a new EPOCH_FORMATS entry needs no second edit here.
+ method_name = _EPOCH_METHOD_ALIASES.get(pdf, f"{pdf}_to_dttm")
+ time_expr = time_expr.replace("{col}", getattr(cls, method_name)())
elif pdf == "%Y":
# a bare four-digit year (e.g. the `year` column on the `video_game_sales`
# example dataset) has no native date type to lean on; without this the
diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py
index 2f395a0da01..6990fd008c1 100644
--- a/tests/unit_tests/models/helpers_test.py
+++ b/tests/unit_tests/models/helpers_test.py
@@ -4084,73 +4084,35 @@ def _raw_query_object() -> MagicMock:
return query_object
-def test_normalize_df_applies_python_date_format_to_unaggregated_columns() -> None:
+@pytest.mark.parametrize(
+ "python_date_format,values",
+ [
+ ("epoch_s", [1577836800, 1609459200, 1640995200]),
+ ("epoch_ms", [1577836800000, 1609459200000, 1640995200000]),
+ ("epoch_us", [1577836800000000, 1609459200000000, 1640995200000000]),
+ ],
+)
+def test_normalize_df_applies_python_date_format_to_unaggregated_columns(
+ python_date_format: str, values: list[int]
+) -> None:
"""A temporal column selected in a raw/unaggregated query is a plain column
name rather than a base-axis adhoc column, so it is excluded from
``get_base_axis_labels``. It must still receive its ``python_date_format``,
- so an ``epoch_s`` column is converted to datetimes the same way it is for
- aggregated charts."""
- import pandas as pd
- from pandas.api.types import is_datetime64_any_dtype
-
- ts_col = MagicMock(
- column_name="ts",
- is_dttm=True,
- python_date_format="epoch_s",
- datetime_format=None,
- )
- datasource = _normalize_df_datasource(ts_col)
-
- # 2020-01-01, 2021-01-01, 2022-01-01 as epoch seconds
- df = pd.DataFrame({"ts": [1577836800, 1609459200, 1640995200]})
-
- result = datasource.normalize_df(df, _raw_query_object())
-
- assert is_datetime64_any_dtype(result["ts"])
- assert result["ts"][0].strftime("%Y-%m-%d") == "2020-01-01"
- assert result["ts"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_df_applies_epoch_ms_to_unaggregated_columns() -> None:
- """``epoch_ms`` is a separate conversion branch from ``epoch_s``; an
- unaggregated column declaring it must also be converted to datetimes."""
- import pandas as pd
- from pandas.api.types import is_datetime64_any_dtype
-
- ts_col = MagicMock(
- column_name="ts",
- is_dttm=True,
- python_date_format="epoch_ms",
- datetime_format=None,
- )
- datasource = _normalize_df_datasource(ts_col)
-
- # 2020-01-01, 2021-01-01, 2022-01-01 as epoch milliseconds
- df = pd.DataFrame({"ts": [1577836800000, 1609459200000, 1640995200000]})
-
- result = datasource.normalize_df(df, _raw_query_object())
-
- assert is_datetime64_any_dtype(result["ts"])
- assert result["ts"][0].strftime("%Y-%m-%d") == "2020-01-01"
- assert result["ts"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_df_applies_epoch_us_to_unaggregated_columns() -> None:
- """``epoch_us`` values are microseconds; the raw-column path must convert
- them with the matching pandas unit."""
+ so an epoch column (seconds, milliseconds, or microseconds) is converted
+ to datetimes the same way it is for aggregated charts."""
import pandas as pd
from pandas.api.types import is_datetime64_any_dtype
ts_col = MagicMock(
column_name="ts",
is_dttm=True,
- python_date_format="epoch_us",
+ python_date_format=python_date_format,
datetime_format=None,
)
datasource = _normalize_df_datasource(ts_col)
- # 2020-01-01, 2021-01-01, 2022-01-01 as epoch microseconds
- df = pd.DataFrame({"ts": [1577836800000000, 1609459200000000, 1640995200000000]})
+ # values are 2020-01-01, 2021-01-01, 2022-01-01 in python_date_format's unit
+ df = pd.DataFrame({"ts": values})
result = datasource.normalize_df(df, _raw_query_object())
diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py
index 08b9fd8e9f2..6439e771edb 100644
--- a/tests/unit_tests/utils/test_core.py
+++ b/tests/unit_tests/utils/test_core.py
@@ -309,18 +309,19 @@ def test_normalize_dttm_col_mismatched_format_keeps_values() -> None:
assert df["year"].tolist() == before
-def test_normalize_dttm_col_epoch_seconds() -> None:
- """Test conversion of epoch seconds."""
- df = pd.DataFrame(
- {
- "epoch_col": [
- 1577836800,
- 1609459200,
- 1640995200,
- ] # 2020-01-01, 2021-01-01, 2022-01-01
- }
- )
- dttm_cols = (DateColumn(col_label="epoch_col", timestamp_format="epoch_s"),)
+@pytest.mark.parametrize(
+ "timestamp_format,values",
+ [
+ ("epoch_s", [1577836800, 1609459200, 1640995200]),
+ ("epoch_ms", [1577836800000, 1609459200000, 1640995200000]),
+ ("epoch_us", [1577836800000000, 1609459200000000, 1640995200000000]),
+ ],
+)
+def test_normalize_dttm_col_epoch(timestamp_format: str, values: list[int]) -> None:
+ """Test conversion of epoch seconds, milliseconds, and microseconds."""
+ # values are 2020-01-01, 2021-01-01, 2022-01-01 in timestamp_format's unit
+ df = pd.DataFrame({"epoch_col": values})
+ dttm_cols = (DateColumn(col_label="epoch_col", timestamp_format=timestamp_format),)
normalize_dttm_col(df, dttm_cols)
@@ -330,48 +331,6 @@ def test_normalize_dttm_col_epoch_seconds() -> None:
assert df["epoch_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
-def test_normalize_dttm_col_epoch_milliseconds() -> None:
- """Test conversion of epoch milliseconds."""
- df = pd.DataFrame(
- {
- "epoch_ms_col": [
- 1577836800000,
- 1609459200000,
- 1640995200000,
- ] # 2020-01-01, 2021-01-01, 2022-01-01
- }
- )
- dttm_cols = (DateColumn(col_label="epoch_ms_col", timestamp_format="epoch_ms"),)
-
- normalize_dttm_col(df, dttm_cols)
-
- assert is_datetime64_dtype(df["epoch_ms_col"])
- assert df["epoch_ms_col"][0].strftime("%Y-%m-%d") == "2020-01-01"
- assert df["epoch_ms_col"][1].strftime("%Y-%m-%d") == "2021-01-01"
- assert df["epoch_ms_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_dttm_col_epoch_microseconds() -> None:
- """Test conversion of epoch microseconds."""
- df = pd.DataFrame(
- {
- "epoch_us_col": [
- 1577836800000000,
- 1609459200000000,
- 1640995200000000,
- ] # 2020-01-01, 2021-01-01, 2022-01-01
- }
- )
- dttm_cols = (DateColumn(col_label="epoch_us_col", timestamp_format="epoch_us"),)
-
- normalize_dttm_col(df, dttm_cols)
-
- assert is_datetime64_dtype(df["epoch_us_col"])
- assert df["epoch_us_col"][0].strftime("%Y-%m-%d") == "2020-01-01"
- assert df["epoch_us_col"][1].strftime("%Y-%m-%d") == "2021-01-01"
- assert df["epoch_us_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
def test_normalize_dttm_col_formatted_date() -> None:
"""Test conversion of formatted date strings."""
df = pd.DataFrame({"date_col": ["2020-01-01", "2021-01-01", "2022-01-01"]})