You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/09/29 02:48:21 UTC

(superset) 01/01: refactor(db_engine_specs): derive epoch method dispatch from EPOCH_FORMATS

This is an automated email from the ASF dual-hosted git repository.

rusackas pushed a commit to branch chore/epoch-format-dispatch-dry
in repository https://gitbox.apache.org/repos/asf/superset.git

commit 02bae63d34b378ad133fba84a5db9a8c4b2c1b63
Author: Evan Rusackas <ev...@rusackas.com>
AuthorDate: Mon Sep 28 19:47:41 2026 -0700

    refactor(db_engine_specs): derive epoch method dispatch from EPOCH_FORMATS
    
    Follow-up to #44500, flagged by bito during that review: get_timestamp_expr
    hardcoded a {"epoch_s": ..., "epoch_ms": ..., "epoch_us": ...} dict re-listing
    the same keys EPOCH_FORMATS already declares in superset/constants.py. A
    future EPOCH_FORMATS entry (e.g. epoch_ns) would pass the `pdf in
    EPOCH_FORMATS` guard but KeyError here instead of failing cleanly, since
    nothing kept the two collections in sync.
    
    Every epoch method follows f"{pdf}_to_dttm" except epoch_s, whose method
    predates epoch_ms/epoch_us and kept the shorter legacy name epoch_to_dttm.
    Resolving the method name via getattr(), with a one-entry alias dict for that
    sole exception, removes the duplicated key list entirely: a new EPOCH_FORMATS
    entry following the naming convention needs no second edit here.
    
    Also collapses two triplicated epoch_s/epoch_ms/epoch_us test bodies
    (helpers_test.py, test_core.py) into parametrized tests -- same pattern
    already used by the other epoch tests #44500 added.
    
    Looked at whether a lint rule could catch "these two collections must stay
    in sync" generally: not realistically buildable without heavy false-positive
    risk (no syntactic signal ties two arbitrary literals together), and this
    specific fix removes the duplication rather than just detecting it, so
    there's nothing left here to lint for. A plain sync-assertion test (the
    pattern test_dataset_dao.py's EPOCH_FORMATS-parametrized test already uses)
    is the standard, zero-false-positive way this codebase already reaches for
    if this class of bug recurs elsewhere.
    
    Co-Authored-By: Evan Rusackas <ev...@preset.io>
    Co-Authored-By: Claude Sonnet 5 <no...@anthropic.com>
---
 superset/db_engine_specs/base.py        | 18 ++++++---
 tests/unit_tests/models/helpers_test.py | 70 ++++++++-------------------------
 tests/unit_tests/utils/test_core.py     | 67 ++++++-------------------------
 3 files changed, 41 insertions(+), 114 deletions(-)

diff --git a/superset/db_engine_specs/base.py b/superset/db_engine_specs/base.py
index 5deaf1e4d00..1215f8058a7 100644
--- a/superset/db_engine_specs/base.py
+++ b/superset/db_engine_specs/base.py
@@ -322,6 +322,13 @@ AURORA_DATA_API_KNOWN_INCOMPATIBILITIES: list[KnownIncompatibility] = [
     }
 ]
 
+# EPOCH_FORMATS keys resolve to a f"{pdf}_to_dttm" classmethod, except
+# "epoch_s": its method predates "epoch_ms"/"epoch_us" and kept the shorter
+# legacy name `epoch_to_dttm`. Only exceptions to the naming convention
+# belong here; a new EPOCH_FORMATS entry that follows the convention needs
+# no matching entry in this dict.
+_EPOCH_METHOD_ALIASES = {"epoch_s": "epoch_to_dttm"}
+
 
 class DBEngineSpecMetadata(TypedDict, total=False):
     """
@@ -1358,12 +1365,11 @@ class BaseEngineSpec:  # pylint: disable=too-many-public-methods
 
         # if epoch, translate to DATE using db specific conf
         if pdf in EPOCH_FORMATS:
-            epoch_to_dttm = {
-                "epoch_s": cls.epoch_to_dttm,
-                "epoch_ms": cls.epoch_ms_to_dttm,
-                "epoch_us": cls.epoch_us_to_dttm,
-            }[pdf]
-            time_expr = time_expr.replace("{col}", epoch_to_dttm())
+            # "epoch_s" predates "epoch_ms"/"epoch_us" and kept the shorter
+            # legacy method name; every other format follows f"{pdf}_to_dttm",
+            # so adding a new EPOCH_FORMATS entry needs no second edit here.
+            method_name = _EPOCH_METHOD_ALIASES.get(pdf, f"{pdf}_to_dttm")
+            time_expr = time_expr.replace("{col}", getattr(cls, method_name)())
         elif pdf == "%Y":
             # a bare four-digit year (e.g. the `year` column on the `video_game_sales`
             # example dataset) has no native date type to lean on; without this the
diff --git a/tests/unit_tests/models/helpers_test.py b/tests/unit_tests/models/helpers_test.py
index 2f395a0da01..6990fd008c1 100644
--- a/tests/unit_tests/models/helpers_test.py
+++ b/tests/unit_tests/models/helpers_test.py
@@ -4084,73 +4084,35 @@ def _raw_query_object() -> MagicMock:
     return query_object
 
 
-def test_normalize_df_applies_python_date_format_to_unaggregated_columns() -> None:
+@pytest.mark.parametrize(
+    "python_date_format,values",
+    [
+        ("epoch_s", [1577836800, 1609459200, 1640995200]),
+        ("epoch_ms", [1577836800000, 1609459200000, 1640995200000]),
+        ("epoch_us", [1577836800000000, 1609459200000000, 1640995200000000]),
+    ],
+)
+def test_normalize_df_applies_python_date_format_to_unaggregated_columns(
+    python_date_format: str, values: list[int]
+) -> None:
     """A temporal column selected in a raw/unaggregated query is a plain column
     name rather than a base-axis adhoc column, so it is excluded from
     ``get_base_axis_labels``. It must still receive its ``python_date_format``,
-    so an ``epoch_s`` column is converted to datetimes the same way it is for
-    aggregated charts."""
-    import pandas as pd
-    from pandas.api.types import is_datetime64_any_dtype
-
-    ts_col = MagicMock(
-        column_name="ts",
-        is_dttm=True,
-        python_date_format="epoch_s",
-        datetime_format=None,
-    )
-    datasource = _normalize_df_datasource(ts_col)
-
-    # 2020-01-01, 2021-01-01, 2022-01-01 as epoch seconds
-    df = pd.DataFrame({"ts": [1577836800, 1609459200, 1640995200]})
-
-    result = datasource.normalize_df(df, _raw_query_object())
-
-    assert is_datetime64_any_dtype(result["ts"])
-    assert result["ts"][0].strftime("%Y-%m-%d") == "2020-01-01"
-    assert result["ts"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_df_applies_epoch_ms_to_unaggregated_columns() -> None:
-    """``epoch_ms`` is a separate conversion branch from ``epoch_s``; an
-    unaggregated column declaring it must also be converted to datetimes."""
-    import pandas as pd
-    from pandas.api.types import is_datetime64_any_dtype
-
-    ts_col = MagicMock(
-        column_name="ts",
-        is_dttm=True,
-        python_date_format="epoch_ms",
-        datetime_format=None,
-    )
-    datasource = _normalize_df_datasource(ts_col)
-
-    # 2020-01-01, 2021-01-01, 2022-01-01 as epoch milliseconds
-    df = pd.DataFrame({"ts": [1577836800000, 1609459200000, 1640995200000]})
-
-    result = datasource.normalize_df(df, _raw_query_object())
-
-    assert is_datetime64_any_dtype(result["ts"])
-    assert result["ts"][0].strftime("%Y-%m-%d") == "2020-01-01"
-    assert result["ts"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_df_applies_epoch_us_to_unaggregated_columns() -> None:
-    """``epoch_us`` values are microseconds; the raw-column path must convert
-    them with the matching pandas unit."""
+    so an epoch column (seconds, milliseconds, or microseconds) is converted
+    to datetimes the same way it is for aggregated charts."""
     import pandas as pd
     from pandas.api.types import is_datetime64_any_dtype
 
     ts_col = MagicMock(
         column_name="ts",
         is_dttm=True,
-        python_date_format="epoch_us",
+        python_date_format=python_date_format,
         datetime_format=None,
     )
     datasource = _normalize_df_datasource(ts_col)
 
-    # 2020-01-01, 2021-01-01, 2022-01-01 as epoch microseconds
-    df = pd.DataFrame({"ts": [1577836800000000, 1609459200000000, 1640995200000000]})
+    # values are 2020-01-01, 2021-01-01, 2022-01-01 in python_date_format's unit
+    df = pd.DataFrame({"ts": values})
 
     result = datasource.normalize_df(df, _raw_query_object())
 
diff --git a/tests/unit_tests/utils/test_core.py b/tests/unit_tests/utils/test_core.py
index 08b9fd8e9f2..6439e771edb 100644
--- a/tests/unit_tests/utils/test_core.py
+++ b/tests/unit_tests/utils/test_core.py
@@ -309,18 +309,19 @@ def test_normalize_dttm_col_mismatched_format_keeps_values() -> None:
     assert df["year"].tolist() == before
 
 
-def test_normalize_dttm_col_epoch_seconds() -> None:
-    """Test conversion of epoch seconds."""
-    df = pd.DataFrame(
-        {
-            "epoch_col": [
-                1577836800,
-                1609459200,
-                1640995200,
-            ]  # 2020-01-01, 2021-01-01, 2022-01-01
-        }
-    )
-    dttm_cols = (DateColumn(col_label="epoch_col", timestamp_format="epoch_s"),)
+@pytest.mark.parametrize(
+    "timestamp_format,values",
+    [
+        ("epoch_s", [1577836800, 1609459200, 1640995200]),
+        ("epoch_ms", [1577836800000, 1609459200000, 1640995200000]),
+        ("epoch_us", [1577836800000000, 1609459200000000, 1640995200000000]),
+    ],
+)
+def test_normalize_dttm_col_epoch(timestamp_format: str, values: list[int]) -> None:
+    """Test conversion of epoch seconds, milliseconds, and microseconds."""
+    # values are 2020-01-01, 2021-01-01, 2022-01-01 in timestamp_format's unit
+    df = pd.DataFrame({"epoch_col": values})
+    dttm_cols = (DateColumn(col_label="epoch_col", timestamp_format=timestamp_format),)
 
     normalize_dttm_col(df, dttm_cols)
 
@@ -330,48 +331,6 @@ def test_normalize_dttm_col_epoch_seconds() -> None:
     assert df["epoch_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
 
 
-def test_normalize_dttm_col_epoch_milliseconds() -> None:
-    """Test conversion of epoch milliseconds."""
-    df = pd.DataFrame(
-        {
-            "epoch_ms_col": [
-                1577836800000,
-                1609459200000,
-                1640995200000,
-            ]  # 2020-01-01, 2021-01-01, 2022-01-01
-        }
-    )
-    dttm_cols = (DateColumn(col_label="epoch_ms_col", timestamp_format="epoch_ms"),)
-
-    normalize_dttm_col(df, dttm_cols)
-
-    assert is_datetime64_dtype(df["epoch_ms_col"])
-    assert df["epoch_ms_col"][0].strftime("%Y-%m-%d") == "2020-01-01"
-    assert df["epoch_ms_col"][1].strftime("%Y-%m-%d") == "2021-01-01"
-    assert df["epoch_ms_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
-def test_normalize_dttm_col_epoch_microseconds() -> None:
-    """Test conversion of epoch microseconds."""
-    df = pd.DataFrame(
-        {
-            "epoch_us_col": [
-                1577836800000000,
-                1609459200000000,
-                1640995200000000,
-            ]  # 2020-01-01, 2021-01-01, 2022-01-01
-        }
-    )
-    dttm_cols = (DateColumn(col_label="epoch_us_col", timestamp_format="epoch_us"),)
-
-    normalize_dttm_col(df, dttm_cols)
-
-    assert is_datetime64_dtype(df["epoch_us_col"])
-    assert df["epoch_us_col"][0].strftime("%Y-%m-%d") == "2020-01-01"
-    assert df["epoch_us_col"][1].strftime("%Y-%m-%d") == "2021-01-01"
-    assert df["epoch_us_col"][2].strftime("%Y-%m-%d") == "2022-01-01"
-
-
 def test_normalize_dttm_col_formatted_date() -> None:
     """Test conversion of formatted date strings."""
     df = pd.DataFrame({"date_col": ["2020-01-01", "2021-01-01", "2022-01-01"]})