You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by hu...@apache.org on 2026/10/13 18:15:23 UTC

(superset) 03/07: fix(datasets): drop only the transform a request introduces, not an echo

This is an automated email from the ASF dual-hosted git repository.

hughhhh pushed a commit to branch hughhhh/pfm-6-explore-indicator
in repository https://gitbox.apache.org/repos/asf/superset.git

commit 027e8b54ca58fc63dfc560b8b6c3f05c0634fce0
Author: Hugh A. Miles II <hu...@gmail.com>
AuthorDate: Tue Oct 13 16:11:25 2026 +0000

    fix(datasets): drop only the transform a request introduces, not an echo
    
    `drop_unmapped_value_transforms` nulled the transform on every incoming
    column the mapping does not mirror, without asking what was already
    stored. Every client of these endpoints does GET-then-PUT, so the payload
    carries each column back exactly as it arrived: changing the default
    datetime column away from the mapped one made that unchanged copy look
    like a value being parked, and nulling it destroyed configuration the
    owner never touched. Unrecoverable, with the flag off -- the editor hides
    the fields and the model-level pass declines to run, so switching the
    column back and enabling the flag could not bring it back. That is the
    same permanent loss the editor-side guard exists to prevent.
    
    The gate's purpose is unchanged: a flag-off window is precisely when a
    value gets parked for later, and an introduction is still dropped --
    whether the expression is different or only the monotonicity flag is. What
    it no longer does is destroy a value equal to what storage already holds,
    which is the model-level pass's business under its own flag. That is the
    line this function's docstring already drew for a column the payload omits
    entirely; it now holds for one the payload echoes.
    
    `stored_by_name` is optional, so a caller that cannot see storage keeps the
    stricter behaviour. All three writers can, and all three pass it.
    
    Co-Authored-By: Claude Opus 5 (1M context) <no...@anthropic.com>
    Signed-off-by: Hugh A. Miles II <hu...@gmail.com>
---
 superset/commands/dataset/importers/v1/utils.py    |  9 ++
 superset/connectors/sqla/partition_mapping.py      | 32 ++++++--
 superset/daos/dataset.py                           |  9 ++
 superset/views/datasource/views.py                 | 10 +++
 .../connectors/sqla/partition_mapping_test.py      | 96 ++++++++++++++++++++++
 5 files changed, 150 insertions(+), 6 deletions(-)

diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py
index 7ad522634e6..2de2f793a6d 100644
--- a/superset/commands/dataset/importers/v1/utils.py
+++ b/superset/commands/dataset/importers/v1/utils.py
@@ -639,6 +639,15 @@ def import_dataset(  # noqa: C901
         main_dttm_col=config.get(
             "main_dttm_col", getattr(existing, "main_dttm_col", None)
         ),
+        # So an overwrite that echoes the stored transform keeps it; only a
+        # value the bundle introduces on an unmapped column is dropped.
+        stored_by_name={
+            column.column_name: (
+                column.partition_value_transform,
+                bool(column.partition_transform_is_monotonic),
+            )
+            for column in getattr(existing, "columns", None) or []
+        },
     ):
         logger.warning(
             "Dropping the partition value transform on %s.%s (dataset %s) "
diff --git a/superset/connectors/sqla/partition_mapping.py b/superset/connectors/sqla/partition_mapping.py
index 69f348ef24e..7730fecdb22 100644
--- a/superset/connectors/sqla/partition_mapping.py
+++ b/superset/connectors/sqla/partition_mapping.py
@@ -2087,6 +2087,7 @@ def drop_unmapped_value_transforms(
     partition_mapped_column: str | None,
     main_dttm_col: str | None,
     names_by_id: dict[int, str] | None = None,
+    stored_by_name: dict[str, tuple[str | None, bool]] | None = None,
 ) -> list[str]:
     """
     Null the value transform on every *incoming* column the mapping cannot mirror.
@@ -2135,6 +2136,17 @@ def drop_unmapped_value_transforms(
         one, and every transform in it was dropped. Callers whose writer keys
         on the name instead (`update_from_object`, the importer) pass nothing.
         An entry this cannot name is left alone rather than guessed at.
+    :param stored_by_name: ``(transform, is_monotonic)`` as currently stored,
+        per column name, so an *echo* can be told from an introduction. A
+        GET-then-PUT client sends back every column exactly as it received it,
+        so changing the default datetime column away from the mapped one made
+        the payload's unchanged copy of its transform look like a value being
+        parked -- and nulling it destroyed configuration the owner never
+        touched, unrecoverably, with the flag off and the editor's own fields
+        hidden. An unchanged value is the model-level pass's business under its
+        own flag, which is the same line this function's docstring already
+        draws for a column the payload omits. Callers that cannot see storage
+        pass nothing and keep the stricter behaviour.
     """
     mapped_column = (
         (partition_mapped_column or main_dttm_col) if partition_column else None
@@ -2148,12 +2160,20 @@ def drop_unmapped_value_transforms(
                 name = names_by_id.get(column_id)
         if name is None or name == mapped_column:
             continue
-        if column.get("partition_value_transform") or column.get(
-            "partition_transform_is_monotonic"
-        ):
-            column["partition_value_transform"] = None
-            column["partition_transform_is_monotonic"] = False
-            cleared.append(name)
+        transform = column.get("partition_value_transform")
+        is_monotonic = column.get("partition_transform_is_monotonic")
+        if not transform and not is_monotonic:
+            continue
+        if stored_by_name is not None and name in stored_by_name:
+            stored_transform, stored_monotonic = stored_by_name[name]
+            if transform == stored_transform and bool(is_monotonic) == bool(
+                stored_monotonic
+            ):
+                # An echo, not an introduction. See `stored_by_name`.
+                continue
+        column["partition_value_transform"] = None
+        column["partition_transform_is_monotonic"] = False
+        cleared.append(name)
     return cleared
 
 
diff --git a/superset/daos/dataset.py b/superset/daos/dataset.py
index 18c494c8973..d241806298a 100644
--- a/superset/daos/dataset.py
+++ b/superset/daos/dataset.py
@@ -504,6 +504,15 @@ class DatasetDAO(BaseDAO[SqlaTable]):
                 # primary key and lets `column_name` be omitted, which is what
                 # the dataset editor's own PUT sends.
                 names_by_id={column.id: column.column_name for column in item.columns},
+                # So an unchanged echo of stored configuration is left alone;
+                # only a value this request introduces is dropped.
+                stored_by_name={
+                    column.column_name: (
+                        column.partition_value_transform,
+                        bool(column.partition_transform_is_monotonic),
+                    )
+                    for column in item.columns
+                },
             ):
                 logger.info(
                     "Dataset %s: dropped an incoming partition value transform "
diff --git a/superset/views/datasource/views.py b/superset/views/datasource/views.py
index 3c662927a03..c6b10d82238 100644
--- a/superset/views/datasource/views.py
+++ b/superset/views/datasource/views.py
@@ -457,6 +457,16 @@ class Datasource(BaseSupersetView):
             partition_column=datasource_dict.get("partition_column"),
             partition_mapped_column=datasource_dict.get("partition_mapped_column"),
             main_dttm_col=datasource_dict.get("main_dttm_col"),
+            # So an unchanged echo of stored configuration survives: this
+            # endpoint's own client does GET-then-POST, which is exactly the
+            # shape that looked like a value being parked.
+            stored_by_name={
+                column.column_name: (
+                    column.partition_value_transform,
+                    bool(column.partition_transform_is_monotonic),
+                )
+                for column in getattr(orm_datasource, "columns", None) or []
+            },
         ):
             logger.info(
                 "Dataset %s: dropped an incoming partition value transform "
diff --git a/tests/unit_tests/connectors/sqla/partition_mapping_test.py b/tests/unit_tests/connectors/sqla/partition_mapping_test.py
index 932d66508a7..6cfb5a3c08a 100644
--- a/tests/unit_tests/connectors/sqla/partition_mapping_test.py
+++ b/tests/unit_tests/connectors/sqla/partition_mapping_test.py
@@ -2962,6 +2962,102 @@ def test_without_a_partition_column_no_incoming_transform_survives() -> None:
     assert columns[0]["partition_value_transform"] is None
 
 
+def test_an_echo_of_a_stored_transform_is_not_dropped() -> None:
+    """
+    The GET-then-PUT shape every client of these endpoints uses: the payload
+    carries every column back exactly as it arrived. Changing the default
+    datetime column away from the mapped one then made the unchanged copy of
+    its transform look like a value being parked, and nulling it destroyed
+    configuration the owner never touched -- unrecoverably, since with the flag
+    off the editor hides the fields and the model-level pass declines to run.
+
+    An unchanged value is the model-level pass's business, under its own flag,
+    which is the line this function already draws for a column the payload
+    omits entirely.
+    """
+    columns = [
+        {
+            "column_name": "event_time",
+            "partition_value_transform": "unix_timestamp(:value)",
+            "partition_transform_is_monotonic": True,
+        }
+    ]
+    stored = {"event_time": ("unix_timestamp(:value)", True)}
+
+    # `event_time2` is the newly chosen default, so `event_time` is no longer
+    # the mapped column -- and its transform is nonetheless an echo.
+    assert (
+        drop_unmapped_value_transforms(
+            columns,
+            partition_column="dt_epoch",
+            partition_mapped_column=None,
+            main_dttm_col="event_time2",
+            stored_by_name=stored,
+        )
+        == []
+    )
+    assert columns[0]["partition_value_transform"] == "unix_timestamp(:value)"
+    assert columns[0]["partition_transform_is_monotonic"] is True
+
+
+@pytest.mark.parametrize(
+    ("transform", "monotonic"),
+    [
+        # A different expression...
+        ("to_unixtime(:value)", True),
+        # ...or the same expression with the flag flipped. Either is the
+        # request introducing something, which is what the gate is for.
+        ("unix_timestamp(:value)", False),
+    ],
+)
+def test_a_changed_transform_on_an_unmapped_column_is_still_dropped(
+    transform: str, monotonic: bool
+) -> None:
+    """
+    The other half: knowing what is stored must not turn the gate off. A
+    flag-off window is precisely when a value gets parked for later, and this
+    is the writer that stops it.
+    """
+    columns = [
+        {
+            "column_name": "event_time",
+            "partition_value_transform": transform,
+            "partition_transform_is_monotonic": monotonic,
+        }
+    ]
+
+    assert drop_unmapped_value_transforms(
+        columns,
+        partition_column="dt_epoch",
+        partition_mapped_column=None,
+        main_dttm_col="event_time2",
+        stored_by_name={"event_time": ("unix_timestamp(:value)", True)},
+    ) == ["event_time"]
+    assert columns[0]["partition_value_transform"] is None
+
+
+def test_a_caller_that_cannot_see_storage_keeps_the_stricter_behaviour() -> None:
+    """
+    `stored_by_name` is optional, and omitting it drops an unmapped column's
+    transform whether or not it is an echo -- the behaviour every caller had
+    before. A column storage does not mention is treated as an introduction
+    for the same reason.
+    """
+    columns = [
+        {
+            "column_name": "event_time",
+            "partition_value_transform": "unix_timestamp(:value)",
+        }
+    ]
+
+    assert drop_unmapped_value_transforms(
+        columns,
+        partition_column="dt_epoch",
+        partition_mapped_column=None,
+        main_dttm_col="event_time2",
+    ) == ["event_time"]
+
+
 def test_a_column_the_payload_says_nothing_about_is_left_alone() -> None:
     """
     "Incoming" is the whole scope. A transform already in storage on a column