You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by hu...@apache.org on 2026/10/13 18:15:23 UTC
(superset) 03/07: fix(datasets): drop only the transform a request introduces, not an echo
This is an automated email from the ASF dual-hosted git repository.
hughhhh pushed a commit to branch hughhhh/pfm-6-explore-indicator
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 027e8b54ca58fc63dfc560b8b6c3f05c0634fce0
Author: Hugh A. Miles II <hu...@gmail.com>
AuthorDate: Tue Oct 13 16:11:25 2026 +0000
fix(datasets): drop only the transform a request introduces, not an echo
`drop_unmapped_value_transforms` nulled the transform on every incoming
column the mapping does not mirror, without asking what was already
stored. Every client of these endpoints does GET-then-PUT, so the payload
carries each column back exactly as it arrived: changing the default
datetime column away from the mapped one made that unchanged copy look
like a value being parked, and nulling it destroyed configuration the
owner never touched. Unrecoverable, with the flag off -- the editor hides
the fields and the model-level pass declines to run, so switching the
column back and enabling the flag could not bring it back. That is the
same permanent loss the editor-side guard exists to prevent.
The gate's purpose is unchanged: a flag-off window is precisely when a
value gets parked for later, and an introduction is still dropped --
whether the expression is different or only the monotonicity flag is. What
it no longer does is destroy a value equal to what storage already holds,
which is the model-level pass's business under its own flag. That is the
line this function's docstring already drew for a column the payload omits
entirely; it now holds for one the payload echoes.
`stored_by_name` is optional, so a caller that cannot see storage keeps the
stricter behaviour. All three writers can, and all three pass it.
Co-Authored-By: Claude Opus 5 (1M context) <no...@anthropic.com>
Signed-off-by: Hugh A. Miles II <hu...@gmail.com>
---
superset/commands/dataset/importers/v1/utils.py | 9 ++
superset/connectors/sqla/partition_mapping.py | 32 ++++++--
superset/daos/dataset.py | 9 ++
superset/views/datasource/views.py | 10 +++
.../connectors/sqla/partition_mapping_test.py | 96 ++++++++++++++++++++++
5 files changed, 150 insertions(+), 6 deletions(-)
diff --git a/superset/commands/dataset/importers/v1/utils.py b/superset/commands/dataset/importers/v1/utils.py
index 7ad522634e6..2de2f793a6d 100644
--- a/superset/commands/dataset/importers/v1/utils.py
+++ b/superset/commands/dataset/importers/v1/utils.py
@@ -639,6 +639,15 @@ def import_dataset( # noqa: C901
main_dttm_col=config.get(
"main_dttm_col", getattr(existing, "main_dttm_col", None)
),
+ # So an overwrite that echoes the stored transform keeps it; only a
+ # value the bundle introduces on an unmapped column is dropped.
+ stored_by_name={
+ column.column_name: (
+ column.partition_value_transform,
+ bool(column.partition_transform_is_monotonic),
+ )
+ for column in getattr(existing, "columns", None) or []
+ },
):
logger.warning(
"Dropping the partition value transform on %s.%s (dataset %s) "
diff --git a/superset/connectors/sqla/partition_mapping.py b/superset/connectors/sqla/partition_mapping.py
index 69f348ef24e..7730fecdb22 100644
--- a/superset/connectors/sqla/partition_mapping.py
+++ b/superset/connectors/sqla/partition_mapping.py
@@ -2087,6 +2087,7 @@ def drop_unmapped_value_transforms(
partition_mapped_column: str | None,
main_dttm_col: str | None,
names_by_id: dict[int, str] | None = None,
+ stored_by_name: dict[str, tuple[str | None, bool]] | None = None,
) -> list[str]:
"""
Null the value transform on every *incoming* column the mapping cannot mirror.
@@ -2135,6 +2136,17 @@ def drop_unmapped_value_transforms(
one, and every transform in it was dropped. Callers whose writer keys
on the name instead (`update_from_object`, the importer) pass nothing.
An entry this cannot name is left alone rather than guessed at.
+ :param stored_by_name: ``(transform, is_monotonic)`` as currently stored,
+ per column name, so an *echo* can be told from an introduction. A
+ GET-then-PUT client sends back every column exactly as it received it,
+ so changing the default datetime column away from the mapped one made
+ the payload's unchanged copy of its transform look like a value being
+ parked -- and nulling it destroyed configuration the owner never
+ touched, unrecoverably, with the flag off and the editor's own fields
+ hidden. An unchanged value is the model-level pass's business under its
+ own flag, which is the same line this function's docstring already
+ draws for a column the payload omits. Callers that cannot see storage
+ pass nothing and keep the stricter behaviour.
"""
mapped_column = (
(partition_mapped_column or main_dttm_col) if partition_column else None
@@ -2148,12 +2160,20 @@ def drop_unmapped_value_transforms(
name = names_by_id.get(column_id)
if name is None or name == mapped_column:
continue
- if column.get("partition_value_transform") or column.get(
- "partition_transform_is_monotonic"
- ):
- column["partition_value_transform"] = None
- column["partition_transform_is_monotonic"] = False
- cleared.append(name)
+ transform = column.get("partition_value_transform")
+ is_monotonic = column.get("partition_transform_is_monotonic")
+ if not transform and not is_monotonic:
+ continue
+ if stored_by_name is not None and name in stored_by_name:
+ stored_transform, stored_monotonic = stored_by_name[name]
+ if transform == stored_transform and bool(is_monotonic) == bool(
+ stored_monotonic
+ ):
+ # An echo, not an introduction. See `stored_by_name`.
+ continue
+ column["partition_value_transform"] = None
+ column["partition_transform_is_monotonic"] = False
+ cleared.append(name)
return cleared
diff --git a/superset/daos/dataset.py b/superset/daos/dataset.py
index 18c494c8973..d241806298a 100644
--- a/superset/daos/dataset.py
+++ b/superset/daos/dataset.py
@@ -504,6 +504,15 @@ class DatasetDAO(BaseDAO[SqlaTable]):
# primary key and lets `column_name` be omitted, which is what
# the dataset editor's own PUT sends.
names_by_id={column.id: column.column_name for column in item.columns},
+ # So an unchanged echo of stored configuration is left alone;
+ # only a value this request introduces is dropped.
+ stored_by_name={
+ column.column_name: (
+ column.partition_value_transform,
+ bool(column.partition_transform_is_monotonic),
+ )
+ for column in item.columns
+ },
):
logger.info(
"Dataset %s: dropped an incoming partition value transform "
diff --git a/superset/views/datasource/views.py b/superset/views/datasource/views.py
index 3c662927a03..c6b10d82238 100644
--- a/superset/views/datasource/views.py
+++ b/superset/views/datasource/views.py
@@ -457,6 +457,16 @@ class Datasource(BaseSupersetView):
partition_column=datasource_dict.get("partition_column"),
partition_mapped_column=datasource_dict.get("partition_mapped_column"),
main_dttm_col=datasource_dict.get("main_dttm_col"),
+ # So an unchanged echo of stored configuration survives: this
+ # endpoint's own client does GET-then-POST, which is exactly the
+ # shape that looked like a value being parked.
+ stored_by_name={
+ column.column_name: (
+ column.partition_value_transform,
+ bool(column.partition_transform_is_monotonic),
+ )
+ for column in getattr(orm_datasource, "columns", None) or []
+ },
):
logger.info(
"Dataset %s: dropped an incoming partition value transform "
diff --git a/tests/unit_tests/connectors/sqla/partition_mapping_test.py b/tests/unit_tests/connectors/sqla/partition_mapping_test.py
index 932d66508a7..6cfb5a3c08a 100644
--- a/tests/unit_tests/connectors/sqla/partition_mapping_test.py
+++ b/tests/unit_tests/connectors/sqla/partition_mapping_test.py
@@ -2962,6 +2962,102 @@ def test_without_a_partition_column_no_incoming_transform_survives() -> None:
assert columns[0]["partition_value_transform"] is None
+def test_an_echo_of_a_stored_transform_is_not_dropped() -> None:
+ """
+ The GET-then-PUT shape every client of these endpoints uses: the payload
+ carries every column back exactly as it arrived. Changing the default
+ datetime column away from the mapped one then made the unchanged copy of
+ its transform look like a value being parked, and nulling it destroyed
+ configuration the owner never touched -- unrecoverably, since with the flag
+ off the editor hides the fields and the model-level pass declines to run.
+
+ An unchanged value is the model-level pass's business, under its own flag,
+ which is the line this function already draws for a column the payload
+ omits entirely.
+ """
+ columns = [
+ {
+ "column_name": "event_time",
+ "partition_value_transform": "unix_timestamp(:value)",
+ "partition_transform_is_monotonic": True,
+ }
+ ]
+ stored = {"event_time": ("unix_timestamp(:value)", True)}
+
+ # `event_time2` is the newly chosen default, so `event_time` is no longer
+ # the mapped column -- and its transform is nonetheless an echo.
+ assert (
+ drop_unmapped_value_transforms(
+ columns,
+ partition_column="dt_epoch",
+ partition_mapped_column=None,
+ main_dttm_col="event_time2",
+ stored_by_name=stored,
+ )
+ == []
+ )
+ assert columns[0]["partition_value_transform"] == "unix_timestamp(:value)"
+ assert columns[0]["partition_transform_is_monotonic"] is True
+
+
+@pytest.mark.parametrize(
+ ("transform", "monotonic"),
+ [
+ # A different expression...
+ ("to_unixtime(:value)", True),
+ # ...or the same expression with the flag flipped. Either is the
+ # request introducing something, which is what the gate is for.
+ ("unix_timestamp(:value)", False),
+ ],
+)
+def test_a_changed_transform_on_an_unmapped_column_is_still_dropped(
+ transform: str, monotonic: bool
+) -> None:
+ """
+ The other half: knowing what is stored must not turn the gate off. A
+ flag-off window is precisely when a value gets parked for later, and this
+ is the writer that stops it.
+ """
+ columns = [
+ {
+ "column_name": "event_time",
+ "partition_value_transform": transform,
+ "partition_transform_is_monotonic": monotonic,
+ }
+ ]
+
+ assert drop_unmapped_value_transforms(
+ columns,
+ partition_column="dt_epoch",
+ partition_mapped_column=None,
+ main_dttm_col="event_time2",
+ stored_by_name={"event_time": ("unix_timestamp(:value)", True)},
+ ) == ["event_time"]
+ assert columns[0]["partition_value_transform"] is None
+
+
+def test_a_caller_that_cannot_see_storage_keeps_the_stricter_behaviour() -> None:
+ """
+ `stored_by_name` is optional, and omitting it drops an unmapped column's
+ transform whether or not it is an echo -- the behaviour every caller had
+ before. A column storage does not mention is treated as an introduction
+ for the same reason.
+ """
+ columns = [
+ {
+ "column_name": "event_time",
+ "partition_value_transform": "unix_timestamp(:value)",
+ }
+ ]
+
+ assert drop_unmapped_value_transforms(
+ columns,
+ partition_column="dt_epoch",
+ partition_mapped_column=None,
+ main_dttm_col="event_time2",
+ ) == ["event_time"]
+
+
def test_a_column_the_payload_says_nothing_about_is_left_alone() -> None:
"""
"Incoming" is the whole scope. A transform already in storage on a column