You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/06/19 17:02:41 UTC
(superset) branch master updated: chore(deps): bump marshmallow from 3.26.2 to 4.3.0 (#39751)
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 919bd350288 chore(deps): bump marshmallow from 3.26.2 to 4.3.0 (#39751)
919bd350288 is described below
commit 919bd35028887803c1722c8ee5267b17d5180853
Author: dependabot[bot] <49...@users.noreply.github.com>
AuthorDate: Fri Jun 19 10:02:35 2026 -0700
chore(deps): bump marshmallow from 3.26.2 to 4.3.0 (#39751)
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
Co-authored-by: Evan <ev...@preset.io>
Co-authored-by: Claude Sonnet 4.6 <no...@anthropic.com>
Co-authored-by: Amin Ghadersohi <am...@gmail.com>
---
pyproject.toml | 6 +-
requirements/base.in | 9 +-
requirements/base.txt | 4 +-
requirements/development.txt | 4 +-
superset/app.py | 3 +
superset/charts/schemas.py | 4 +-
superset/databases/schemas.py | 2 +-
superset/marshmallow_compatibility.py | 88 ++++++++++++++
superset/reports/schemas.py | 6 +-
superset/themes/schemas.py | 10 +-
tests/unit_tests/dashboards/api_test.py | 5 +
tests/unit_tests/test_marshmallow4_compat.py | 164 +++++++++++++++++++++++++++
12 files changed, 285 insertions(+), 20 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 3facbd4555c..b5f540c0433 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,8 +67,10 @@ dependencies = [
"jsonpath-ng>=1.8.0, <2",
"Mako>=1.2.2",
"markdown>=3.10.2",
- # marshmallow>=4 has issues: https://github.com/apache/superset/issues/33162
- "marshmallow>=3.0, <4",
+ # marshmallow 4 compatibility: see superset/marshmallow_compatibility.py for a
+ # Flask-AppBuilder workaround. Tracking issue:
+ # https://github.com/apache/superset/issues/33162
+ "marshmallow>=3.0, <5",
"marshmallow-union>=0.1",
"msgpack>=1.0.0, <1.2",
"nh3>=0.3.5, <0.4",
diff --git a/requirements/base.in b/requirements/base.in
index b34d4afc04a..374c8d19469 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -44,11 +44,10 @@ async_timeout>=4.0.0,<5.0.0
# a bit of attention to bump.
apispec>=6.0.0,<6.7.0
-# 1.4.1 appears to use much more memory, where the python test suite runs out of memory
-# causing CI to fail. 1.4.0 is the last version that works.
-# https://marshmallow-sqlalchemy.readthedocs.io/en/latest/changelog.html#id3
-# Opened this issue https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/665
-marshmallow-sqlalchemy>=1.3.0,<1.4.1
+# 1.4.1 introduced a memory regression that exhausts memory in the test suite
+# (https://github.com/marshmallow-code/marshmallow-sqlalchemy/issues/665). 1.4.2
+# claimed a fix but did not address the root cause; only 1.5.0 actually fixes it.
+marshmallow-sqlalchemy>=1.5.0
# needed for python 3.12 support
openapi-schema-validator>=0.6.3
diff --git a/requirements/base.txt b/requirements/base.txt
index a60bddc7378..e8ec36d67a8 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -223,13 +223,13 @@ markupsafe==3.0.2
# mako
# werkzeug
# wtforms
-marshmallow==3.26.2
+marshmallow==4.3.0
# via
# apache-superset (pyproject.toml)
# flask-appbuilder
# marshmallow-sqlalchemy
# marshmallow-union
-marshmallow-sqlalchemy==1.4.0
+marshmallow-sqlalchemy==1.5.0
# via
# -r requirements/base.in
# flask-appbuilder
diff --git a/requirements/development.txt b/requirements/development.txt
index c948d7e80bc..cabef0499f4 100644
--- a/requirements/development.txt
+++ b/requirements/development.txt
@@ -527,14 +527,14 @@ markupsafe==3.0.2
# mako
# werkzeug
# wtforms
-marshmallow==3.26.2
+marshmallow==4.3.0
# via
# -c requirements/base-constraint.txt
# apache-superset
# flask-appbuilder
# marshmallow-sqlalchemy
# marshmallow-union
-marshmallow-sqlalchemy==1.4.0
+marshmallow-sqlalchemy==1.5.0
# via
# -c requirements/base-constraint.txt
# flask-appbuilder
diff --git a/superset/app.py b/superset/app.py
index e8671c4f88e..391d633488d 100644
--- a/superset/app.py
+++ b/superset/app.py
@@ -41,6 +41,9 @@ from superset.extensions.local_extensions_watcher import (
start_local_extensions_watcher_thread,
)
from superset.initialization import SupersetAppInitializer
+from superset.marshmallow_compatibility import patch_marshmallow_for_flask_appbuilder
+
+patch_marshmallow_for_flask_appbuilder()
logger = logging.getLogger(__name__)
diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py
index 26e2470fbff..1e514a20950 100644
--- a/superset/charts/schemas.py
+++ b/superset/charts/schemas.py
@@ -855,7 +855,9 @@ class ChartDataPivotOptionsSchema(ChartDataPostProcessingOperationOptionsSchema)
fields.String(allow_none=False),
metadata={"description": "Columns to group by on the table columns"},
)
- metric_fill_value = fields.Number(
+ # `fields.Number` became abstract in marshmallow 4; use `Float`, which
+ # preserves the previous "any numeric value" semantics for this field.
+ metric_fill_value = fields.Float(
metadata={
"description": "Value to replace missing values with in "
"aggregate calculations."
diff --git a/superset/databases/schemas.py b/superset/databases/schemas.py
index ce4cbda151f..32a949eec0d 100644
--- a/superset/databases/schemas.py
+++ b/superset/databases/schemas.py
@@ -1208,7 +1208,7 @@ class DelimitedListField(fields.List):
class BaseUploadFilePostSchemaMixin(Schema):
@validates("file")
- def validate_file_extension(self, file: FileStorage) -> None:
+ def validate_file_extension(self, file: FileStorage, **kwargs: Any) -> None:
allowed_extensions = current_app.config["ALLOWED_EXTENSIONS"]
file_suffix = Path(file.filename).suffix
if not file_suffix:
diff --git a/superset/marshmallow_compatibility.py b/superset/marshmallow_compatibility.py
new file mode 100644
index 00000000000..852627b4085
--- /dev/null
+++ b/superset/marshmallow_compatibility.py
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Marshmallow 4.x compatibility patch for Flask-AppBuilder.
+
+Flask-AppBuilder auto-generates schema classes (via
+``Model2SchemaConverter._meta_schema_factory``) whose ``Meta.fields`` reference
+SQLAlchemy relationship names that are never declared as marshmallow fields.
+
+Marshmallow 3 tolerated this: ``_init_fields`` resolved each name with
+``declared_fields.get(name, Inferred())``. Marshmallow 4 made the lookup strict
+(``declared_fields[name]``) and raises ``KeyError`` for those names. This patch
+restores the lenient behaviour by stubbing the missing FAB names with ``Raw``
+fields before ``_init_fields`` runs.
+"""
+
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def patch_marshmallow_for_flask_appbuilder() -> None:
+ """Apply a compatibility patch to marshmallow for Flask-AppBuilder.
+
+ Wraps ``marshmallow.Schema._init_fields`` so that any name referenced by a
+ schema's ``Meta.fields``/``Meta.additional`` that is not backed by a declared
+ field is stubbed with a ``Raw`` field. This mirrors marshmallow 3 semantics
+ and keeps Flask-AppBuilder's auto-generated schemas working under
+ marshmallow 4, where the field lookup became strict.
+
+ The stubbing happens in a single pass per instantiation (no retry loop), so
+ it stays cheap even though FAB mints a fresh schema class on every call and
+ marshmallow rebuilds ``declared_fields`` for every instance.
+
+ The patch is idempotent and will not apply twice.
+ """
+ import marshmallow
+
+ if getattr(marshmallow.Schema._init_fields, "_fab_patched", False):
+ return # already patched
+
+ original_init_fields = marshmallow.Schema._init_fields
+
+ def patched_init_fields(self: "marshmallow.Schema") -> Any:
+ opts = self.opts
+ # ``_init_fields`` iterates Meta.fields (falling back to declared field
+ # names) plus Meta.additional, then strictly looks each up in
+ # ``declared_fields``. Pre-stub the FAB relationship names it would
+ # otherwise choke on; unknown non-FAB names are left to raise so genuine
+ # schema bugs still surface.
+ candidates = set(opts.fields or ()) | set(getattr(opts, "additional", ()) or ())
+ for name in candidates:
+ if name not in self.declared_fields and _looks_like_fab_field(name):
+ logger.debug(
+ "marshmallow FAB compat: stubbing Raw field for %r on %s",
+ name,
+ type(self).__name__,
+ )
+ self.declared_fields[name] = marshmallow.fields.Raw()
+ return original_init_fields(self)
+
+ patched_init_fields._fab_patched = True # type: ignore[attr-defined]
+ marshmallow.Schema._init_fields = patched_init_fields
+
+
+def _looks_like_fab_field(name: str) -> bool:
+ """Return True if the field name looks like a FAB auto-generated relationship field.
+
+ FAB generates fields like 'related_model', 'parent_id', etc.
+ These are typically snake_case and don't start with underscore.
+ """
+ return bool(name) and not name.startswith("_") and name.replace("_", "").isalnum()
diff --git a/superset/reports/schemas.py b/superset/reports/schemas.py
index 8f7314d2aa1..5bc1516bae6 100644
--- a/superset/reports/schemas.py
+++ b/superset/reports/schemas.py
@@ -270,13 +270,14 @@ class ReportSchedulePostSchema(Schema):
},
allow_none=True,
required=False,
- default=None,
+ dump_default=None,
)
@validates("custom_width")
def validate_custom_width(
self,
value: Optional[int],
+ **kwargs: Any,
) -> None:
if value is None:
return
@@ -432,13 +433,14 @@ class ReportSchedulePutSchema(Schema):
},
allow_none=True,
required=False,
- default=None,
+ dump_default=None,
)
@validates("custom_width")
def validate_custom_width(
self,
value: Optional[int],
+ **kwargs: Any,
) -> None:
if value is None:
return
diff --git a/superset/themes/schemas.py b/superset/themes/schemas.py
index 4c7d7c5bf86..eaa40fdd8bf 100644
--- a/superset/themes/schemas.py
+++ b/superset/themes/schemas.py
@@ -54,7 +54,7 @@ class ImportV1ThemeSchema(Schema):
version = fields.String(required=True)
@validates("json_data")
- def validate_json_data(self, value: dict[str, Any]) -> None:
+ def validate_json_data(self, value: dict[str, Any], **kwargs: Any) -> None:
# Convert dict to JSON string for validation
if isinstance(value, dict):
json_str = json.dumps(value)
@@ -85,12 +85,12 @@ class ThemePostSchema(Schema):
json_data = fields.String(required=True, allow_none=False)
@validates("theme_name")
- def validate_theme_name(self, value: str) -> None:
+ def validate_theme_name(self, value: str, **kwargs: Any) -> None:
if not value or not value.strip():
raise ValidationError("Theme name cannot be empty.")
@validates("json_data")
- def validate_and_sanitize_json_data(self, value: str) -> None:
+ def validate_and_sanitize_json_data(self, value: str, **kwargs: Any) -> None:
# Parse JSON
try:
theme_config = json.loads(value) if isinstance(value, str) else value
@@ -112,12 +112,12 @@ class ThemePutSchema(Schema):
json_data = fields.String(required=True, allow_none=False)
@validates("theme_name")
- def validate_theme_name(self, value: str) -> None:
+ def validate_theme_name(self, value: str, **kwargs: Any) -> None:
if not value or not value.strip():
raise ValidationError("Theme name cannot be empty.")
@validates("json_data")
- def validate_and_sanitize_json_data(self, value: str) -> None:
+ def validate_and_sanitize_json_data(self, value: str, **kwargs: Any) -> None:
# Parse JSON
try:
theme_config = json.loads(value) if isinstance(value, str) else value
diff --git a/tests/unit_tests/dashboards/api_test.py b/tests/unit_tests/dashboards/api_test.py
index 5f4c5e680d9..deaa07c456d 100644
--- a/tests/unit_tests/dashboards/api_test.py
+++ b/tests/unit_tests/dashboards/api_test.py
@@ -25,6 +25,11 @@ from superset.dashboards.schemas import DashboardGetResponseSchema
@pytest.fixture
def mock_dashboard() -> MagicMock:
dash = MagicMock()
+ # Real Dashboard objects are not subscriptable. Without this, marshmallow's
+ # get_value reads ``dash["changed_on"]`` (which a MagicMock happily returns
+ # as another mock) instead of the attribute, and marshmallow 4's DateTime
+ # field then fails serializing the mock via datetime.isoformat().
+ dash.__getitem__.side_effect = TypeError
dash.id = 1
dash.slug = "test-slug"
dash.url = "/superset/dashboard/test-slug/"
diff --git a/tests/unit_tests/test_marshmallow4_compat.py b/tests/unit_tests/test_marshmallow4_compat.py
new file mode 100644
index 00000000000..e78611505ac
--- /dev/null
+++ b/tests/unit_tests/test_marshmallow4_compat.py
@@ -0,0 +1,164 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Regression tests for marshmallow 4.x compatibility.
+
+Marshmallow 4.x requires **kwargs on @validates-decorated methods and
+ships with marshmallow-sqlalchemy >= 1.5.0. These tests confirm that
+the codebase handles those requirements correctly.
+"""
+# pylint: disable=import-outside-toplevel
+
+from importlib.metadata import version
+from unittest.mock import MagicMock
+
+import marshmallow
+import pytest
+from marshmallow import ValidationError
+
+
+def test_validates_kwargs_database_schema() -> None:
+ """Regression: marshmallow 4.x requires **kwargs on @validates methods.
+
+ Without **kwargs on validate_file_extension, marshmallow 4.x passes extra
+ keyword arguments to the validator and raises TypeError instead of
+ ValidationError when validating an uploaded file with a bad extension.
+ """
+ from flask import Flask
+
+ from superset.databases.schemas import BaseUploadFilePostSchemaMixin
+
+ app = Flask(__name__)
+ app.config["ALLOWED_EXTENSIONS"] = {"csv", "json", "parquet", "zip"}
+
+ # Build a minimal schema that inherits the mixin so @validates fires
+ class TestUploadSchema(BaseUploadFilePostSchemaMixin):
+ file = marshmallow.fields.Raw()
+
+ schema = TestUploadSchema()
+
+ # Construct a mock FileStorage with a disallowed extension
+ mock_file = MagicMock()
+ mock_file.filename = "malware.exe"
+
+ with app.app_context():
+ with pytest.raises(ValidationError):
+ # Must raise ValidationError, not TypeError
+ schema.load({"file": mock_file})
+
+
+def test_patch_marshmallow_for_flask_appbuilder_idempotent() -> None:
+ """Regression: patch_marshmallow_for_flask_appbuilder must be idempotent.
+
+ Calling the patch function more than once must not create nested wrappers.
+ The _fab_patched sentinel on the patched method guards against re-patching.
+ """
+ import marshmallow as mm
+
+ from superset.marshmallow_compatibility import (
+ patch_marshmallow_for_flask_appbuilder,
+ )
+
+ # Capture the state of _init_fields before the first call
+ original_init_fields = mm.Schema._init_fields
+
+ patch_marshmallow_for_flask_appbuilder()
+ after_first = mm.Schema._init_fields
+ assert getattr(after_first, "_fab_patched", False), (
+ "Expected _init_fields to be marked _fab_patched after first call"
+ )
+
+ patch_marshmallow_for_flask_appbuilder()
+ after_second = mm.Schema._init_fields
+ assert after_second is after_first, (
+ "Expected _init_fields to be unchanged after second call (idempotent)"
+ )
+
+ # Restore original to avoid side-effects on other tests
+ mm.Schema._init_fields = original_init_fields
+
+
+@pytest.mark.skipif(
+ tuple(int(p) for p in version("marshmallow").split(".")[:1]) < (4,),
+ reason="FAB _init_fields KeyError only occurs on marshmallow >= 4",
+)
+def test_fab_field_stubbing_is_single_pass() -> None:
+ """Regression: FAB compat must stub missing fields in one pass per instance.
+
+ Flask-AppBuilder mints a fresh schema class on every call and marshmallow
+ rebuilds ``declared_fields`` for every instance, so any approach that re-runs
+ ``_init_fields`` on KeyError (a retry loop) re-pays its cost on every single
+ instantiation. Under coverage tracing that blew the unit-test job past its
+ timeout. The patch must instead stub the undeclared FAB names before the
+ original runs, so ``_init_fields`` executes exactly once per instance.
+ """
+ import marshmallow as mm
+
+ from superset.marshmallow_compatibility import (
+ patch_marshmallow_for_flask_appbuilder,
+ )
+
+ # Count invocations of the underlying (pre-patch) _init_fields, then patch.
+ base_init_fields = mm.Schema._init_fields
+ calls = {"n": 0}
+
+ def counting(self: mm.Schema) -> object:
+ calls["n"] += 1
+ return base_init_fields(self)
+
+ mm.Schema._init_fields = counting
+ try:
+ patch_marshmallow_for_flask_appbuilder()
+
+ # FAB-style schema: Meta.fields references undeclared relationship names,
+ # which marshmallow 4.x rejects with KeyError.
+ class FabStyleSchema(mm.Schema):
+ a = mm.fields.String()
+
+ class Meta:
+ fields = ("a", "related_model", "owner", "parent_id")
+
+ calls["n"] = 0
+ for _ in range(50):
+ FabStyleSchema()
+ # Exactly one underlying _init_fields per instance — no retry blowup.
+ assert calls["n"] == 50, (
+ f"expected 50 _init_fields calls for 50 instances, got {calls['n']} "
+ "— single-pass stubbing regressed into a retry loop"
+ )
+
+ # The undeclared FAB names resolve to Raw stubs in the built field set.
+ instance = FabStyleSchema()
+ assert isinstance(instance.fields["related_model"], mm.fields.Raw)
+ finally:
+ mm.Schema._init_fields = base_init_fields
+
+
+def test_marshmallow_sqlalchemy_version() -> None:
+ """Regression: marshmallow-sqlalchemy >= 1.5.0 is required.
+
+ marshmallow 4.x needs marshmallow-sqlalchemy >= 1.4.x, but 1.4.1/1.4.2 carry
+ a memory regression (issue #665) that exhausts memory in the test suite; only
+ 1.5.0 fixes it. This test ensures the installed version is recent enough.
+ """
+ import marshmallow_sqlalchemy # noqa: F401 # import to confirm it's installed
+
+ installed = version("marshmallow-sqlalchemy")
+ parts = [int(x) for x in installed.split(".")[:3]]
+ minimum = [1, 5, 0]
+ assert parts >= minimum, (
+ f"marshmallow-sqlalchemy {installed} is too old; need >= 1.5.0"
+ )