You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/08/05 03:40:15 UTC
(superset) branch master updated: fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)
This is an automated email from the ASF dual-hosted git repository.
rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 956231b9eec fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)
956231b9eec is described below
commit 956231b9eecb10981721113a012873862435a496
Author: Abdul Rehman <76...@users.noreply.github.com>
AuthorDate: Wed Aug 5 08:40:09 2026 +0500
fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)
---
superset/commands/dataset/refresh.py | 22 ++-
superset/connectors/sqla/utils.py | 34 +++-
superset/exceptions.py | 10 ++
tests/unit_tests/commands/dataset/refresh_test.py | 186 ++++++++++++++++++++++
tests/unit_tests/connectors/sqla/test_utils.py | 103 +++++++++++-
5 files changed, 351 insertions(+), 4 deletions(-)
diff --git a/superset/commands/dataset/refresh.py b/superset/commands/dataset/refresh.py
index 6dd5c5b9341..01c089205f3 100644
--- a/superset/commands/dataset/refresh.py
+++ b/superset/commands/dataset/refresh.py
@@ -31,7 +31,10 @@ from superset.commands.dataset.exceptions import (
from superset.connectors.sqla.models import SqlaTable
from superset.daos.dataset import DatasetDAO
from superset.datasets.datetime_format_detector import DatetimeFormatDetector
-from superset.exceptions import SupersetSecurityException
+from superset.exceptions import (
+ SupersetSecurityException,
+ SupersetVirtualTableParseException,
+)
from superset.utils.decorators import on_error, transaction
logger = logging.getLogger(__name__)
@@ -46,7 +49,22 @@ class RefreshDatasetCommand(BaseCommand):
def run(self) -> Model:
self.validate()
assert self._model
- self._model.fetch_metadata()
+ try:
+ self._model.fetch_metadata()
+ except SupersetVirtualTableParseException as ex:
+ # The virtual dataset's SQL could not be parsed or templated at
+ # save time — typically Jinja blocks (e.g. ``{% if from_dttm %}``)
+ # that have no runtime context here. The row has already been
+ # persisted by ``UpdateDatasetCommand`` before this refresh runs,
+ # so surfacing an "Invalid SQL" toast is misleading. Genuine
+ # driver, connection, and permission errors still raise the
+ # broader ``SupersetGenericDBErrorException`` and continue to
+ # bubble up. See #38012.
+ logger.warning(
+ "Dataset column refresh skipped for %s: %s",
+ self._model.table_name,
+ ex.message,
+ )
# Detect datetime formats if feature is enabled
if current_app.config.get("DATASET_AUTO_DETECT_DATETIME_FORMATS", True):
diff --git a/superset/connectors/sqla/utils.py b/superset/connectors/sqla/utils.py
index 106d024efb7..67dde802b4e 100644
--- a/superset/connectors/sqla/utils.py
+++ b/superset/connectors/sqla/utils.py
@@ -23,6 +23,7 @@ from typing import Callable, TYPE_CHECKING, TypeVar
from uuid import UUID
from flask_babel import lazy_gettext as _
+from jinja2 import UndefinedError
from sqlalchemy.engine.url import URL as SqlaURL # noqa: N811
from sqlalchemy.exc import NoSuchTableError
from sqlalchemy.orm import DeclarativeMeta
@@ -37,6 +38,7 @@ from superset.exceptions import (
SupersetParseError,
SupersetSecurityException,
SupersetSyntaxErrorException,
+ SupersetVirtualTableParseException,
)
from superset.models.core import Database
from superset.result_set import SupersetResultSet
@@ -96,6 +98,11 @@ def get_physical_table_metadata(
return cols
+def _has_jinja_markers(sql: str) -> bool:
+ """Return True if ``sql`` contains Jinja template markers (``{%`` or ``{{``)."""
+ return "{%" in sql or "{{" in sql
+
+
def get_virtual_table_metadata(dataset: SqlaTable) -> list[ResultSetColumnType]:
"""Use SQLparser to get virtual dataset metadata"""
if not dataset.sql:
@@ -104,17 +111,42 @@ def get_virtual_table_metadata(dataset: SqlaTable) -> list[ResultSetColumnType]:
)
db_engine_spec = dataset.database.db_engine_spec
+ original_sql = dataset.sql
try:
sql = dataset.get_template_processor().process_template(
- dataset.sql, **dataset.template_params_dict
+ original_sql, **dataset.template_params_dict
)
except SupersetSyntaxErrorException as ex:
+ # ``process_template`` aggregates several jinja2 exceptions
+ # (``TemplateSyntaxError``, ``SecurityError``, ``UndefinedError``,
+ # ``Unicode*Error``) into ``SupersetSyntaxErrorException``. Only
+ # the ``UndefinedError`` case is a "missing runtime context"
+ # signal that ``RefreshDatasetCommand`` can safely soften — the
+ # rest (sandbox violations, malformed template syntax, encoding
+ # errors) indicate a real problem with the template that must
+ # surface. See #38012.
+ if isinstance(ex.__cause__, UndefinedError):
+ raise SupersetVirtualTableParseException(
+ message=_("Template processing error: %(error)s", error=str(ex)),
+ ) from ex
raise SupersetGenericDBErrorException(
message=_("Template processing error: %(error)s", error=str(ex)),
) from ex
try:
parsed_script = SQLScript(sql, engine=db_engine_spec.engine)
except SupersetParseError as ex:
+ # ``SQLScript`` fails on any invalid SQL, including static SQL
+ # with no template dependency. Only soften when the input
+ # contained Jinja markers — in that case an "Invalid SQL"
+ # outcome is very likely a rendering artifact (e.g. an empty
+ # ``filter_values('x')`` producing ``WHERE col IN ()``) rather
+ # than a genuine defect in the user's SQL, and the row is
+ # already persisted by ``UpdateDatasetCommand``. Genuinely
+ # invalid static SQL must still hard-error. See #38012.
+ if _has_jinja_markers(original_sql):
+ raise SupersetVirtualTableParseException(
+ message=_("Invalid SQL: %(error)s", error=ex.error.message),
+ ) from ex
raise SupersetGenericDBErrorException(
message=_("Invalid SQL: %(error)s", error=ex.error.message),
) from ex
diff --git a/superset/exceptions.py b/superset/exceptions.py
index be21c4a8aac..2fe5b9aad68 100644
--- a/superset/exceptions.py
+++ b/superset/exceptions.py
@@ -147,6 +147,16 @@ class SupersetGenericDBErrorException(SupersetErrorFromParamsException):
)
+class SupersetVirtualTableParseException(SupersetGenericDBErrorException):
+ """Raised when a virtual dataset's SQL cannot be parsed/templated.
+
+ Distinct from generic DB errors so callers can soften template/parse
+ failures (which cannot be validated at save time without runtime
+ context) without also swallowing genuine driver, connection, or
+ permission errors from the same code path. See #38012.
+ """
+
+
class SupersetTemplateParamsErrorException(SupersetErrorFromParamsException):
status = 400
diff --git a/tests/unit_tests/commands/dataset/refresh_test.py b/tests/unit_tests/commands/dataset/refresh_test.py
new file mode 100644
index 00000000000..4f61d1eef74
--- /dev/null
+++ b/tests/unit_tests/commands/dataset/refresh_test.py
@@ -0,0 +1,186 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Regression test for #38012.
+
+``RefreshDatasetCommand`` runs after ``UpdateDatasetCommand`` when the
+Dataset Editor modal saves with ``override_columns=True``. Historically
+it re-parsed the dataset's SQL with an empty runtime template context;
+Jinja-templated virtual datasets like
+``SELECT * FROM t {% if from_dttm %}WHERE ds > '{{ from_dttm }}'{% endif %}``
+render to malformed SQL, sqlglot rejects it, and the whole PUT surfaces
+as an "Invalid SQL" toast — even though the dataset row was already
+committed successfully by ``UpdateDatasetCommand``.
+
+``get_virtual_table_metadata`` now raises a dedicated
+``SupersetVirtualTableParseException`` for the specific template/parse
+failure paths, and ``RefreshDatasetCommand`` catches only that
+subclass. Genuine driver, connection, and permission errors still raise
+the broader ``SupersetGenericDBErrorException`` (from
+``get_columns_description``'s catch-all) and bubble up unchanged.
+
+This module covers:
+
+1. Template/parse failures become a warning (no re-raise).
+2. Generic DB errors (connection, permission, driver) still propagate.
+3. Security failures still propagate.
+4. Pre-existing validation paths (not-found, forbidden) still work.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+from pytest_mock import MockerFixture
+
+from superset.commands.dataset.exceptions import (
+ DatasetForbiddenError,
+ DatasetNotFoundError,
+)
+from superset.commands.dataset.refresh import RefreshDatasetCommand
+from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
+from superset.exceptions import (
+ SupersetGenericDBErrorException,
+ SupersetSecurityException,
+ SupersetVirtualTableParseException,
+)
+
+
+def test_refresh_swallows_virtual_table_parse_exception(
+ mocker: MockerFixture, caplog: pytest.LogCaptureFixture
+) -> None:
+ """Template/parse failures during metadata refresh must NOT bubble
+ up as a hard error — the dataset has already been persisted by the
+ caller (``UpdateDatasetCommand``), so the refresh is best-effort for
+ SQL that cannot be validated at save time. See #38012.
+ """
+ mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+ mock_model = mocker.MagicMock()
+ mock_model.table_name = "jinja_dataset"
+ mock_model.fetch_metadata.side_effect = SupersetVirtualTableParseException(
+ message="Invalid SQL: unexpected token"
+ )
+ mock_dataset_dao.find_by_id.return_value = mock_model
+ mocker.patch(
+ "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+ )
+ # Skip datetime-format detection to keep the test focused.
+ mocker.patch(
+ "superset.commands.dataset.refresh.current_app.config.get",
+ return_value=False,
+ )
+
+ with caplog.at_level(logging.WARNING, logger="superset.commands.dataset.refresh"):
+ result = RefreshDatasetCommand(model_id=1).run()
+
+ assert result is mock_model, "command should return the model, not re-raise"
+ assert any(
+ "Dataset column refresh skipped for jinja_dataset" in rec.message
+ for rec in caplog.records
+ ), "expected a warning naming the dataset"
+
+
+def test_refresh_still_raises_generic_db_error(mocker: MockerFixture) -> None:
+ """A plain ``SupersetGenericDBErrorException`` from ``fetch_metadata``
+ (raised by ``get_columns_description`` for connection / permission /
+ driver failures) must still bubble up. The subclass-based catch in
+ the refresh command must NOT silence these — even when the same
+ dataset also contains Jinja markers. See #38012."""
+ mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+ mock_model = mocker.MagicMock()
+ mock_model.table_name = "jinja_dataset_with_bad_connection"
+ mock_model.sql = (
+ "SELECT * FROM foo {% if from_dttm %}WHERE ds > '{{ from_dttm }}'{% endif %}"
+ )
+ mock_model.fetch_metadata.side_effect = SupersetGenericDBErrorException(
+ message="Connection refused"
+ )
+ mock_dataset_dao.find_by_id.return_value = mock_model
+ mocker.patch(
+ "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+ )
+ mocker.patch(
+ "superset.commands.dataset.refresh.current_app.config.get",
+ return_value=False,
+ )
+
+ with pytest.raises(SupersetGenericDBErrorException):
+ RefreshDatasetCommand(model_id=1).run()
+
+
+def test_refresh_still_raises_on_security_exception(
+ mocker: MockerFixture,
+) -> None:
+ """``SupersetSecurityException`` must still fail hard — softening the
+ refresh path must not become a bypass for security checks. See #38012.
+ """
+ mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+ mock_model = mocker.MagicMock()
+ mock_model.table_name = "restricted_dataset"
+ mock_model.fetch_metadata.side_effect = SupersetSecurityException(
+ SupersetError(
+ error_type=SupersetErrorType.DATASOURCE_SECURITY_ACCESS_ERROR,
+ message="Access denied",
+ level=ErrorLevel.ERROR,
+ )
+ )
+ mock_dataset_dao.find_by_id.return_value = mock_model
+ mocker.patch(
+ "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+ )
+ mocker.patch(
+ "superset.commands.dataset.refresh.current_app.config.get",
+ return_value=False,
+ )
+
+ # ``on_error`` only wraps ``SQLAlchemyError`` into
+ # ``DatasetRefreshFailedError``; other exception classes propagate
+ # unchanged. We just need to prove the softening does NOT swallow
+ # this one.
+ with pytest.raises(SupersetSecurityException):
+ RefreshDatasetCommand(model_id=1).run()
+
+
+def test_refresh_dataset_not_found(mocker: MockerFixture) -> None:
+ """Sanity check that the pre-existing not-found path still works.
+ Guards against the softening change accidentally masking validation
+ errors.
+ """
+ mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+ mock_dataset_dao.find_by_id.return_value = None
+
+ with pytest.raises(DatasetNotFoundError):
+ RefreshDatasetCommand(model_id=999).run()
+
+
+def test_refresh_forbidden(mocker: MockerFixture) -> None:
+ """Sanity check that the pre-existing forbidden path still works."""
+ mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+ mock_dataset_dao.find_by_id.return_value = mocker.MagicMock()
+ mocker.patch(
+ "superset.commands.dataset.refresh.security_manager.raise_for_editorship",
+ side_effect=SupersetSecurityException(
+ SupersetError(
+ error_type=SupersetErrorType.DATASOURCE_SECURITY_ACCESS_ERROR,
+ message="Access denied",
+ level=ErrorLevel.ERROR,
+ )
+ ),
+ )
+
+ with pytest.raises(DatasetForbiddenError):
+ RefreshDatasetCommand(model_id=1).run()
diff --git a/tests/unit_tests/connectors/sqla/test_utils.py b/tests/unit_tests/connectors/sqla/test_utils.py
index be3e3662e19..c10fe2b3d02 100644
--- a/tests/unit_tests/connectors/sqla/test_utils.py
+++ b/tests/unit_tests/connectors/sqla/test_utils.py
@@ -17,13 +17,17 @@
from unittest.mock import Mock, patch
import pytest
+from jinja2 import UndefinedError
+from jinja2.exceptions import SecurityError
from superset.connectors.sqla.models import SqlaTable
from superset.connectors.sqla.utils import get_virtual_table_metadata
-from superset.errors import SupersetErrorType
+from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
from superset.exceptions import (
SupersetGenericDBErrorException,
SupersetSecurityException,
+ SupersetSyntaxErrorException,
+ SupersetVirtualTableParseException,
)
from superset.models.core import Database
@@ -90,6 +94,103 @@ def test_get_virtual_table_metadata_mutation_not_allowed():
assert "Only `SELECT` statements are allowed" in exc_info.value.error.message
+def test_get_virtual_table_metadata_jinja_parse_error_is_softened():
+ """A parse error on SQL that contained Jinja markers is likely a
+ rendering artifact (e.g. an empty ``filter_values('x')`` producing
+ ``WHERE col IN ()`` or similar) — must raise the soften-able
+ ``SupersetVirtualTableParseException`` so ``RefreshDatasetCommand``
+ can treat it as best-effort. See #38012.
+ """
+ mock_dataset = Mock(spec=SqlaTable)
+ mock_database = Mock(spec=Database)
+ mock_dataset.database = mock_database
+ # Input has Jinja markers, so ``_has_jinja_markers(original_sql)`` is True.
+ mock_dataset.sql = "SELECT INVALID SYNTAX FROM {{ some_var }}"
+ mock_database.db_engine_spec.engine = "postgresql"
+
+ # Template renders (with empty runtime context) to genuinely
+ # unparseable SQL that ``SQLScript`` rejects with
+ # ``SupersetParseError``.
+ mock_template_processor = Mock()
+ mock_template_processor.process_template.return_value = "SELECT INVALID SYNTAX FROM"
+ mock_dataset.get_template_processor.return_value = mock_template_processor
+ mock_dataset.template_params_dict = {}
+
+ with pytest.raises(SupersetVirtualTableParseException) as exc_info:
+ get_virtual_table_metadata(mock_dataset)
+
+ assert "Invalid SQL:" in str(exc_info.value.message)
+
+
+def test_get_virtual_table_metadata_template_undefined_is_softened():
+ """A template render failure caused by ``UndefinedError`` (missing
+ runtime variable) is a "no runtime context" signal and must raise
+ the soften-able ``SupersetVirtualTableParseException``. See #38012.
+ """
+ mock_dataset = Mock(spec=SqlaTable)
+ mock_database = Mock(spec=Database)
+ mock_dataset.database = mock_database
+ mock_dataset.sql = "SELECT '{{ from_dttm.isoformat() }}'"
+
+ # Template processor wraps jinja's ``UndefinedError`` in
+ # ``SupersetSyntaxErrorException`` (see ``jinja_context.py:807-848``).
+ ex = SupersetSyntaxErrorException(
+ [
+ SupersetError(
+ message="Jinja2 template error",
+ error_type=SupersetErrorType.GENERIC_COMMAND_ERROR,
+ level=ErrorLevel.ERROR,
+ )
+ ]
+ )
+ ex.__cause__ = UndefinedError("'from_dttm' is undefined")
+ mock_template_processor = Mock()
+ mock_template_processor.process_template.side_effect = ex
+ mock_dataset.get_template_processor.return_value = mock_template_processor
+ mock_dataset.template_params_dict = {}
+
+ with pytest.raises(SupersetVirtualTableParseException) as exc_info:
+ get_virtual_table_metadata(mock_dataset)
+
+ assert "Template processing error" in str(exc_info.value.message)
+
+
+def test_get_virtual_table_metadata_template_security_error_is_not_softened():
+ """A Jinja ``SecurityError`` (sandbox violation) must NOT be
+ softened — it indicates a real security-policy failure that the
+ operator needs to see, not a "no runtime context" artifact. See
+ codeant review on #42463.
+ """
+ mock_dataset = Mock(spec=SqlaTable)
+ mock_database = Mock(spec=Database)
+ mock_dataset.database = mock_database
+ mock_dataset.sql = "SELECT {{ some_disallowed_attr }}"
+
+ # Template processor wraps jinja's ``SecurityError`` in
+ # ``SupersetSyntaxErrorException`` (see ``jinja_context.py:807-848``).
+ ex = SupersetSyntaxErrorException(
+ [
+ SupersetError(
+ message="Jinja2 template error",
+ error_type=SupersetErrorType.GENERIC_COMMAND_ERROR,
+ level=ErrorLevel.ERROR,
+ )
+ ]
+ )
+ ex.__cause__ = SecurityError("access to attribute 'x' of ... is unsafe")
+ mock_template_processor = Mock()
+ mock_template_processor.process_template.side_effect = ex
+ mock_dataset.get_template_processor.return_value = mock_template_processor
+ mock_dataset.template_params_dict = {}
+
+ with pytest.raises(SupersetGenericDBErrorException) as exc_info:
+ get_virtual_table_metadata(mock_dataset)
+
+ # Must be the base class, not the soften-able subclass.
+ assert not isinstance(exc_info.value, SupersetVirtualTableParseException)
+ assert "Template processing error" in str(exc_info.value.message)
+
+
def test_get_virtual_table_metadata_multiple_statements_not_allowed():
"""Test that multiple SQL statements raise security error."""
mock_dataset = Mock(spec=SqlaTable)