You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ru...@apache.org on 2026/08/05 03:40:15 UTC

(superset) branch master updated: fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)

This is an automated email from the ASF dual-hosted git repository.

rusackas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git


The following commit(s) were added to refs/heads/master by this push:
     new 956231b9eec fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)
956231b9eec is described below

commit 956231b9eecb10981721113a012873862435a496
Author: Abdul Rehman <76...@users.noreply.github.com>
AuthorDate: Wed Aug 5 08:40:09 2026 +0500

    fix(dataset): make post-save column refresh best-effort for Jinja SQL (#42463)
---
 superset/commands/dataset/refresh.py              |  22 ++-
 superset/connectors/sqla/utils.py                 |  34 +++-
 superset/exceptions.py                            |  10 ++
 tests/unit_tests/commands/dataset/refresh_test.py | 186 ++++++++++++++++++++++
 tests/unit_tests/connectors/sqla/test_utils.py    | 103 +++++++++++-
 5 files changed, 351 insertions(+), 4 deletions(-)

diff --git a/superset/commands/dataset/refresh.py b/superset/commands/dataset/refresh.py
index 6dd5c5b9341..01c089205f3 100644
--- a/superset/commands/dataset/refresh.py
+++ b/superset/commands/dataset/refresh.py
@@ -31,7 +31,10 @@ from superset.commands.dataset.exceptions import (
 from superset.connectors.sqla.models import SqlaTable
 from superset.daos.dataset import DatasetDAO
 from superset.datasets.datetime_format_detector import DatetimeFormatDetector
-from superset.exceptions import SupersetSecurityException
+from superset.exceptions import (
+    SupersetSecurityException,
+    SupersetVirtualTableParseException,
+)
 from superset.utils.decorators import on_error, transaction
 
 logger = logging.getLogger(__name__)
@@ -46,7 +49,22 @@ class RefreshDatasetCommand(BaseCommand):
     def run(self) -> Model:
         self.validate()
         assert self._model
-        self._model.fetch_metadata()
+        try:
+            self._model.fetch_metadata()
+        except SupersetVirtualTableParseException as ex:
+            # The virtual dataset's SQL could not be parsed or templated at
+            # save time — typically Jinja blocks (e.g. ``{% if from_dttm %}``)
+            # that have no runtime context here. The row has already been
+            # persisted by ``UpdateDatasetCommand`` before this refresh runs,
+            # so surfacing an "Invalid SQL" toast is misleading. Genuine
+            # driver, connection, and permission errors still raise the
+            # broader ``SupersetGenericDBErrorException`` and continue to
+            # bubble up. See #38012.
+            logger.warning(
+                "Dataset column refresh skipped for %s: %s",
+                self._model.table_name,
+                ex.message,
+            )
 
         # Detect datetime formats if feature is enabled
         if current_app.config.get("DATASET_AUTO_DETECT_DATETIME_FORMATS", True):
diff --git a/superset/connectors/sqla/utils.py b/superset/connectors/sqla/utils.py
index 106d024efb7..67dde802b4e 100644
--- a/superset/connectors/sqla/utils.py
+++ b/superset/connectors/sqla/utils.py
@@ -23,6 +23,7 @@ from typing import Callable, TYPE_CHECKING, TypeVar
 from uuid import UUID
 
 from flask_babel import lazy_gettext as _
+from jinja2 import UndefinedError
 from sqlalchemy.engine.url import URL as SqlaURL  # noqa: N811
 from sqlalchemy.exc import NoSuchTableError
 from sqlalchemy.orm import DeclarativeMeta
@@ -37,6 +38,7 @@ from superset.exceptions import (
     SupersetParseError,
     SupersetSecurityException,
     SupersetSyntaxErrorException,
+    SupersetVirtualTableParseException,
 )
 from superset.models.core import Database
 from superset.result_set import SupersetResultSet
@@ -96,6 +98,11 @@ def get_physical_table_metadata(
     return cols
 
 
+def _has_jinja_markers(sql: str) -> bool:
+    """Return True if ``sql`` contains Jinja template markers (``{%`` or ``{{``)."""
+    return "{%" in sql or "{{" in sql
+
+
 def get_virtual_table_metadata(dataset: SqlaTable) -> list[ResultSetColumnType]:
     """Use SQLparser to get virtual dataset metadata"""
     if not dataset.sql:
@@ -104,17 +111,42 @@ def get_virtual_table_metadata(dataset: SqlaTable) -> list[ResultSetColumnType]:
         )
 
     db_engine_spec = dataset.database.db_engine_spec
+    original_sql = dataset.sql
     try:
         sql = dataset.get_template_processor().process_template(
-            dataset.sql, **dataset.template_params_dict
+            original_sql, **dataset.template_params_dict
         )
     except SupersetSyntaxErrorException as ex:
+        # ``process_template`` aggregates several jinja2 exceptions
+        # (``TemplateSyntaxError``, ``SecurityError``, ``UndefinedError``,
+        # ``Unicode*Error``) into ``SupersetSyntaxErrorException``. Only
+        # the ``UndefinedError`` case is a "missing runtime context"
+        # signal that ``RefreshDatasetCommand`` can safely soften — the
+        # rest (sandbox violations, malformed template syntax, encoding
+        # errors) indicate a real problem with the template that must
+        # surface. See #38012.
+        if isinstance(ex.__cause__, UndefinedError):
+            raise SupersetVirtualTableParseException(
+                message=_("Template processing error: %(error)s", error=str(ex)),
+            ) from ex
         raise SupersetGenericDBErrorException(
             message=_("Template processing error: %(error)s", error=str(ex)),
         ) from ex
     try:
         parsed_script = SQLScript(sql, engine=db_engine_spec.engine)
     except SupersetParseError as ex:
+        # ``SQLScript`` fails on any invalid SQL, including static SQL
+        # with no template dependency. Only soften when the input
+        # contained Jinja markers — in that case an "Invalid SQL"
+        # outcome is very likely a rendering artifact (e.g. an empty
+        # ``filter_values('x')`` producing ``WHERE col IN ()``) rather
+        # than a genuine defect in the user's SQL, and the row is
+        # already persisted by ``UpdateDatasetCommand``. Genuinely
+        # invalid static SQL must still hard-error. See #38012.
+        if _has_jinja_markers(original_sql):
+            raise SupersetVirtualTableParseException(
+                message=_("Invalid SQL: %(error)s", error=ex.error.message),
+            ) from ex
         raise SupersetGenericDBErrorException(
             message=_("Invalid SQL: %(error)s", error=ex.error.message),
         ) from ex
diff --git a/superset/exceptions.py b/superset/exceptions.py
index be21c4a8aac..2fe5b9aad68 100644
--- a/superset/exceptions.py
+++ b/superset/exceptions.py
@@ -147,6 +147,16 @@ class SupersetGenericDBErrorException(SupersetErrorFromParamsException):
         )
 
 
+class SupersetVirtualTableParseException(SupersetGenericDBErrorException):
+    """Raised when a virtual dataset's SQL cannot be parsed/templated.
+
+    Distinct from generic DB errors so callers can soften template/parse
+    failures (which cannot be validated at save time without runtime
+    context) without also swallowing genuine driver, connection, or
+    permission errors from the same code path. See #38012.
+    """
+
+
 class SupersetTemplateParamsErrorException(SupersetErrorFromParamsException):
     status = 400
 
diff --git a/tests/unit_tests/commands/dataset/refresh_test.py b/tests/unit_tests/commands/dataset/refresh_test.py
new file mode 100644
index 00000000000..4f61d1eef74
--- /dev/null
+++ b/tests/unit_tests/commands/dataset/refresh_test.py
@@ -0,0 +1,186 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Regression test for #38012.
+
+``RefreshDatasetCommand`` runs after ``UpdateDatasetCommand`` when the
+Dataset Editor modal saves with ``override_columns=True``. Historically
+it re-parsed the dataset's SQL with an empty runtime template context;
+Jinja-templated virtual datasets like
+``SELECT * FROM t {% if from_dttm %}WHERE ds > '{{ from_dttm }}'{% endif %}``
+render to malformed SQL, sqlglot rejects it, and the whole PUT surfaces
+as an "Invalid SQL" toast — even though the dataset row was already
+committed successfully by ``UpdateDatasetCommand``.
+
+``get_virtual_table_metadata`` now raises a dedicated
+``SupersetVirtualTableParseException`` for the specific template/parse
+failure paths, and ``RefreshDatasetCommand`` catches only that
+subclass. Genuine driver, connection, and permission errors still raise
+the broader ``SupersetGenericDBErrorException`` (from
+``get_columns_description``'s catch-all) and bubble up unchanged.
+
+This module covers:
+
+1. Template/parse failures become a warning (no re-raise).
+2. Generic DB errors (connection, permission, driver) still propagate.
+3. Security failures still propagate.
+4. Pre-existing validation paths (not-found, forbidden) still work.
+"""
+
+from __future__ import annotations
+
+import logging
+
+import pytest
+from pytest_mock import MockerFixture
+
+from superset.commands.dataset.exceptions import (
+    DatasetForbiddenError,
+    DatasetNotFoundError,
+)
+from superset.commands.dataset.refresh import RefreshDatasetCommand
+from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
+from superset.exceptions import (
+    SupersetGenericDBErrorException,
+    SupersetSecurityException,
+    SupersetVirtualTableParseException,
+)
+
+
+def test_refresh_swallows_virtual_table_parse_exception(
+    mocker: MockerFixture, caplog: pytest.LogCaptureFixture
+) -> None:
+    """Template/parse failures during metadata refresh must NOT bubble
+    up as a hard error — the dataset has already been persisted by the
+    caller (``UpdateDatasetCommand``), so the refresh is best-effort for
+    SQL that cannot be validated at save time. See #38012.
+    """
+    mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+    mock_model = mocker.MagicMock()
+    mock_model.table_name = "jinja_dataset"
+    mock_model.fetch_metadata.side_effect = SupersetVirtualTableParseException(
+        message="Invalid SQL: unexpected token"
+    )
+    mock_dataset_dao.find_by_id.return_value = mock_model
+    mocker.patch(
+        "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+    )
+    # Skip datetime-format detection to keep the test focused.
+    mocker.patch(
+        "superset.commands.dataset.refresh.current_app.config.get",
+        return_value=False,
+    )
+
+    with caplog.at_level(logging.WARNING, logger="superset.commands.dataset.refresh"):
+        result = RefreshDatasetCommand(model_id=1).run()
+
+    assert result is mock_model, "command should return the model, not re-raise"
+    assert any(
+        "Dataset column refresh skipped for jinja_dataset" in rec.message
+        for rec in caplog.records
+    ), "expected a warning naming the dataset"
+
+
+def test_refresh_still_raises_generic_db_error(mocker: MockerFixture) -> None:
+    """A plain ``SupersetGenericDBErrorException`` from ``fetch_metadata``
+    (raised by ``get_columns_description`` for connection / permission /
+    driver failures) must still bubble up. The subclass-based catch in
+    the refresh command must NOT silence these — even when the same
+    dataset also contains Jinja markers. See #38012."""
+    mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+    mock_model = mocker.MagicMock()
+    mock_model.table_name = "jinja_dataset_with_bad_connection"
+    mock_model.sql = (
+        "SELECT * FROM foo {% if from_dttm %}WHERE ds > '{{ from_dttm }}'{% endif %}"
+    )
+    mock_model.fetch_metadata.side_effect = SupersetGenericDBErrorException(
+        message="Connection refused"
+    )
+    mock_dataset_dao.find_by_id.return_value = mock_model
+    mocker.patch(
+        "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+    )
+    mocker.patch(
+        "superset.commands.dataset.refresh.current_app.config.get",
+        return_value=False,
+    )
+
+    with pytest.raises(SupersetGenericDBErrorException):
+        RefreshDatasetCommand(model_id=1).run()
+
+
+def test_refresh_still_raises_on_security_exception(
+    mocker: MockerFixture,
+) -> None:
+    """``SupersetSecurityException`` must still fail hard — softening the
+    refresh path must not become a bypass for security checks. See #38012.
+    """
+    mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+    mock_model = mocker.MagicMock()
+    mock_model.table_name = "restricted_dataset"
+    mock_model.fetch_metadata.side_effect = SupersetSecurityException(
+        SupersetError(
+            error_type=SupersetErrorType.DATASOURCE_SECURITY_ACCESS_ERROR,
+            message="Access denied",
+            level=ErrorLevel.ERROR,
+        )
+    )
+    mock_dataset_dao.find_by_id.return_value = mock_model
+    mocker.patch(
+        "superset.commands.dataset.refresh.security_manager.raise_for_editorship"
+    )
+    mocker.patch(
+        "superset.commands.dataset.refresh.current_app.config.get",
+        return_value=False,
+    )
+
+    # ``on_error`` only wraps ``SQLAlchemyError`` into
+    # ``DatasetRefreshFailedError``; other exception classes propagate
+    # unchanged. We just need to prove the softening does NOT swallow
+    # this one.
+    with pytest.raises(SupersetSecurityException):
+        RefreshDatasetCommand(model_id=1).run()
+
+
+def test_refresh_dataset_not_found(mocker: MockerFixture) -> None:
+    """Sanity check that the pre-existing not-found path still works.
+    Guards against the softening change accidentally masking validation
+    errors.
+    """
+    mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+    mock_dataset_dao.find_by_id.return_value = None
+
+    with pytest.raises(DatasetNotFoundError):
+        RefreshDatasetCommand(model_id=999).run()
+
+
+def test_refresh_forbidden(mocker: MockerFixture) -> None:
+    """Sanity check that the pre-existing forbidden path still works."""
+    mock_dataset_dao = mocker.patch("superset.commands.dataset.refresh.DatasetDAO")
+    mock_dataset_dao.find_by_id.return_value = mocker.MagicMock()
+    mocker.patch(
+        "superset.commands.dataset.refresh.security_manager.raise_for_editorship",
+        side_effect=SupersetSecurityException(
+            SupersetError(
+                error_type=SupersetErrorType.DATASOURCE_SECURITY_ACCESS_ERROR,
+                message="Access denied",
+                level=ErrorLevel.ERROR,
+            )
+        ),
+    )
+
+    with pytest.raises(DatasetForbiddenError):
+        RefreshDatasetCommand(model_id=1).run()
diff --git a/tests/unit_tests/connectors/sqla/test_utils.py b/tests/unit_tests/connectors/sqla/test_utils.py
index be3e3662e19..c10fe2b3d02 100644
--- a/tests/unit_tests/connectors/sqla/test_utils.py
+++ b/tests/unit_tests/connectors/sqla/test_utils.py
@@ -17,13 +17,17 @@
 from unittest.mock import Mock, patch
 
 import pytest
+from jinja2 import UndefinedError
+from jinja2.exceptions import SecurityError
 
 from superset.connectors.sqla.models import SqlaTable
 from superset.connectors.sqla.utils import get_virtual_table_metadata
-from superset.errors import SupersetErrorType
+from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
 from superset.exceptions import (
     SupersetGenericDBErrorException,
     SupersetSecurityException,
+    SupersetSyntaxErrorException,
+    SupersetVirtualTableParseException,
 )
 from superset.models.core import Database
 
@@ -90,6 +94,103 @@ def test_get_virtual_table_metadata_mutation_not_allowed():
         assert "Only `SELECT` statements are allowed" in exc_info.value.error.message
 
 
+def test_get_virtual_table_metadata_jinja_parse_error_is_softened():
+    """A parse error on SQL that contained Jinja markers is likely a
+    rendering artifact (e.g. an empty ``filter_values('x')`` producing
+    ``WHERE col IN ()`` or similar) — must raise the soften-able
+    ``SupersetVirtualTableParseException`` so ``RefreshDatasetCommand``
+    can treat it as best-effort. See #38012.
+    """
+    mock_dataset = Mock(spec=SqlaTable)
+    mock_database = Mock(spec=Database)
+    mock_dataset.database = mock_database
+    # Input has Jinja markers, so ``_has_jinja_markers(original_sql)`` is True.
+    mock_dataset.sql = "SELECT INVALID SYNTAX FROM {{ some_var }}"
+    mock_database.db_engine_spec.engine = "postgresql"
+
+    # Template renders (with empty runtime context) to genuinely
+    # unparseable SQL that ``SQLScript`` rejects with
+    # ``SupersetParseError``.
+    mock_template_processor = Mock()
+    mock_template_processor.process_template.return_value = "SELECT INVALID SYNTAX FROM"
+    mock_dataset.get_template_processor.return_value = mock_template_processor
+    mock_dataset.template_params_dict = {}
+
+    with pytest.raises(SupersetVirtualTableParseException) as exc_info:
+        get_virtual_table_metadata(mock_dataset)
+
+    assert "Invalid SQL:" in str(exc_info.value.message)
+
+
+def test_get_virtual_table_metadata_template_undefined_is_softened():
+    """A template render failure caused by ``UndefinedError`` (missing
+    runtime variable) is a "no runtime context" signal and must raise
+    the soften-able ``SupersetVirtualTableParseException``. See #38012.
+    """
+    mock_dataset = Mock(spec=SqlaTable)
+    mock_database = Mock(spec=Database)
+    mock_dataset.database = mock_database
+    mock_dataset.sql = "SELECT '{{ from_dttm.isoformat() }}'"
+
+    # Template processor wraps jinja's ``UndefinedError`` in
+    # ``SupersetSyntaxErrorException`` (see ``jinja_context.py:807-848``).
+    ex = SupersetSyntaxErrorException(
+        [
+            SupersetError(
+                message="Jinja2 template error",
+                error_type=SupersetErrorType.GENERIC_COMMAND_ERROR,
+                level=ErrorLevel.ERROR,
+            )
+        ]
+    )
+    ex.__cause__ = UndefinedError("'from_dttm' is undefined")
+    mock_template_processor = Mock()
+    mock_template_processor.process_template.side_effect = ex
+    mock_dataset.get_template_processor.return_value = mock_template_processor
+    mock_dataset.template_params_dict = {}
+
+    with pytest.raises(SupersetVirtualTableParseException) as exc_info:
+        get_virtual_table_metadata(mock_dataset)
+
+    assert "Template processing error" in str(exc_info.value.message)
+
+
+def test_get_virtual_table_metadata_template_security_error_is_not_softened():
+    """A Jinja ``SecurityError`` (sandbox violation) must NOT be
+    softened — it indicates a real security-policy failure that the
+    operator needs to see, not a "no runtime context" artifact. See
+    codeant review on #42463.
+    """
+    mock_dataset = Mock(spec=SqlaTable)
+    mock_database = Mock(spec=Database)
+    mock_dataset.database = mock_database
+    mock_dataset.sql = "SELECT {{ some_disallowed_attr }}"
+
+    # Template processor wraps jinja's ``SecurityError`` in
+    # ``SupersetSyntaxErrorException`` (see ``jinja_context.py:807-848``).
+    ex = SupersetSyntaxErrorException(
+        [
+            SupersetError(
+                message="Jinja2 template error",
+                error_type=SupersetErrorType.GENERIC_COMMAND_ERROR,
+                level=ErrorLevel.ERROR,
+            )
+        ]
+    )
+    ex.__cause__ = SecurityError("access to attribute 'x' of ... is unsafe")
+    mock_template_processor = Mock()
+    mock_template_processor.process_template.side_effect = ex
+    mock_dataset.get_template_processor.return_value = mock_template_processor
+    mock_dataset.template_params_dict = {}
+
+    with pytest.raises(SupersetGenericDBErrorException) as exc_info:
+        get_virtual_table_metadata(mock_dataset)
+
+    # Must be the base class, not the soften-able subclass.
+    assert not isinstance(exc_info.value, SupersetVirtualTableParseException)
+    assert "Template processing error" in str(exc_info.value.message)
+
+
 def test_get_virtual_table_metadata_multiple_statements_not_allowed():
     """Test that multiple SQL statements raise security error."""
     mock_dataset = Mock(spec=SqlaTable)