You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2026/06/05 00:59:31 UTC
(superset) 03/03: parity
This is an automated email from the ASF dual-hosted git repository.
betodealmeida pushed a commit to branch builtin-sl
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 439bcf8d7929ff408b5a2dcfe15e97813339987f
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Thu Jun 4 20:59:15 2026 -0400
parity
---
superset/semantic_layers/adhoc.py | 196 +++++++++++++++
.../src/preset_io/dataset_semantic_layer/view.py | 209 ++++++++++++++--
.../extension/backend/tests/test_semantic_view.py | 274 +++++++++++++++++++++
superset/semantic_layers/mapper.py | 195 ++++++++++++---
superset/semantic_layers/rls.py | 120 +++++++++
tests/unit_tests/semantic_layers/adhoc_test.py | 268 ++++++++++++++++++++
tests/unit_tests/semantic_layers/mapper_test.py | 209 +++++++++++++++-
tests/unit_tests/semantic_layers/rls_test.py | 166 +++++++++++++
8 files changed, 1569 insertions(+), 68 deletions(-)
diff --git a/superset/semantic_layers/adhoc.py b/superset/semantic_layers/adhoc.py
new file mode 100644
index 00000000000..bd8b5fad9f7
--- /dev/null
+++ b/superset/semantic_layers/adhoc.py
@@ -0,0 +1,196 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Convert frontend "adhoc" metric / column dictionaries into semantic-layer
+``Metric`` / ``Dimension`` objects.
+
+Adhoc metrics and columns are user-defined SQL expressions that don't
+correspond to a saved metric or physical column on the dataset. The legacy
+``get_sqla_query`` path supports them through ``adhoc_metric_to_sqla`` /
+``adhoc_column_to_sqla``; the semantic-layer path needs an equivalent so
+that flipping ``USE_DATASET_SEMANTIC_VIEW`` doesn't regress charts that
+use them.
+
+The strategy is light: synthesize a ``Metric(definition=<sql>)`` /
+``Dimension(definition=<sql>)`` and let the view's existing
+``definition``-parsing path handle the rest. Jinja templating + safety
+checks are delegated to the dataset's own
+``_process_select_expression`` (already battle-tested).
+"""
+
+from __future__ import annotations
+
+from typing import Any, TYPE_CHECKING
+
+import pyarrow as pa
+from superset_core.semantic_layers.types import (
+ AggregationType,
+ Dimension,
+ Metric,
+)
+
+from superset.exceptions import QueryObjectValidationError
+from superset.utils.core import AdhocMetricExpressionType
+
+if TYPE_CHECKING:
+ from superset.connectors.sqla.models import SqlaTable
+ from superset.jinja_context import BaseTemplateProcessor
+
+
+# Map Superset's aggregate strings to the semantic-layer ``AggregationType``.
+# Anything unmapped (median, percentile, …) falls back to OTHER, which the
+# view treats as a non-rollup-safe metric.
+_AGGREGATE_MAP: dict[str, AggregationType] = {
+ "COUNT": AggregationType.COUNT,
+ "SUM": AggregationType.SUM,
+ "AVG": AggregationType.AVG,
+ "MIN": AggregationType.MIN,
+ "MAX": AggregationType.MAX,
+ "COUNT_DISTINCT": AggregationType.COUNT_DISTINCT,
+}
+
+
+def _process_adhoc_sql(
+ dataset: "SqlaTable",
+ sql_expression: str,
+ template_processor: "BaseTemplateProcessor | None",
+) -> str:
+ """
+ Run Jinja + safety validation on a free-form SQL expression.
+
+ Delegates to the dataset's ``_process_select_expression`` so adhoc
+ columns and metrics go through the same Jinja rendering, subquery
+ rules, and clause sanitization as the legacy path.
+ """
+ processed = dataset._process_select_expression( # noqa: SLF001
+ expression=sql_expression,
+ database_id=dataset.database_id,
+ engine=dataset.database.db_engine_spec.engine,
+ schema=dataset.schema or "",
+ template_processor=template_processor,
+ )
+ if not processed:
+ raise QueryObjectValidationError(
+ "Adhoc SQL expression resolved to an empty string."
+ )
+ return processed
+
+
+def _simple_metric_definition(adhoc: dict[str, Any], dataset: "SqlaTable") -> str:
+ """
+ Build the SQL for a SIMPLE adhoc metric: ``AGGREGATE(column)``.
+
+ Uses the database's identifier quoter so the column name is rendered
+ in the right dialect.
+ """
+ aggregate = (adhoc.get("aggregate") or "").upper()
+ column = adhoc.get("column") or {}
+ column_name = column.get("column_name")
+ if not aggregate or not column_name:
+ raise QueryObjectValidationError(
+ "SIMPLE adhoc metric requires both an aggregate and a column."
+ )
+ if aggregate == "COUNT_DISTINCT":
+ return f"COUNT(DISTINCT {dataset.quote_identifier(column_name)})"
+ return f"{aggregate}({dataset.quote_identifier(column_name)})"
+
+
+def adhoc_metric_to_semantic_metric(
+ adhoc: dict[str, Any],
+ dataset: "SqlaTable",
+ template_processor: "BaseTemplateProcessor | None" = None,
+) -> Metric:
+ """
+ Convert an ``AdhocMetric`` dict to a semantic-layer ``Metric``.
+
+ SIMPLE metrics build ``f"{aggregate}({column})"`` and tag the
+ aggregation type so the view can preserve rollup semantics. SQL
+ metrics use the user-provided ``sqlExpression`` (after Jinja +
+ safety validation) and report ``AggregationType.OTHER`` because the
+ expression is opaque.
+
+ Adhoc dtype is set to ``pa.null()`` — the view aliases the metric
+ by id and downstream chart code coerces at render time.
+ """
+ label = adhoc.get("label")
+ if not label:
+ raise QueryObjectValidationError("Adhoc metric is missing a ``label``.")
+
+ expression_type = adhoc.get("expressionType")
+ if expression_type == AdhocMetricExpressionType.SIMPLE.value:
+ definition = _simple_metric_definition(adhoc, dataset)
+ # Templating is irrelevant for SIMPLE metrics (no user-supplied SQL),
+ # so the safety pass is skipped here.
+ aggregate = (adhoc.get("aggregate") or "").upper()
+ aggregation = _AGGREGATE_MAP.get(aggregate, AggregationType.OTHER)
+ elif expression_type == AdhocMetricExpressionType.SQL.value:
+ sql_expression = adhoc.get("sqlExpression")
+ if not sql_expression:
+ raise QueryObjectValidationError(
+ "SQL adhoc metric is missing ``sqlExpression``."
+ )
+ definition = _process_adhoc_sql(dataset, sql_expression, template_processor)
+ aggregation = AggregationType.OTHER
+ else:
+ raise QueryObjectValidationError(
+ f"Unknown adhoc metric expressionType: {expression_type!r}"
+ )
+
+ return Metric(
+ id=label,
+ name=label,
+ type=pa.null(),
+ definition=definition,
+ aggregation=aggregation,
+ )
+
+
+def adhoc_column_to_semantic_dimension(
+ adhoc: dict[str, Any],
+ dataset: "SqlaTable",
+ dimensions_by_name: dict[str, Dimension],
+ template_processor: "BaseTemplateProcessor | None" = None,
+) -> Dimension:
+ """
+ Convert an ``AdhocColumn`` dict to a semantic-layer ``Dimension``.
+
+ When the adhoc is a thin wrapper around a real column
+ (``isColumnReference=True`` and ``sqlExpression`` matches a known
+ dimension name), the existing ``Dimension`` is returned so that
+ metadata such as type and grain are preserved. Otherwise a fresh
+ ``Dimension`` with ``definition=<rendered SQL>`` is synthesized.
+ """
+ label = adhoc.get("label")
+ if not label:
+ raise QueryObjectValidationError("Adhoc column is missing a ``label``.")
+ sql_expression = adhoc.get("sqlExpression")
+ if not sql_expression:
+ raise QueryObjectValidationError(
+ "Adhoc column is missing ``sqlExpression``."
+ )
+
+ if adhoc.get("isColumnReference") and sql_expression in dimensions_by_name:
+ return dimensions_by_name[sql_expression]
+
+ definition = _process_adhoc_sql(dataset, sql_expression, template_processor)
+ return Dimension(
+ id=label,
+ name=label,
+ type=pa.null(),
+ definition=definition,
+ )
diff --git a/superset/semantic_layers/extension/backend/src/preset_io/dataset_semantic_layer/view.py b/superset/semantic_layers/extension/backend/src/preset_io/dataset_semantic_layer/view.py
index 37e1ccab6a6..6fb6c0d87a1 100644
--- a/superset/semantic_layers/extension/backend/src/preset_io/dataset_semantic_layer/view.py
+++ b/superset/semantic_layers/extension/backend/src/preset_io/dataset_semantic_layer/view.py
@@ -176,10 +176,14 @@ class DatasetSemanticView(SemanticView):
# ------------------------------------------------------------------
def _dimension_column(self, dimension: Dimension) -> "TableColumn":
+ # Callers (currently only ``_dimension_expression``) check membership
+ # before invoking this helper, so the match is guaranteed.
for column in self.dataset.columns:
if column.column_name == dimension.id:
return column
- raise ValueError(f'Dimension "{dimension.id}" is not part of this dataset.')
+ raise ValueError( # pragma: no cover - guarded by caller
+ f'Dimension "{dimension.id}" is not part of this dataset.'
+ )
def _metric_column(self, metric: Metric) -> "SqlMetric":
for sql_metric in self.dataset.metrics:
@@ -188,10 +192,76 @@ class DatasetSemanticView(SemanticView):
raise ValueError(f'Metric "{metric.id}" is not part of this dataset.')
def _dimension_expression(self, dimension: Dimension) -> sqlglot_exp.Expression:
+ # Synthesised adhoc dimensions don't correspond to a dataset column;
+ # their ``definition`` carries the SQL we should emit.
+ dataset_column_names = {col.column_name for col in self.dataset.columns}
+ if dimension.id not in dataset_column_names:
+ if not dimension.definition:
+ raise ValueError(
+ f'Dimension "{dimension.id}" is not part of this dataset.'
+ )
+ base: sqlglot_exp.Expression = sqlglot.parse_one(
+ dimension.definition,
+ dialect=self._sqlglot_dialect,
+ )
+ return self._apply_grain(dimension, base)
+
column = self._dimension_column(dimension)
if column.expression:
- return sqlglot.parse_one(column.expression, dialect=self._sqlglot_dialect)
- return sqlglot_exp.column(column.column_name, quoted=True)
+ base = sqlglot.parse_one(
+ column.expression,
+ dialect=self._sqlglot_dialect,
+ )
+ else:
+ base = sqlglot_exp.column(column.column_name, quoted=True)
+ return self._apply_grain(dimension, base)
+
+ def _apply_grain(
+ self,
+ dimension: Dimension,
+ base_expr: sqlglot_exp.Expression,
+ ) -> sqlglot_exp.Expression:
+ """
+ Wrap ``base_expr`` in the engine spec's time-grain template when the
+ dimension carries a grain. Engines that don't model the requested
+ grain return the unwrapped expression — same fallback the legacy path
+ uses.
+ """
+ if dimension.grain is None:
+ return base_expr
+
+ engine_spec = self.dataset.database.db_engine_spec
+ template = engine_spec.get_time_grain_expressions().get(
+ dimension.grain.representation
+ )
+ if not template:
+ return base_expr
+
+ # ``{func}`` / ``{type}`` placeholders (BigQuery, MS SQL) require the
+ # engine spec's full ``get_timestamp_expr`` machinery to resolve.
+ if "{func}" in template or "{type}" in template:
+ import sqlalchemy as sa
+
+ sa_col = sa.column(
+ base_expr.sql(dialect=self._sqlglot_dialect),
+ is_literal=True,
+ )
+ tse = engine_spec.get_timestamp_expr(
+ sa_col,
+ None,
+ dimension.grain.representation,
+ )
+ sql_str = str(
+ tse.compile(
+ dialect=self.dataset.database.get_dialect(),
+ compile_kwargs={"literal_binds": True},
+ )
+ )
+ return sqlglot.parse_one(sql_str, dialect=self._sqlglot_dialect)
+
+ col_sql = base_expr.sql(dialect=self._sqlglot_dialect)
+ grain_sql = template.replace("{col}", col_sql)
+ return sqlglot.parse_one(grain_sql, dialect=self._sqlglot_dialect)
def _metric_expression(self, metric: Metric) -> sqlglot_exp.Expression:
sql_metric = self._metric_column(metric)
@@ -202,22 +272,82 @@ class DatasetSemanticView(SemanticView):
Build the FROM clause. Physical datasets reference the table directly;
virtual datasets wrap their SQL as a subquery so the rest of the AST
can treat the source like a table.
+
+ For virtual datasets, RLS rules from the underlying tables referenced
+ in ``dataset.sql`` are injected into the inner SQL. When the engine
+ spec asks for ``AS_SUBQUERY``-style RLS, the dataset's own RLS
+ predicates wrap the source in an extra ``SELECT * FROM … WHERE`` so
+ the rest of the AST is unaffected.
"""
dataset = self.dataset
+
if dataset.sql:
- inner = sqlglot.parse_one(dataset.sql, dialect=self._sqlglot_dialect)
- return sqlglot_exp.Subquery(this=inner, alias="virtual_dataset")
+ inner_sql = self._rls_apply_to_virtual_sql() or dataset.sql
+ inner = sqlglot.parse_one(inner_sql, dialect=self._sqlglot_dialect)
+ source: sqlglot_exp.Expression = sqlglot_exp.Subquery(
+ this=inner, alias="virtual_dataset"
+ )
+ else:
+ parts = [
+ sqlglot_exp.to_identifier(part, quoted=True)
+ for part in (dataset.catalog, dataset.schema, dataset.table_name)
+ if part
+ ]
+ if len(parts) == 1:
+ source = sqlglot_exp.Table(this=parts[0])
+ elif len(parts) == 2:
+ source = sqlglot_exp.Table(this=parts[1], db=parts[0])
+ else:
+ source = sqlglot_exp.Table(this=parts[2], db=parts[1], catalog=parts[0])
+
+ if self._rls_should_wrap_subquery():
+ rls_clauses = self._rls_predicates()
+ if rls_clauses:
+ source = self._wrap_with_rls(source, rls_clauses)
+
+ return source
+
+ # ------------------------------------------------------------------
+ # RLS
+ # ------------------------------------------------------------------
+
+ def _rls_predicates(self) -> list[str]:
+ """Outer RLS predicates for the dataset, rendered as SQL strings."""
+ from superset.semantic_layers.rls import render_rls_predicates
+
+ return render_rls_predicates(self.dataset)
- parts = [
- sqlglot_exp.to_identifier(part, quoted=True)
- for part in (dataset.catalog, dataset.schema, dataset.table_name)
- if part
+ def _rls_apply_to_virtual_sql(self) -> str | None:
+ from superset.semantic_layers.rls import apply_rls_to_virtual_sql
+
+ return apply_rls_to_virtual_sql(self.dataset)
+
+ def _rls_should_wrap_subquery(self) -> bool:
+ from superset.semantic_layers.rls import get_rls_method
+ from superset.sql.parse import RLSMethod
+
+ return get_rls_method(self.dataset) == RLSMethod.AS_SUBQUERY
+
+ def _wrap_with_rls(
+ self,
+ source: sqlglot_exp.Expression,
+ rls_clauses: list[str],
+ ) -> sqlglot_exp.Subquery:
+ """Wrap ``source`` in ``SELECT * FROM source WHERE <AND-ed RLS>``."""
+ parsed = [
+ sqlglot.parse_one(clause, dialect=self._sqlglot_dialect)
+ for clause in rls_clauses
]
- if len(parts) == 1:
- return sqlglot_exp.Table(this=parts[0])
- if len(parts) == 2:
- return sqlglot_exp.Table(this=parts[1], db=parts[0])
- return sqlglot_exp.Table(this=parts[2], db=parts[1], catalog=parts[0])
+ combined = parsed[0]
+ for predicate in parsed[1:]:
+ combined = sqlglot_exp.And(this=combined, expression=predicate)
+ inner = (
+ sqlglot_exp.Select()
+ .select(sqlglot_exp.Star())
+ .from_(source)
+ .where(combined)
+ )
+ return sqlglot_exp.Subquery(this=inner, alias="rls_wrapped")
def _filter_predicate(self, filter_: Filter) -> sqlglot_exp.Expression:
if filter_.operator == Operator.ADHOC:
@@ -316,6 +446,20 @@ class DatasetSemanticView(SemanticView):
where_filters = {f for f in filters if f.type == PredicateType.WHERE}
having_filters = {f for f in filters if f.type == PredicateType.HAVING}
+ # Add the dataset's outer RLS predicates as ADHOC where-filters when
+ # the engine wants AS_PREDICATE-style RLS. AS_SUBQUERY-style RLS is
+ # handled in ``_source_table`` instead.
+ if not self._rls_should_wrap_subquery():
+ for clause in self._rls_predicates():
+ where_filters = where_filters | {
+ Filter(
+ type=PredicateType.WHERE,
+ column=None,
+ operator=Operator.ADHOC,
+ value=clause,
+ )
+ }
+
if query.group_limit is not None and query.group_limit.group_others:
return self._build_with_others(query, where_filters, having_filters)
if query.group_limit is not None:
@@ -356,10 +500,16 @@ class DatasetSemanticView(SemanticView):
select = select.where(where_predicate)
# GROUP BY when aggregating: any metric is an aggregate, so all
- # non-aggregate dimensions need to be in GROUP BY.
+ # non-aggregate dimensions need to be in GROUP BY. For grain-bucketed
+ # dimensions we GROUP BY the full bucketed expression because dialects
+ # disagree on whether SELECT aliases are visible in GROUP BY.
if query.metrics and query.dimensions:
group_by_columns = [
- sqlglot_exp.column(dimension.id, quoted=True)
+ (
+ self._dimension_expression(dimension)
+ if dimension.grain is not None
+ else sqlglot_exp.column(dimension.id, quoted=True)
+ )
for dimension in query.dimensions
]
select = select.group_by(*group_by_columns)
@@ -427,10 +577,16 @@ class DatasetSemanticView(SemanticView):
# When ordering by a metric we need an aggregation; GROUP BY the
# limited dimensions so each combination collapses to a single row.
+ # Use full expressions for grain-bucketed dims (alias resolution in
+ # GROUP BY isn't portable).
if group_limit.metric is not None:
select = select.group_by(
*[
- sqlglot_exp.column(dim.id, quoted=True)
+ (
+ self._dimension_expression(dim)
+ if dim.grain is not None
+ else sqlglot_exp.column(dim.id, quoted=True)
+ )
for dim in group_limit.dimensions
]
)
@@ -582,9 +738,22 @@ class DatasetSemanticView(SemanticView):
dimension: Dimension,
filters: set[Filter] | None = None,
) -> SemanticResult:
- where = self._combine_predicates(
- {f for f in (filters or set()) if f.type == PredicateType.WHERE}
- )
+ where_filters = {
+ f for f in (filters or set()) if f.type == PredicateType.WHERE
+ }
+ # Add outer RLS predicates as ADHOC filters; AS_SUBQUERY-mode RLS is
+ # wrapped into ``_source_table`` already.
+ if not self._rls_should_wrap_subquery():
+ for clause in self._rls_predicates():
+ where_filters.add(
+ Filter(
+ type=PredicateType.WHERE,
+ column=None,
+ operator=Operator.ADHOC,
+ value=clause,
+ )
+ )
+ where = self._combine_predicates(where_filters)
select = (
sqlglot_exp.Select()
.distinct()
diff --git a/superset/semantic_layers/extension/backend/tests/test_semantic_view.py b/superset/semantic_layers/extension/backend/tests/test_semantic_view.py
index e90d72f0809..8744afb1348 100644
--- a/superset/semantic_layers/extension/backend/tests/test_semantic_view.py
+++ b/superset/semantic_layers/extension/backend/tests/test_semantic_view.py
@@ -39,6 +39,26 @@ def _make_metric(name, expression) -> MagicMock:
return metric
+@pytest.fixture(autouse=True)
+def stub_rls_module(monkeypatch: pytest.MonkeyPatch):
+ """
+ Stub ``superset.semantic_layers.rls`` so tests don't need a Flask app.
+ Individual tests can override these by re-monkeypatching the module's
+ attributes.
+ """
+ import sys
+ import types
+
+ from superset.sql.parse import RLSMethod
+
+ rls_stub = types.ModuleType("superset.semantic_layers.rls")
+ rls_stub.render_rls_predicates = lambda dataset: []
+ rls_stub.apply_rls_to_virtual_sql = lambda dataset: None
+ rls_stub.get_rls_method = lambda dataset: RLSMethod.AS_PREDICATE
+ monkeypatch.setitem(sys.modules, "superset.semantic_layers.rls", rls_stub)
+ yield rls_stub
+
+
@pytest.fixture
def dataset() -> MagicMock:
db = MagicMock()
@@ -863,3 +883,257 @@ def test_get_row_count_wraps_query(dataset: MagicMock) -> None:
assert "COUNT(*)" in sql_arg
assert "FROM (" in sql_arg
assert result.results.to_pydict()["COUNT"][0] == 3
+
+
+# ---------------------------------------------------------------------------
+# RLS
+# ---------------------------------------------------------------------------
+
+
+def test_rls_as_predicate_appended_to_where(dataset: MagicMock, stub_rls_module) -> None:
+ stub_rls_module.render_rls_predicates = lambda ds: ["tenant_id = 7"]
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ assert 'WHERE tenant_id = 7' in sql
+
+
+def test_rls_as_predicate_combined_with_existing_filter(
+ dataset: MagicMock,
+ stub_rls_module,
+) -> None:
+ stub_rls_module.render_rls_predicates = lambda ds: ["tenant_id = 7"]
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+ status = next(d for d in view.dimensions if d.id == "status")
+
+ sql = view.compile_query(
+ SemanticQuery(
+ metrics=[],
+ dimensions=[country],
+ filters={
+ Filter(
+ type=PredicateType.WHERE,
+ column=status,
+ operator=Operator.EQUALS,
+ value="open",
+ ),
+ },
+ )
+ )
+ assert "tenant_id = 7" in sql
+ assert "\"status\" = 'open'" in sql
+ assert " AND " in sql
+
+
+def test_rls_multiple_clauses_anded(dataset: MagicMock, stub_rls_module) -> None:
+ stub_rls_module.render_rls_predicates = lambda ds: ["a = 1", "b = 2"]
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ # Both clauses present, AND-ed.
+ assert "a = 1" in sql
+ assert "b = 2" in sql
+
+
+def test_rls_as_subquery_multiple_clauses_anded(
+ dataset: MagicMock,
+ stub_rls_module,
+) -> None:
+ """Subquery-wrap mode AND-s multiple RLS clauses inside the wrap."""
+ from superset.sql.parse import RLSMethod
+
+ stub_rls_module.get_rls_method = lambda ds: RLSMethod.AS_SUBQUERY
+ stub_rls_module.render_rls_predicates = lambda ds: ["a = 1", "b = 2"]
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ assert "a = 1" in sql
+ assert "b = 2" in sql
+ assert " AND " in sql
+
+
+def test_adhoc_dimension_compiles_via_definition(dataset: MagicMock) -> None:
+ """An adhoc dimension (no matching column, but with a ``definition``) is
+ parsed and emitted as the SELECT expression."""
+ adhoc = Dimension(
+ id="upper_country",
+ name="upper_country",
+ type=pa.null(),
+ definition="UPPER(country)",
+ )
+ view = DatasetSemanticView(dataset)
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[adhoc]))
+ assert 'UPPER(country) AS "upper_country"' in sql
+
+
+def test_rls_as_subquery_wraps_source(dataset: MagicMock, stub_rls_module) -> None:
+ from superset.sql.parse import RLSMethod
+
+ stub_rls_module.get_rls_method = lambda ds: RLSMethod.AS_SUBQUERY
+ stub_rls_module.render_rls_predicates = lambda ds: ["tenant_id = 7"]
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ # Source wrapped in a subquery containing the RLS predicate; no top-level WHERE.
+ assert "FROM (SELECT * FROM" in sql
+ assert "WHERE tenant_id = 7" in sql
+ assert sql.count(" WHERE ") == 1 # only the wrap, not also outer
+
+
+def test_rls_as_subquery_no_wrap_when_no_predicates(
+ dataset: MagicMock,
+ stub_rls_module,
+) -> None:
+ from superset.sql.parse import RLSMethod
+
+ stub_rls_module.get_rls_method = lambda ds: RLSMethod.AS_SUBQUERY
+ stub_rls_module.render_rls_predicates = lambda ds: []
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ # No RLS rules → no wrap, plain table reference.
+ assert 'FROM "public"."orders"' in sql
+ assert "rls_wrapped" not in sql
+
+
+def test_rls_virtual_dataset_inner_apply(dataset: MagicMock, stub_rls_module) -> None:
+ dataset.sql = "SELECT * FROM raw_orders"
+ stub_rls_module.apply_rls_to_virtual_sql = (
+ lambda ds: "SELECT * FROM raw_orders WHERE tenant_id = 7"
+ )
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ assert "raw_orders WHERE tenant_id = 7" in sql
+
+
+def test_rls_virtual_dataset_no_inner_change_falls_back(
+ dataset: MagicMock,
+ stub_rls_module,
+) -> None:
+ dataset.sql = "SELECT * FROM raw_orders"
+ stub_rls_module.apply_rls_to_virtual_sql = lambda ds: None
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[country]))
+ assert "FROM (SELECT * FROM raw_orders)" in sql
+
+
+# ---------------------------------------------------------------------------
+# Time grain
+# ---------------------------------------------------------------------------
+
+
+def _engine_spec_with_grain_map(grain_map: dict) -> MagicMock:
+ spec = MagicMock()
+ spec.engine = "postgresql"
+ spec.get_time_grain_expressions.return_value = grain_map
+ return spec
+
+
+def test_grain_simple_template_buckets_dimension(dataset: MagicMock) -> None:
+ """``{col}``-only templates are formatted directly and reparsed."""
+ import dataclasses
+ from superset_core.semantic_layers.types import Grains
+
+ dataset.database.db_engine_spec = _engine_spec_with_grain_map(
+ {"P1D": "DATE_TRUNC('day', {col})"}
+ )
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+ grained = dataclasses.replace(country, grain=Grains.DAY)
+ total = next(m for m in view.metrics if m.id == "total_amount")
+
+ sql = view.compile_query(
+ SemanticQuery(metrics=[total], dimensions=[grained])
+ )
+ # sqlglot normalizes the time-unit literal's case for Postgres.
+ assert 'DATE_TRUNC(\'DAY\', "country") AS "country"' in sql
+ # GROUP BY uses the full bucketed expression, not the alias.
+ assert 'GROUP BY DATE_TRUNC(\'DAY\', "country")' in sql
+
+
+def test_grain_unsupported_falls_back_to_raw(dataset: MagicMock) -> None:
+ """When the engine doesn't model the requested grain, no wrap is applied."""
+ import dataclasses
+ from superset_core.semantic_layers.types import Grain, Grains
+
+ # The map is empty → no template matches.
+ dataset.database.db_engine_spec = _engine_spec_with_grain_map({})
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+ grained = dataclasses.replace(country, grain=Grains.DAY)
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[grained]))
+ assert 'SELECT "country" AS "country"' in sql
+ assert "DATE_TRUNC" not in sql
+
+
+def test_grain_func_placeholder_uses_engine_spec(
+ dataset: MagicMock,
+ mocker,
+) -> None:
+ """``{func}``/``{type}`` placeholders defer to engine spec's get_timestamp_expr."""
+ import dataclasses
+ from superset_core.semantic_layers.types import Grains
+
+ fake_spec = _engine_spec_with_grain_map({"P1D": "{func}({col}, DAY)"})
+ # get_timestamp_expr is the engine spec's escape hatch; return a fake
+ # SQLAlchemy expression whose str() form is the rendered SQL.
+ fake_te = MagicMock()
+ fake_te.compile.return_value = 'TIMESTAMP_TRUNC("country", DAY)'
+ fake_spec.get_timestamp_expr.return_value = fake_te
+ dataset.database.db_engine_spec = fake_spec
+
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+ grained = dataclasses.replace(country, grain=Grains.DAY)
+
+ view.compile_query(SemanticQuery(metrics=[], dimensions=[grained]))
+ # The placeholder path must consult the engine spec's get_timestamp_expr
+ # rather than substituting directly. We don't assert on the rendered SQL
+ # because sqlglot may canonicalize non-native function names per dialect.
+ fake_spec.get_timestamp_expr.assert_called_once()
+
+
+def test_grain_applied_to_calculated_dimension(dataset: MagicMock) -> None:
+ """Grain wraps a calculated column's expression, not a bare column."""
+ import dataclasses
+ from superset_core.semantic_layers.types import Grains
+
+ calc = _make_column("created_at_minus_1", expression="created_at - INTERVAL 1 DAY")
+ dataset.columns = list(dataset.columns) + [calc]
+ dataset.database.db_engine_spec = _engine_spec_with_grain_map(
+ {"P1D": "DATE_TRUNC('day', {col})"}
+ )
+
+ view = DatasetSemanticView(dataset)
+ dim = next(d for d in view.dimensions if d.id == "created_at_minus_1")
+ grained = dataclasses.replace(dim, grain=Grains.DAY)
+
+ sql = view.compile_query(SemanticQuery(metrics=[], dimensions=[grained]))
+ # The grain wraps the calculated expression; sqlglot re-renders the
+ # interval literal so we check the structural pieces individually.
+ assert "DATE_TRUNC('DAY'," in sql
+ assert "created_at - INTERVAL" in sql
+
+
+def test_rls_applied_to_get_values(dataset: MagicMock, stub_rls_module) -> None:
+ import pandas as pd
+
+ stub_rls_module.render_rls_predicates = lambda ds: ["tenant_id = 7"]
+ dataset.database.get_df.return_value = pd.DataFrame({"country": []})
+ view = DatasetSemanticView(dataset)
+ country = next(d for d in view.dimensions if d.id == "country")
+
+ view.get_values(country)
+ sql_arg = dataset.database.get_df.call_args.args[0]
+ assert "tenant_id = 7" in sql_arg
diff --git a/superset/semantic_layers/mapper.py b/superset/semantic_layers/mapper.py
index b71dfeb44db..ee652da3870 100644
--- a/superset/semantic_layers/mapper.py
+++ b/superset/semantic_layers/mapper.py
@@ -24,6 +24,7 @@ single dataframe.
"""
+import dataclasses
from datetime import date, datetime, time, timedelta, tzinfo
from time import time as current_time
from typing import Any, cast, Sequence, TypeGuard
@@ -56,7 +57,11 @@ from superset.common.utils.time_range_utils import get_since_until_from_query_ob
from superset.connectors.sqla.models import BaseDatasource
from superset.constants import NO_TIME_RANGE
from superset.models.helpers import QueryResult
-from superset.superset_typing import AdhocColumn
+from superset.semantic_layers.adhoc import (
+ adhoc_column_to_semantic_dimension,
+ adhoc_metric_to_semantic_metric,
+)
+from superset.superset_typing import AdhocColumn, AdhocMetric
from superset.utils.core import (
FilterOperator,
QueryObjectFilterClause,
@@ -258,16 +263,92 @@ def _normalize_column(column: str | AdhocColumn, dimension_names: set[str]) -> s
- A string (dimension name directly)
- An AdhocColumn with isColumnReference=True and sqlExpression containing the
dimension name
+
+ Used by callers that only care about the resolved *name* (e.g. validating
+ that a granularity column matches a real dimension). Synthetic adhocs
+ (non-column-reference) carry no dimension name and should be routed
+ through :func:`_resolve_dimension` instead.
"""
if isinstance(column, str):
return column
- # Handle column references (e.g., from time-series charts)
+ # Column reference adhocs unwrap to their underlying column name.
if column.get("isColumnReference") and (sql_expr := column.get("sqlExpression")):
if sql_expr in dimension_names:
return sql_expr
- raise ValueError("Adhoc dimensions are not supported in Semantic Views.")
+ # Synthetic adhoc: the label *is* the resolved name in semantic-layer
+ # terms (we use it as the Dimension id/name when materialising). Falling
+ # back here keeps validators happy with adhoc dicts and defers actual
+ # resolution to ``_resolve_dimension``.
+ if label := column.get("label"):
+ return label
+
+ raise ValueError("Adhoc column is missing a ``label``.")
+
+
+def _resolve_dimension(
+ column: str | AdhocColumn,
+ dataset: BaseDatasource,
+ dimensions_by_name: dict[str, Dimension],
+ template_processor: Any | None,
+) -> Dimension:
+ """
+ Resolve a column entry (string name or adhoc dict) to a ``Dimension``.
+
+ Strings look up a saved dimension by name. Adhoc dicts marked
+ ``isColumnReference=True`` re-use the matching saved dimension so its
+ metadata (type, grain) is preserved. Anything else is synthesised via
+ :func:`adhoc_column_to_semantic_dimension`.
+ """
+ if isinstance(column, str):
+ if column not in dimensions_by_name:
+ raise ValueError(
+ f"Dimension {column!r} is not defined in the Semantic View."
+ )
+ return dimensions_by_name[column]
+
+ return adhoc_column_to_semantic_dimension(
+ column,
+ dataset,
+ dimensions_by_name,
+ template_processor,
+ )
+
+
+def _resolve_metric(
+ metric: str | AdhocMetric,
+ dataset: BaseDatasource,
+ metrics_by_name: dict[str, Metric],
+ template_processor: Any | None,
+) -> Metric:
+ """
+ Resolve a metric entry (string name or adhoc dict) to a ``Metric``.
+ """
+ if isinstance(metric, str):
+ if metric not in metrics_by_name:
+ raise ValueError(
+ f"Metric {metric!r} is not defined in the Semantic View."
+ )
+ return metrics_by_name[metric]
+ return adhoc_metric_to_semantic_metric(metric, dataset, template_processor)
+
+
+def _get_template_processor(dataset: BaseDatasource) -> Any | None:
+ """
+ Build a template processor for adhoc Jinja rendering, or ``None`` when
+ the datasource doesn't expose one (e.g. a cube-mode semantic view, where
+ adhocs aren't supported anyway).
+ """
+ get_template_processor = getattr(dataset, "get_template_processor", None)
+ if get_template_processor is None:
+ return None
+ return get_template_processor()
+
+
+def _stamp_grain(dimension: Dimension, grain: Grain) -> Dimension:
+ """Return a copy of ``dimension`` with the supplied grain set."""
+ return dataclasses.replace(dimension, grain=grain)
def map_query_object(query_object: ValidatedQueryObject) -> list[SemanticQuery]:
@@ -278,33 +359,54 @@ def map_query_object(query_object: ValidatedQueryObject) -> list[SemanticQuery]:
visualization and more on semantics.
"""
semantic_view = query_object.datasource.implementation
+ dataset = query_object.datasource
all_metrics = {metric.name: metric for metric in semantic_view.metrics}
all_dimensions = {
dimension.name: dimension for dimension in semantic_view.dimensions
}
- # Normalize columns (may be dicts with isColumnReference=True for time-series)
- dimension_names = set(all_dimensions.keys())
- normalized_columns = {
- _normalize_column(column, dimension_names) for column in query_object.columns
- }
+ template_processor = _get_template_processor(dataset)
- metrics = [all_metrics[metric] for metric in (query_object.metrics or [])]
+ metrics = [
+ _resolve_metric(metric, dataset, all_metrics, template_processor)
+ for metric in (query_object.metrics or [])
+ ]
- grain = _convert_time_grain(query_object.extras.get("time_grain_sqla"))
- dimensions = [
- dimension
- for dimension in semantic_view.dimensions
- if dimension.name in normalized_columns
- and (
- # if a grain is specified, only include the time dimension if its grain
- # matches the requested grain
- grain is None
- or dimension.name != query_object.granularity
- or dimension.grain == grain
+ # Resolve each requested column, preserving order and deduplicating by id.
+ seen_dim_ids: set[str] = set()
+ dimensions: list[Dimension] = []
+ for column in query_object.columns:
+ dim = _resolve_dimension(
+ column,
+ dataset,
+ all_dimensions,
+ template_processor,
)
- ]
+ if dim.id in seen_dim_ids:
+ continue
+ seen_dim_ids.add(dim.id)
+ dimensions.append(dim)
+
+ grain = _convert_time_grain(query_object.extras.get("time_grain_sqla"))
+ if grain is not None and query_object.granularity:
+ # Apply the requested grain to the granularity dimension. For cube-mode
+ # views that pre-declare grain on their dimensions, keep the existing
+ # "only include the matching-grain version" semantic. For dataset-mode
+ # views (dimensions have grain=None), the view applies the grain via
+ # the engine spec at SQL build time, so we just stamp the grain on
+ # the dimension here.
+ dimensions = [
+ (
+ _stamp_grain(dim, grain)
+ if dim.name == query_object.granularity and dim.grain is None
+ else dim
+ )
+ for dim in dimensions
+ if dim.name != query_object.granularity
+ or dim.grain is None
+ or dim.grain == grain
+ ]
order = _get_order_from_query_object(query_object, all_metrics, all_dimensions)
limit = query_object.row_limit
@@ -928,32 +1030,53 @@ def validate_query_object(
def _validate_metrics(query_object: ValidatedQueryObject) -> None:
"""
- Make sure metrics are defined in the semantic view.
+ Make sure metrics are defined in the semantic view or are valid adhocs.
+
+ Validation of adhoc metrics is delegated to ``_resolve_metric`` — it
+ raises if the SIMPLE shape is malformed or the SQL fails Jinja /
+ safety checks.
"""
semantic_view = query_object.datasource.implementation
+ dataset = query_object.datasource
+ metrics_by_name = {metric.name: metric for metric in semantic_view.metrics}
+ template_processor = _get_template_processor(dataset)
+
+ labels: list[str] = []
+ for metric in query_object.metrics or []:
+ resolved = _resolve_metric(
+ metric, dataset, metrics_by_name, template_processor
+ )
+ labels.append(resolved.id)
- if any(not isinstance(metric, str) for metric in (query_object.metrics or [])):
- raise ValueError("Adhoc metrics are not supported in Semantic Views.")
-
- metric_names = {metric.name for metric in semantic_view.metrics}
- if not set(query_object.metrics or []) <= metric_names:
- raise ValueError("All metrics must be defined in the Semantic View.")
+ if len(labels) != len(set(labels)):
+ raise ValueError(
+ "Duplicate metric labels are not supported in Semantic Views."
+ )
def _validate_dimensions(query_object: ValidatedQueryObject) -> None:
"""
- Make sure all dimensions are defined in the semantic view.
+ Make sure all dimensions are defined in the semantic view or are valid
+ adhocs. Synthesized adhoc columns are validated via ``_resolve_dimension``.
"""
semantic_view = query_object.datasource.implementation
- dimension_names = {dimension.name for dimension in semantic_view.dimensions}
+ dataset = query_object.datasource
+ dimensions_by_name = {
+ dimension.name: dimension for dimension in semantic_view.dimensions
+ }
+ template_processor = _get_template_processor(dataset)
- # Normalize all columns to dimension names
- normalized_columns = [
- _normalize_column(column, dimension_names) for column in query_object.columns
- ]
+ labels: list[str] = []
+ for column in query_object.columns:
+ resolved = _resolve_dimension(
+ column, dataset, dimensions_by_name, template_processor
+ )
+ labels.append(resolved.id)
- if not set(normalized_columns) <= dimension_names:
- raise ValueError("All dimensions must be defined in the Semantic View.")
+ if len(labels) != len(set(labels)):
+ raise ValueError(
+ "Duplicate column labels are not supported in Semantic Views."
+ )
def _validate_filters(query_object: ValidatedQueryObject) -> None:
diff --git a/superset/semantic_layers/rls.py b/superset/semantic_layers/rls.py
new file mode 100644
index 00000000000..92597a6848e
--- /dev/null
+++ b/superset/semantic_layers/rls.py
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+RLS helpers used by the dataset semantic-view path.
+
+The legacy ``get_sqla_query`` applies row-level security in two places:
+
+* The dataset's own RLS rules are AND-ed into the outer ``WHERE`` (or
+ the source table is wrapped in a subquery when the engine spec
+ returns ``RLSMethod.AS_SUBQUERY``).
+* For virtual datasets, RLS rules from the *underlying* tables
+ referenced by ``dataset.sql`` are injected via
+ :func:`superset.utils.rls.apply_rls`.
+
+This module exposes both flows as pure functions returning SQL strings,
+which the ``DatasetSemanticView`` plugs into its sqlglot AST.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from superset.sql.parse import RLSMethod, SQLScript
+from superset.utils.rls import apply_rls
+
+if TYPE_CHECKING:
+ from superset.connectors.sqla.models import SqlaTable
+
+logger = logging.getLogger(__name__)
+
+
+def get_rls_method(dataset: "SqlaTable") -> RLSMethod:
+ """Return the engine's preferred RLS injection method."""
+ return dataset.database.db_engine_spec.get_rls_method()
+
+
+def render_rls_predicates(dataset: "SqlaTable") -> list[str]:
+ """
+ Render the dataset's outer RLS predicates as SQL strings.
+
+ The clauses are already Jinja-rendered by
+ :meth:`SqlaTable.get_sqla_row_level_filters`; we then compile them
+ against the database's dialect with ``literal_binds=True`` so the
+ resulting strings are safe to splice into a sqlglot AST.
+
+ Returns an empty list when the dataset has no applicable RLS rules
+ for the current user.
+ """
+ text_clauses = dataset.get_sqla_row_level_filters()
+ if not text_clauses:
+ return []
+ dialect = dataset.database.get_dialect()
+ return [
+ str(clause.compile(dialect=dialect, compile_kwargs={"literal_binds": True}))
+ for clause in text_clauses
+ ]
+
+
+def apply_rls_to_virtual_sql(dataset: "SqlaTable") -> str | None:
+ """
+ Rewrite a virtual dataset's inner SQL to enforce RLS on the
+ underlying tables it references.
+
+ Returns the rewritten SQL string when RLS predicates were injected,
+ or ``None`` when no rules applied (caller should use the original
+ ``dataset.sql`` in that case). Returns ``None`` for physical
+ datasets too.
+ """
+ if not dataset.sql:
+ return None
+
+ engine = dataset.database.db_engine_spec.engine
+ try:
+ parsed_script = SQLScript(dataset.sql, engine=engine)
+ except Exception: # pylint: disable=broad-except
+ # Mirror the legacy behavior — failure to parse should not block
+ # the query; the caller will fall back to the original SQL and
+ # RLS will simply not be applied to inner tables.
+ logger.warning(
+ "Failed to parse virtual dataset SQL for RLS application",
+ exc_info=True,
+ )
+ return None
+
+ default_schema = dataset.database.get_default_schema(dataset.catalog)
+ rls_applied = False
+ try:
+ for statement in parsed_script.statements:
+ if apply_rls(
+ dataset.database,
+ dataset.catalog,
+ dataset.schema or default_schema or "",
+ statement,
+ exclude_dataset_id=dataset.id,
+ ):
+ rls_applied = True
+ except Exception: # pylint: disable=broad-except
+ logger.warning(
+ "Failed to apply RLS to virtual dataset inner SQL",
+ exc_info=True,
+ )
+ return None
+
+ return parsed_script.format() if rls_applied else None
diff --git a/tests/unit_tests/semantic_layers/adhoc_test.py b/tests/unit_tests/semantic_layers/adhoc_test.py
new file mode 100644
index 00000000000..756d59d17dd
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/adhoc_test.py
@@ -0,0 +1,268 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=invalid-name
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pyarrow as pa
+import pytest
+from superset_core.semantic_layers.types import AggregationType, Dimension
+
+from superset.exceptions import QueryObjectValidationError
+from superset.semantic_layers.adhoc import (
+ adhoc_column_to_semantic_dimension,
+ adhoc_metric_to_semantic_metric,
+)
+
+
+@pytest.fixture
+def dataset() -> MagicMock:
+ ds = MagicMock()
+ ds.database_id = 1
+ ds.schema = "public"
+ ds.database.db_engine_spec.engine = "postgresql"
+ # Identifier quoter — return double-quoted name.
+ ds.quote_identifier = lambda name: f'"{name}"'
+ # ``_process_select_expression`` echoes back the SQL after "rendering".
+ ds._process_select_expression = (
+ lambda expression, **kwargs: expression # noqa: U100
+ )
+ return ds
+
+
+# ---------------------------------------------------------------------------
+# Adhoc metric
+# ---------------------------------------------------------------------------
+
+
+def test_simple_metric_sum(dataset: MagicMock) -> None:
+ metric = adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "total_amount",
+ "aggregate": "SUM",
+ "column": {"column_name": "amount"},
+ },
+ dataset,
+ )
+ assert metric.id == "total_amount"
+ assert metric.name == "total_amount"
+ assert metric.definition == 'SUM("amount")'
+ assert metric.aggregation == AggregationType.SUM
+ assert metric.type == pa.null()
+
+
+def test_simple_metric_count_distinct(dataset: MagicMock) -> None:
+ metric = adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "unique_customers",
+ "aggregate": "COUNT_DISTINCT",
+ "column": {"column_name": "customer_id"},
+ },
+ dataset,
+ )
+ assert metric.definition == 'COUNT(DISTINCT "customer_id")'
+ assert metric.aggregation == AggregationType.COUNT_DISTINCT
+
+
+def test_simple_metric_unknown_aggregate_falls_back_to_other(
+ dataset: MagicMock,
+) -> None:
+ metric = adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "median_x",
+ "aggregate": "MEDIAN",
+ "column": {"column_name": "x"},
+ },
+ dataset,
+ )
+ # MEDIAN isn't in the rollup-safe map.
+ assert metric.aggregation == AggregationType.OTHER
+
+
+def test_simple_metric_missing_aggregate_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="aggregate and a column"):
+ adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "bad",
+ "column": {"column_name": "amount"},
+ },
+ dataset,
+ )
+
+
+def test_simple_metric_missing_column_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="aggregate and a column"):
+ adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "bad",
+ "aggregate": "SUM",
+ "column": {},
+ },
+ dataset,
+ )
+
+
+def test_sql_metric(dataset: MagicMock) -> None:
+ metric = adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SQL",
+ "label": "profit_margin",
+ "sqlExpression": "SUM(revenue - cost) / SUM(revenue)",
+ },
+ dataset,
+ )
+ assert metric.definition == "SUM(revenue - cost) / SUM(revenue)"
+ assert metric.aggregation == AggregationType.OTHER
+
+
+def test_sql_metric_missing_expression_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="sqlExpression"):
+ adhoc_metric_to_semantic_metric(
+ {"expressionType": "SQL", "label": "bad"},
+ dataset,
+ )
+
+
+def test_metric_missing_label_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="missing a ``label``"):
+ adhoc_metric_to_semantic_metric(
+ {"expressionType": "SIMPLE", "aggregate": "SUM"},
+ dataset,
+ )
+
+
+def test_metric_unknown_expression_type_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="Unknown adhoc metric"):
+ adhoc_metric_to_semantic_metric(
+ {"label": "weird", "expressionType": "MYSTERY"},
+ dataset,
+ )
+
+
+def test_sql_metric_jinja_applied(dataset: MagicMock) -> None:
+ # ``_process_select_expression`` is where Jinja and safety validation
+ # live in the dataset model. We verify the helper is invoked.
+ dataset._process_select_expression = MagicMock(return_value="user_id = 42")
+ metric = adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SQL",
+ "label": "rendered",
+ "sqlExpression": "user_id = {{ current_user_id() }}",
+ },
+ dataset,
+ template_processor=MagicMock(),
+ )
+ assert metric.definition == "user_id = 42"
+ dataset._process_select_expression.assert_called_once()
+
+
+def test_sql_metric_empty_processed_raises(dataset: MagicMock) -> None:
+ dataset._process_select_expression = MagicMock(return_value=None)
+ with pytest.raises(QueryObjectValidationError, match="empty string"):
+ adhoc_metric_to_semantic_metric(
+ {
+ "expressionType": "SQL",
+ "label": "bad",
+ "sqlExpression": "{{ '' }}",
+ },
+ dataset,
+ template_processor=MagicMock(),
+ )
+
+
+# ---------------------------------------------------------------------------
+# Adhoc column
+# ---------------------------------------------------------------------------
+
+
+def test_adhoc_column_reference_uses_existing_dimension(dataset: MagicMock) -> None:
+ existing = Dimension(id="country", name="country", type=pa.utf8())
+ result = adhoc_column_to_semantic_dimension(
+ {
+ "label": "country",
+ "sqlExpression": "country",
+ "isColumnReference": True,
+ },
+ dataset,
+ {"country": existing},
+ )
+ assert result is existing
+
+
+def test_adhoc_column_synthesises_dimension(dataset: MagicMock) -> None:
+ dataset._process_select_expression = MagicMock(return_value="UPPER(country)")
+ result = adhoc_column_to_semantic_dimension(
+ {
+ "label": "upper_country",
+ "sqlExpression": "UPPER(country)",
+ },
+ dataset,
+ {},
+ template_processor=MagicMock(),
+ )
+ assert result.id == "upper_country"
+ assert result.name == "upper_country"
+ assert result.definition == "UPPER(country)"
+ assert result.type == pa.null()
+
+
+def test_adhoc_column_missing_label_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="``label``"):
+ adhoc_column_to_semantic_dimension(
+ {"sqlExpression": "x"},
+ dataset,
+ {},
+ )
+
+
+def test_adhoc_column_missing_sql_raises(dataset: MagicMock) -> None:
+ with pytest.raises(QueryObjectValidationError, match="``sqlExpression``"):
+ adhoc_column_to_semantic_dimension(
+ {"label": "x"},
+ dataset,
+ {},
+ )
+
+
+def test_adhoc_column_reference_falls_back_when_not_matching(
+ dataset: MagicMock,
+) -> None:
+ """
+ A column-reference adhoc whose sqlExpression doesn't match an existing
+ dimension is treated as a synthesized adhoc.
+ """
+ dataset._process_select_expression = MagicMock(return_value="ghost")
+ result = adhoc_column_to_semantic_dimension(
+ {
+ "label": "spooky",
+ "sqlExpression": "ghost",
+ "isColumnReference": True,
+ },
+ dataset,
+ {"country": Dimension(id="country", name="country", type=pa.utf8())},
+ template_processor=MagicMock(),
+ )
+ assert result.id == "spooky"
+ assert result.definition == "ghost"
diff --git a/tests/unit_tests/semantic_layers/mapper_test.py b/tests/unit_tests/semantic_layers/mapper_test.py
index 0cbb59c8430..faaf4b7f4aa 100644
--- a/tests/unit_tests/semantic_layers/mapper_test.py
+++ b/tests/unit_tests/semantic_layers/mapper_test.py
@@ -50,9 +50,14 @@ from superset.semantic_layers.mapper import (
_get_group_limit_filters,
_get_group_limit_from_query_object,
_get_order_from_query_object,
+ _get_template_processor,
_get_time_bounds,
_get_time_filter,
_normalize_column,
+ _resolve_dimension,
+ _resolve_metric,
+ _stamp_grain,
+ _validate_dimensions,
_validate_filters,
_validate_granularity,
_validate_group_limit,
@@ -1035,7 +1040,10 @@ def test_validate_query_object_undefined_metric_error(
columns=["order_date"],
)
- with pytest.raises(ValueError, match="All metrics must be defined"):
+ with pytest.raises(
+ ValueError,
+ match="Metric 'undefined_metric' is not defined in the Semantic View",
+ ):
validate_query_object(query_object)
@@ -1051,7 +1059,10 @@ def test_validate_query_object_undefined_dimension_error(
columns=["undefined_dimension"],
)
- with pytest.raises(ValueError, match="All dimensions must be defined"):
+ with pytest.raises(
+ ValueError,
+ match="Dimension 'undefined_dimension' is not defined in the Semantic View",
+ ):
validate_query_object(query_object)
@@ -1800,28 +1811,31 @@ def test_get_results_empty_requests(
def test_normalize_column_adhoc_not_in_dimensions() -> None:
"""
- Test _normalize_column raises error for AdhocColumn with sqlExpression not in dims.
+ Adhoc columns whose sqlExpression doesn't match an existing dimension fall
+ back to using the label as the resolved name. Actual SQL synthesis is the
+ job of _resolve_dimension; _normalize_column only surfaces a name.
"""
dimension_names = {"category", "region"}
adhoc_column: AdhocColumn = {
+ "label": "custom_dim",
"isColumnReference": True,
"sqlExpression": "unknown_dimension",
}
- with pytest.raises(ValueError, match="Adhoc dimensions are not supported"):
- _normalize_column(adhoc_column, dimension_names)
+ assert _normalize_column(adhoc_column, dimension_names) == "custom_dim"
-def test_normalize_column_adhoc_missing_sql_expression() -> None:
+def test_normalize_column_adhoc_missing_label_raises() -> None:
"""
- Test _normalize_column raises error for AdhocColumn without sqlExpression.
+ When neither a matching column reference nor a label is provided there's
+ no resolvable name and _normalize_column raises.
"""
dimension_names = {"category", "region"}
adhoc_column: AdhocColumn = {
"isColumnReference": True,
}
- with pytest.raises(ValueError, match="Adhoc dimensions are not supported"):
+ with pytest.raises(ValueError, match="Adhoc column is missing"):
_normalize_column(adhoc_column, dimension_names)
@@ -2265,11 +2279,12 @@ def test_validate_query_object_no_datasource() -> None:
assert result is False
-def test_validate_metrics_adhoc_error(
+def test_validate_metrics_adhoc_with_bad_shape_raises(
mocker: MockerFixture,
) -> None:
"""
- Test validation error for adhoc metrics.
+ Adhoc metrics are now supported, but invalid shapes (missing
+ expressionType, etc.) still raise via the adhoc resolver.
"""
mock_datasource = mocker.Mock()
category_dim = Dimension("category", "category", pa.utf8(), "category", "Category")
@@ -2279,13 +2294,15 @@ def test_validate_metrics_adhoc_error(
mock_datasource.implementation.dimensions = {category_dim}
mock_datasource.implementation.metrics = {sales_metric}
+ # Strip the template processor; we just want to verify the shape check.
+ mock_datasource.get_template_processor.return_value = None
- # Manually create a query object with an adhoc metric
query_object = mocker.Mock()
query_object.datasource = mock_datasource
+ # Missing expressionType — the resolver doesn't know how to interpret this.
query_object.metrics = [{"label": "adhoc", "sqlExpression": "SUM(x)"}]
- with pytest.raises(ValueError, match="Adhoc metrics are not supported"):
+ with pytest.raises(Exception, match="expressionType"):
_validate_metrics(query_object)
@@ -3111,3 +3128,171 @@ def test_coerce_time_invalid_string_raises() -> None:
def test_coerce_time_rejects_other_types() -> None:
with pytest.raises(ValueError, match="Invalid time value"):
_coerce_scalar_filter_value(123, _dim(pa.time64("us")))
+
+
+# ---------------------------------------------------------------------------
+# Adhoc + grain resolver coverage
+# ---------------------------------------------------------------------------
+
+
+def test_get_template_processor_returns_none_when_unsupported() -> None:
+ """A bare object without ``get_template_processor`` returns None."""
+
+ class Plain:
+ pass
+
+ assert _get_template_processor(Plain()) is None
+
+
+def test_stamp_grain_returns_new_dimension_with_grain() -> None:
+ dim = Dimension(id="dt", name="dt", type=pa.timestamp("us"))
+ stamped = _stamp_grain(dim, Grains.DAY)
+ assert stamped.grain == Grains.DAY
+ # Original is unchanged (frozen dataclass).
+ assert dim.grain is None
+
+
+def _adhoc_dataset() -> MagicMock:
+ ds = MagicMock()
+ ds.database_id = 1
+ ds.schema = "public"
+ ds.database.db_engine_spec.engine = "postgresql"
+ ds.quote_identifier = lambda name: f'"{name}"'
+ ds._process_select_expression = lambda expression, **kwargs: expression # noqa: U100
+ ds.get_template_processor.return_value = None
+ return ds
+
+
+def test_resolve_metric_for_adhoc_simple_dict() -> None:
+ ds = _adhoc_dataset()
+ metric = _resolve_metric(
+ {
+ "expressionType": "SIMPLE",
+ "label": "total",
+ "aggregate": "SUM",
+ "column": {"column_name": "x"},
+ },
+ ds,
+ {},
+ None,
+ )
+ assert metric.id == "total"
+ assert metric.definition == 'SUM("x")'
+
+
+def test_resolve_dimension_for_adhoc_dict() -> None:
+ ds = _adhoc_dataset()
+ dim = _resolve_dimension(
+ {"label": "calc", "sqlExpression": "UPPER(country)"},
+ ds,
+ {},
+ None,
+ )
+ assert dim.id == "calc"
+ assert dim.definition == "UPPER(country)"
+
+
+def test_validate_metrics_duplicate_label_raises() -> None:
+ ds = _adhoc_dataset()
+ view = MagicMock()
+ view.metrics = []
+ view.dimensions = []
+ ds.implementation = view
+
+ query = MagicMock()
+ query.datasource = ds
+ query.metrics = [
+ {
+ "expressionType": "SIMPLE",
+ "label": "dup",
+ "aggregate": "SUM",
+ "column": {"column_name": "x"},
+ },
+ {
+ "expressionType": "SIMPLE",
+ "label": "dup",
+ "aggregate": "SUM",
+ "column": {"column_name": "y"},
+ },
+ ]
+
+ with pytest.raises(ValueError, match="Duplicate metric labels"):
+ _validate_metrics(query)
+
+
+def test_validate_dimensions_duplicate_label_raises() -> None:
+ ds = _adhoc_dataset()
+ view = MagicMock()
+ view.metrics = []
+ view.dimensions = []
+ ds.implementation = view
+
+ query = MagicMock()
+ query.datasource = ds
+ query.columns = [
+ {"label": "dup", "sqlExpression": "x"},
+ {"label": "dup", "sqlExpression": "y"},
+ ]
+
+ with pytest.raises(ValueError, match="Duplicate column labels"):
+ _validate_dimensions(query)
+
+
+def test_map_query_object_dedups_dimensions_by_id(mocker: MockerFixture) -> None:
+ """Two requested columns resolving to the same dim id collapse to one."""
+ ds = _adhoc_dataset()
+ existing = Dimension(id="country", name="country", type=pa.utf8())
+ view = MagicMock()
+ view.metrics = []
+ view.dimensions = [existing]
+ ds.implementation = view
+ ds.fetch_values_predicate = None
+
+ query = ValidatedQueryObject(
+ datasource=ds,
+ metrics=[],
+ # Same dim referenced twice — once as a string, once as a column ref.
+ columns=[
+ "country",
+ {"label": "country", "sqlExpression": "country", "isColumnReference": True},
+ ],
+ )
+
+ sem_queries = map_query_object(query)
+ assert len(sem_queries[0].dimensions) == 1
+
+
+def test_map_query_object_stamps_grain_on_granularity_dim(
+ mocker: MockerFixture,
+) -> None:
+ """A grain request stamps the grain onto the matching dimension."""
+ ds = _adhoc_dataset()
+ dt_dim = Dimension(id="dt", name="dt", type=pa.timestamp("us"))
+ view = MagicMock()
+ view.metrics = []
+ view.dimensions = [dt_dim]
+ ds.implementation = view
+ ds.fetch_values_predicate = None
+
+ query = ValidatedQueryObject(
+ datasource=ds,
+ metrics=[],
+ columns=["dt"],
+ granularity="dt",
+ extras={"time_grain_sqla": "P1D"},
+ )
+
+ sem_queries = map_query_object(query)
+ dim = sem_queries[0].dimensions[0]
+ assert dim.grain == Grains.DAY
+
+
+def test_normalize_column_adhoc_label_only() -> None:
+ """Adhoc with no isColumnReference falls back to its label."""
+ assert (
+ _normalize_column({"label": "calc", "sqlExpression": "x"}, set()) == "calc"
+ )
+
+
+def test_normalize_column_string_passthrough() -> None:
+ assert _normalize_column("category", {"category", "region"}) == "category"
diff --git a/tests/unit_tests/semantic_layers/rls_test.py b/tests/unit_tests/semantic_layers/rls_test.py
new file mode 100644
index 00000000000..0f3871711b8
--- /dev/null
+++ b/tests/unit_tests/semantic_layers/rls_test.py
@@ -0,0 +1,166 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=invalid-name
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+import pytest
+from pytest_mock import MockerFixture
+
+from superset.semantic_layers.rls import (
+ apply_rls_to_virtual_sql,
+ get_rls_method,
+ render_rls_predicates,
+)
+from superset.sql.parse import RLSMethod
+
+
+def _make_dataset(
+ *,
+ sql: str | None = None,
+ rls_clauses: list | None = None,
+) -> MagicMock:
+ ds = MagicMock()
+ ds.id = 1
+ ds.catalog = None
+ ds.schema = "public"
+ ds.sql = sql
+ ds.database.db_engine_spec.engine = "postgresql"
+ ds.database.get_default_schema.return_value = "public"
+ ds.get_sqla_row_level_filters.return_value = rls_clauses or []
+ return ds
+
+
+def test_get_rls_method_delegates_to_engine_spec() -> None:
+ ds = _make_dataset()
+ ds.database.db_engine_spec.get_rls_method.return_value = RLSMethod.AS_SUBQUERY
+ assert get_rls_method(ds) == RLSMethod.AS_SUBQUERY
+
+
+def test_render_rls_predicates_empty_when_no_rules() -> None:
+ ds = _make_dataset()
+ assert render_rls_predicates(ds) == []
+
+
+def test_render_rls_predicates_compiles_each_clause() -> None:
+ clause1 = MagicMock()
+ clause1.compile.return_value = "tenant_id = 7"
+ clause2 = MagicMock()
+ clause2.compile.return_value = "deleted = false"
+ ds = _make_dataset(rls_clauses=[clause1, clause2])
+
+ result = render_rls_predicates(ds)
+ assert result == ["tenant_id = 7", "deleted = false"]
+ # Each clause is compiled with literal_binds so values are inlined.
+ for clause in (clause1, clause2):
+ kwargs = clause.compile.call_args.kwargs
+ assert kwargs["compile_kwargs"] == {"literal_binds": True}
+
+
+def test_apply_rls_to_virtual_sql_returns_none_for_physical(mocker: MockerFixture) -> None:
+ ds = _make_dataset()
+ assert apply_rls_to_virtual_sql(ds) is None
+
+
+def test_apply_rls_to_virtual_sql_returns_none_when_no_rls_applied(
+ mocker: MockerFixture,
+) -> None:
+ ds = _make_dataset(sql="SELECT * FROM raw_orders")
+ # apply_rls returns False → no predicates were injected.
+ mocker.patch("superset.semantic_layers.rls.apply_rls", return_value=False)
+ assert apply_rls_to_virtual_sql(ds) is None
+
+
+def test_apply_rls_to_virtual_sql_returns_rewritten_when_applied(
+ mocker: MockerFixture,
+) -> None:
+ ds = _make_dataset(sql="SELECT * FROM raw_orders")
+ fake_script = MagicMock()
+ fake_statement = MagicMock()
+ fake_script.statements = [fake_statement]
+ fake_script.format.return_value = "SELECT * FROM raw_orders WHERE x = 1"
+ mocker.patch(
+ "superset.semantic_layers.rls.SQLScript",
+ return_value=fake_script,
+ )
+ mocker.patch("superset.semantic_layers.rls.apply_rls", return_value=True)
+
+ result = apply_rls_to_virtual_sql(ds)
+ assert result == "SELECT * FROM raw_orders WHERE x = 1"
+
+
+def test_apply_rls_to_virtual_sql_swallows_parse_error(mocker: MockerFixture) -> None:
+ ds = _make_dataset(sql="totally invalid sql")
+ mocker.patch(
+ "superset.semantic_layers.rls.SQLScript",
+ side_effect=Exception("parse error"),
+ )
+ assert apply_rls_to_virtual_sql(ds) is None
+
+
+def test_apply_rls_to_virtual_sql_swallows_apply_error(mocker: MockerFixture) -> None:
+ ds = _make_dataset(sql="SELECT * FROM raw_orders")
+ fake_script = MagicMock()
+ fake_script.statements = [MagicMock()]
+ mocker.patch(
+ "superset.semantic_layers.rls.SQLScript",
+ return_value=fake_script,
+ )
+ mocker.patch(
+ "superset.semantic_layers.rls.apply_rls",
+ side_effect=Exception("boom"),
+ )
+ assert apply_rls_to_virtual_sql(ds) is None
+
+
+def test_apply_rls_to_virtual_sql_uses_default_schema_when_dataset_schema_missing(
+ mocker: MockerFixture,
+) -> None:
+ ds = _make_dataset(sql="SELECT * FROM raw_orders")
+ ds.schema = None
+ fake_script = MagicMock()
+ fake_statement = MagicMock()
+ fake_script.statements = [fake_statement]
+ mocker.patch(
+ "superset.semantic_layers.rls.SQLScript",
+ return_value=fake_script,
+ )
+ apply_rls_mock = mocker.patch(
+ "superset.semantic_layers.rls.apply_rls",
+ return_value=False,
+ )
+
+ apply_rls_to_virtual_sql(ds)
+ # default_schema was "public" from get_default_schema; that should be the
+ # schema arg passed to apply_rls.
+ args, kwargs = apply_rls_mock.call_args
+ assert args[2] == "public" or kwargs.get("schema") == "public"
+
+
+def test_render_rls_predicates_uses_dialect_for_compile() -> None:
+ clause = MagicMock()
+ clause.compile.return_value = "x = 1"
+ ds = _make_dataset(rls_clauses=[clause])
+
+ render_rls_predicates(ds)
+ kwargs = clause.compile.call_args.kwargs
+ # Verify the dialect from the database is passed (not asserting identity,
+ # just that ``dialect=`` got a value).
+ assert "dialect" in kwargs