You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2026/06/26 16:27:52 UTC
(superset) branch sl-fix-time-grain created (now 31199ae20e4)
This is an automated email from the ASF dual-hosted git repository.
betodealmeida pushed a change to branch sl-fix-time-grain
in repository https://gitbox.apache.org/repos/asf/superset.git
at 31199ae20e4 fix(semantic layers): improve time grain
This branch includes the following new commits:
new 31199ae20e4 fix(semantic layers): improve time grain
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(superset) 01/01: fix(semantic layers): improve time grain
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
betodealmeida pushed a commit to branch sl-fix-time-grain
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 31199ae20e4a274659f7cd0a48072649d0c65167
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Fri Jun 26 12:03:15 2026 -0400
fix(semantic layers): improve time grain
---
.../explore/controlUtils/getSectionsToRender.ts | 6 +-
superset/semantic_layers/mapper.py | 84 ++++++++++++----
superset/semantic_layers/models.py | 70 +++++++++----
tests/unit_tests/semantic_layers/mapper_test.py | 90 +++++++++++++++++
tests/unit_tests/semantic_layers/models_test.py | 112 +++++++++++++++++++++
5 files changed, 323 insertions(+), 39 deletions(-)
diff --git a/superset-frontend/src/explore/controlUtils/getSectionsToRender.ts b/superset-frontend/src/explore/controlUtils/getSectionsToRender.ts
index 4185a631163..0135444c148 100644
--- a/superset-frontend/src/explore/controlUtils/getSectionsToRender.ts
+++ b/superset-frontend/src/explore/controlUtils/getSectionsToRender.ts
@@ -67,7 +67,11 @@ const getMemoizedSectionsToRender = memoizeOne(
} = sections;
// list of datasource-specific controls that should be removed if the datasource is a specific type
- const filterControlsForTypes = [DatasourceType.Query, DatasourceType.Table];
+ const filterControlsForTypes = [
+ DatasourceType.Query,
+ DatasourceType.Table,
+ DatasourceType.SemanticView,
+ ];
const invalidControls = filterControlsForTypes.includes(datasourceType)
? ['granularity']
: ['granularity_sqla', 'time_grain_sqla'];
diff --git a/superset/semantic_layers/mapper.py b/superset/semantic_layers/mapper.py
index e40629c3ce2..5fdafe6a5bd 100644
--- a/superset/semantic_layers/mapper.py
+++ b/superset/semantic_layers/mapper.py
@@ -294,18 +294,26 @@ def map_query_object(query_object: ValidatedQueryObject) -> list[SemanticQuery]:
metrics = [all_metrics[metric] for metric in (query_object.metrics or [])]
grain = _convert_time_grain(query_object.extras.get("time_grain_sqla"))
- dimensions = [
- dimension
- for dimension in semantic_view.dimensions
- if dimension.name in normalized_columns
- and (
- # if a grain is specified, only include the time dimension if its grain
- # matches the requested grain
- grain is None
- or dimension.name != query_object.granularity
- or dimension.grain == grain
- )
- ]
+ time_axis_column = _get_time_axis_column(query_object, all_dimensions)
+ # A semantic view can expose multiple Dimension variants per name (one per
+ # supported time grain). Pick exactly one variant per selected column:
+ # for the time-axis column we honor the user's grain selection (falling
+ # back to the raw / no-grain variant when none was chosen); for every
+ # other selected column we prefer the raw variant and otherwise take any
+ # available variant.
+ dimensions: list[Dimension] = []
+ seen_non_axis: dict[str, Dimension] = {}
+ for dimension in semantic_view.dimensions:
+ if dimension.name not in normalized_columns:
+ continue
+ if dimension.name == time_axis_column:
+ if dimension.grain == grain:
+ dimensions.append(dimension)
+ continue
+ existing = seen_non_axis.get(dimension.name)
+ if existing is None or (existing.grain is not None and dimension.grain is None):
+ seen_non_axis[dimension.name] = dimension
+ dimensions.extend(seen_non_axis.values())
order = _get_order_from_query_object(query_object, all_metrics, all_dimensions)
limit = query_object.row_limit
@@ -889,6 +897,41 @@ def _get_group_limit_filters(
return filters if filters else None
+def _get_time_axis_column(
+ query_object: ValidatedQueryObject,
+ all_dimensions: dict[str, Dimension],
+) -> str | None:
+ """
+ Determine which selected column is the time-axis (the one a time grain
+ applies to).
+
+ Legacy time-series charts encode this as ``query_object.granularity``;
+ modern x-axis charts leave that empty and put the temporal column in
+ ``query_object.columns`` instead, with the grain on
+ ``extras["time_grain_sqla"]``. In that case we pick the first temporal
+ dimension from the normalized column list.
+ """
+ if query_object.granularity:
+ return query_object.granularity
+
+ dimension_names = set(all_dimensions.keys())
+ for column in query_object.columns or []:
+ try:
+ name = _normalize_column(column, dimension_names)
+ except ValueError:
+ continue
+ dim = all_dimensions.get(name)
+ if dim is None:
+ continue
+ if (
+ pa.types.is_timestamp(dim.type)
+ or pa.types.is_date(dim.type)
+ or pa.types.is_time(dim.type)
+ ):
+ return name
+ return None
+
+
def _convert_time_grain(time_grain: str | None) -> Grain | None:
"""
Convert a time grain string (ISO 8601 duration) to a Grain instance.
@@ -975,15 +1018,20 @@ def _validate_granularity(query_object: ValidatedQueryObject) -> None:
Make sure time column and time grain are valid.
"""
semantic_view = query_object.datasource.implementation
- dimension_names = {dimension.name for dimension in semantic_view.dimensions}
+ all_dimensions = {
+ dimension.name: dimension for dimension in semantic_view.dimensions
+ }
+ dimension_names = set(all_dimensions.keys())
- if time_column := query_object.granularity:
- if time_column not in dimension_names:
- raise ValueError(
- "The time column must be defined in the Semantic View dimensions."
- )
+ if (legacy_time_column := query_object.granularity) and (
+ legacy_time_column not in dimension_names
+ ):
+ raise ValueError(
+ "The time column must be defined in the Semantic View dimensions."
+ )
if time_grain := query_object.extras.get("time_grain_sqla"):
+ time_column = _get_time_axis_column(query_object, all_dimensions)
if not time_column:
raise ValueError(
"A time column must be specified when a time grain is provided."
diff --git a/superset/semantic_layers/models.py b/superset/semantic_layers/models.py
index 34cdfbc4792..a0684fc0e30 100644
--- a/superset/semantic_layers/models.py
+++ b/superset/semantic_layers/models.py
@@ -313,8 +313,27 @@ class SemanticView(AuditMixinNullable, Model):
for metric in self.implementation.get_metrics()
]
+ @property
+ def _unique_dimensions(self) -> list[Any]:
+ # A semantic view may expose multiple ``Dimension`` objects sharing the
+ # same ``name`` but different grains (one variant per supported time
+ # grain). For column-list purposes we collapse these into a single
+ # entry; the available grains are surfaced separately via
+ # ``get_time_grains`` and ``data["time_grain_sqla"]``.
+ seen: dict[str, Any] = {}
+ for dimension in self.implementation.get_dimensions():
+ seen.setdefault(dimension.name, dimension)
+ return list(seen.values())
+
@property
def columns(self) -> list[ColumnMetadata]:
+ # ``expression`` is intentionally left unset: the explore UI uses a
+ # non-empty ``expression`` to mean "this column is a SQL-style adhoc
+ # expression" and renders it with the fx icon and no time-grain
+ # affordance. Semantic-view dimensions are physical from the UI's
+ # perspective; the backend is responsible for any underlying
+ # expression. ``dimension.definition`` is preserved on
+ # ``description`` instead, where applicable.
return [
ColumnMetadata(
column_name=dimension.name,
@@ -323,17 +342,17 @@ class SemanticView(AuditMixinNullable, Model):
or pa.types.is_time(dimension.type)
or pa.types.is_timestamp(dimension.type),
description=dimension.description,
- expression=dimension.definition,
+ expression=None,
extra=json.dumps(
{"grain": dimension.grain.name if dimension.grain else None}
),
)
- for dimension in self.implementation.get_dimensions()
+ for dimension in self._unique_dimensions
]
@property
def column_names(self) -> list[str]:
- return [dimension.name for dimension in self.implementation.get_dimensions()]
+ return [dimension.name for dimension in self._unique_dimensions]
@property
def data(self) -> ExplorableData:
@@ -350,7 +369,9 @@ class SemanticView(AuditMixinNullable, Model):
"certified_by": None,
"column_name": dimension.name,
"description": dimension.description,
- "expression": dimension.definition,
+ # See ``columns`` property: leaving ``expression`` empty
+ # avoids the fx-icon / adhoc treatment in the explore UI.
+ "expression": None,
"filterable": True,
"groupby": True,
"id": None,
@@ -365,7 +386,7 @@ class SemanticView(AuditMixinNullable, Model):
"verbose_name": None,
"warning_markdown": None,
}
- for dimension in self.implementation.get_dimensions()
+ for dimension in self._unique_dimensions
],
"metrics": [
{
@@ -397,12 +418,9 @@ class SemanticView(AuditMixinNullable, Model):
"description": self.description,
"table_name": self.name,
"column_types": [
- get_column_type(dimension.type)
- for dimension in self.implementation.get_dimensions()
- ],
- "column_names": [
- dimension.name for dimension in self.implementation.get_dimensions()
+ get_column_type(dimension.type) for dimension in self._unique_dimensions
],
+ "column_names": [dimension.name for dimension in self._unique_dimensions],
# rare
"column_formats": {},
"datasource_name": self.name,
@@ -414,7 +432,12 @@ class SemanticView(AuditMixinNullable, Model):
"schema": None,
"catalog": None,
"main_dttm_col": None,
- "time_grain_sqla": [],
+ # ``time_grain_sqla`` in ``ExplorableData`` is the ``(duration,
+ # name)`` tuple shape the explore UI consumes; the dict shape
+ # lives on ``get_time_grains``.
+ "time_grain_sqla": [
+ (grain["duration"], grain["name"]) for grain in self.get_time_grains()
+ ],
"granularity_sqla": [],
"fetch_values_predicate": None,
"template_params": None,
@@ -460,15 +483,22 @@ class SemanticView(AuditMixinNullable, Model):
return 0
def get_time_grains(self) -> list[TimeGrainDict]:
- return [
- {
- "name": dimension.grain.name,
- "function": "",
- "duration": dimension.grain.representation,
- }
- for dimension in self.implementation.get_dimensions()
- if dimension.grain
- ]
+ # Return the union of grains across all time dimensions, deduped by
+ # ISO duration so the explore-time-grain dropdown shows each grain
+ # once even when multiple time columns expose the same set.
+ seen: dict[str, TimeGrainDict] = {}
+ for dimension in self.implementation.get_dimensions():
+ if not dimension.grain:
+ continue
+ seen.setdefault(
+ dimension.grain.representation,
+ {
+ "name": dimension.grain.name,
+ "function": "",
+ "duration": dimension.grain.representation,
+ },
+ )
+ return list(seen.values())
def has_drill_by_columns(self, column_names: list[str]) -> bool:
dimension_names = {
diff --git a/tests/unit_tests/semantic_layers/mapper_test.py b/tests/unit_tests/semantic_layers/mapper_test.py
index 0cbb59c8430..5eb5fc4d209 100644
--- a/tests/unit_tests/semantic_layers/mapper_test.py
+++ b/tests/unit_tests/semantic_layers/mapper_test.py
@@ -1003,6 +1003,96 @@ def test_map_query_object_with_time_offsets(mock_datasource: MagicMock) -> None:
}
+def _make_grain_variant_datasource(
+ mocker: MockerFixture,
+ granularity_dim_grain: Grain | None,
+ extra_dim_grain: Grain | None = None,
+) -> MagicMock:
+ """Datasource with raw + Hour + Day variants on ``order_date``."""
+ datasource = mocker.Mock()
+ base = {
+ "id": "orders.order_date",
+ "name": "order_date",
+ "type": pa.timestamp("us"),
+ "description": "Order date",
+ "definition": "order_date",
+ }
+ date_variants = {
+ Dimension(**base, grain=None),
+ Dimension(**base, grain=Grains.HOUR),
+ Dimension(**base, grain=Grains.DAY),
+ }
+ category = Dimension(
+ id="products.category",
+ name="category",
+ type=pa.utf8(),
+ description="Product category",
+ definition="category",
+ )
+ sales = Metric(
+ id="orders.total_sales",
+ name="total_sales",
+ type=pa.float64(),
+ definition="SUM(amount)",
+ description="Total sales",
+ )
+ implementation = MockSemanticView(
+ dimensions=date_variants | {category},
+ metrics={sales},
+ features=frozenset(),
+ )
+ datasource.implementation = implementation
+ datasource.fetch_values_predicate = None
+ return datasource
+
+
+def test_map_query_object_picks_grain_variant_matching_user_selection(
+ mocker: MockerFixture,
+) -> None:
+ """Only the variant matching the user's grain is sent through."""
+ datasource = _make_grain_variant_datasource(
+ mocker, granularity_dim_grain=Grains.DAY
+ )
+ query_object = ValidatedQueryObject(
+ datasource=datasource,
+ metrics=["total_sales"],
+ columns=["category", "order_date"],
+ granularity="order_date",
+ extras={"time_grain_sqla": "P1D"},
+ )
+
+ result = map_query_object(query_object)
+
+ selected_grains = {dim.grain for dim in result[0].dimensions}
+ assert selected_grains == {
+ Grains.DAY,
+ None,
+ } # day for order_date, None for category
+
+
+def test_map_query_object_picks_raw_variant_when_no_grain_selected(
+ mocker: MockerFixture,
+) -> None:
+ """
+ No grain selected — the time-axis column must collapse to the raw (grain=None)
+ variant rather than passing all grain variants through to the semantic view.
+ """
+ datasource = _make_grain_variant_datasource(mocker, granularity_dim_grain=None)
+ query_object = ValidatedQueryObject(
+ datasource=datasource,
+ metrics=["total_sales"],
+ columns=["category", "order_date"],
+ granularity="order_date",
+ # No time_grain_sqla in extras
+ )
+
+ result = map_query_object(query_object)
+
+ order_date_dims = [dim for dim in result[0].dimensions if dim.name == "order_date"]
+ assert len(order_date_dims) == 1
+ assert order_date_dims[0].grain is None
+
+
def test_convert_query_object_filter_unknown_operator(
mock_datasource: MagicMock,
) -> None:
diff --git a/tests/unit_tests/semantic_layers/models_test.py b/tests/unit_tests/semantic_layers/models_test.py
index dd8c1e67dd1..d70e70b0744 100644
--- a/tests/unit_tests/semantic_layers/models_test.py
+++ b/tests/unit_tests/semantic_layers/models_test.py
@@ -655,6 +655,118 @@ def test_semantic_view_data(
assert data["offset"] == 0
+@pytest.fixture
+def mock_grain_variant_dimensions() -> list[Dimension]:
+ """Time column exposed as multiple Dimension variants, one per grain."""
+ base = {
+ "id": "orders.created_at",
+ "name": "created_at",
+ "type": pa.timestamp("us"),
+ "definition": "orders.created_at",
+ "description": "Order timestamp",
+ }
+ return [
+ Dimension(**base, grain=Grains.HOUR),
+ Dimension(**base, grain=Grains.DAY),
+ Dimension(**base, grain=Grains.MONTH),
+ Dimension(
+ id="products.category",
+ name="category",
+ type=pa.utf8(),
+ definition="products.category",
+ description="Product category",
+ grain=None,
+ ),
+ ]
+
+
+def test_semantic_view_columns_dedupes_grain_variants(
+ mock_grain_variant_dimensions: list[Dimension],
+) -> None:
+ """Multiple grain variants of the same time column collapse to one column."""
+ impl = MagicMock()
+ impl.get_dimensions.return_value = mock_grain_variant_dimensions
+ view = SemanticView()
+
+ with patch.object(
+ SemanticView,
+ "implementation",
+ new_callable=lambda: property(lambda s: impl),
+ ):
+ columns = view.columns
+ assert [c.column_name for c in columns] == ["created_at", "category"]
+ assert columns[0].is_dttm is True
+ assert view.column_names == ["created_at", "category"]
+
+
+def test_semantic_view_get_time_grains_dedupes_across_dimensions(
+ mock_grain_variant_dimensions: list[Dimension],
+) -> None:
+ """Grains shared across multiple time dimensions are returned once each."""
+ extra_dim = Dimension(
+ id="shipments.shipped_at",
+ name="shipped_at",
+ type=pa.timestamp("us"),
+ definition="shipments.shipped_at",
+ description=None,
+ grain=Grains.DAY,
+ )
+ impl = MagicMock()
+ impl.get_dimensions.return_value = mock_grain_variant_dimensions + [extra_dim]
+ view = SemanticView()
+
+ with patch.object(
+ SemanticView,
+ "implementation",
+ new_callable=lambda: property(lambda s: impl),
+ ):
+ grains = view.get_time_grains()
+
+ durations = sorted(grain["duration"] or "" for grain in grains)
+ assert durations == sorted(["PT1H", "P1D", "P1M"])
+
+
+def test_semantic_view_data_populates_time_grain_sqla(
+ mock_grain_variant_dimensions: list[Dimension],
+ mock_metrics: list[Metric],
+) -> None:
+ """``data['time_grain_sqla']`` mirrors ``get_time_grains`` for the explore UI."""
+ from superset.semantic_layers.models import SemanticLayer
+
+ impl = MagicMock()
+ impl.get_dimensions.return_value = mock_grain_variant_dimensions
+ impl.get_metrics.return_value = mock_metrics
+ impl.uid.return_value = "semantic_view_uid_123"
+
+ layer = SemanticLayer()
+ layer.name = "My Semantic Layer"
+ layer.uuid = uuid.UUID("87654321-4321-8765-4321-876543218765")
+ layer.perm = "[My Semantic Layer](id:87654321432187654321876543218765)"
+
+ view = SemanticView()
+ view.name = "Orders View"
+ view.description = "View of order data"
+ view.id = 1
+ view.uuid = uuid.UUID("12345678-1234-5678-1234-567812345678")
+ view.semantic_layer_uuid = uuid.UUID("87654321-4321-8765-4321-876543218765")
+ view.semantic_layer = layer
+ view.cache_timeout = 3600
+
+ with patch.object(
+ SemanticView,
+ "implementation",
+ new_callable=lambda: property(lambda s: impl),
+ ):
+ data = view.data
+
+ assert data["column_names"] == ["created_at", "category"]
+ assert len(data["columns"]) == 2
+ assert data["columns"][0]["is_dttm"] is True
+ # ``time_grain_sqla`` in ExplorableData is ``(duration, name)`` tuples.
+ grain_durations = sorted(entry[0] for entry in data["time_grain_sqla"])
+ assert grain_durations == sorted(["PT1H", "P1D", "P1M"])
+
+
def test_semantic_view_get_query_result(
mock_implementation: MagicMock,
) -> None: