You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2025/12/10 21:29:14 UTC
(superset) branch semantic-layer-implementation updated (8a0aaa42ec -> 2833b69ca0)
This is an automated email from the ASF dual-hosted git repository.
beto pushed a change to branch semantic-layer-implementation
in repository https://gitbox.apache.org/repos/asf/superset.git
from 8a0aaa42ec feat: semantic layer implementation (Snowflake)
new 6e17714a19 WIP
new 2833b69ca0 WIP
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
setup.py | 7 +++-
superset/commands/explore/get.py | 6 ++-
superset/commands/explore/parameters.py | 4 +-
superset/daos/datasource.py | 2 +
superset/explore/api.py | 2 +-
...6_33d7e0e21daa_add_semantic_layers_and_views.py | 6 ++-
superset/semantic_layers/mapper.py | 35 +++++++++++-----
superset/semantic_layers/models.py | 25 ++++++-----
.../semantic_layers/snowflake/semantic_layer.py | 48 +++++++++++++++++++---
superset/utils/core.py | 1 +
superset/views/utils.py | 13 ++++--
11 files changed, 113 insertions(+), 36 deletions(-)
(superset) 01/02: WIP
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch semantic-layer-implementation
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 6e17714a19e7fac72ad78820590b6ce57fb03f89
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Wed Dec 10 13:54:19 2025 -0500
WIP
---
setup.py | 7 ++++-
superset/commands/explore/parameters.py | 4 +--
superset/daos/datasource.py | 2 ++
superset/explore/api.py | 2 +-
...6_33d7e0e21daa_add_semantic_layers_and_views.py | 6 ++--
superset/semantic_layers/mapper.py | 35 +++++++++++++++------
superset/semantic_layers/models.py | 5 +--
.../semantic_layers/snowflake/semantic_layer.py | 36 +++++++++++++++++++++-
superset/utils/core.py | 1 +
superset/views/utils.py | 13 ++++++--
10 files changed, 89 insertions(+), 22 deletions(-)
diff --git a/setup.py b/setup.py
index 9b834cc241..6a62c552fa 100644
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,9 @@ with open(PACKAGE_JSON) as package_file:
def get_git_sha() -> str:
try:
- output = subprocess.check_output(["git", "rev-parse", "HEAD"]) # noqa: S603, S607
+ output = subprocess.check_output(
+ ["git", "rev-parse", "HEAD"]
+ ) # noqa: S603, S607
return output.decode().strip()
except Exception: # pylint: disable=broad-except
return ""
@@ -58,6 +60,9 @@ setup(
include_package_data=True,
zip_safe=False,
entry_points={
+ "superset.semantic_layers": [
+ "snowflake = superset.semantic_layers.snowflake:SnowflakeSemanticLayer"
+ ],
"console_scripts": ["superset=superset.cli.main:superset"],
# the `postgres` and `postgres+psycopg2://` schemes were removed in SQLAlchemy 1.4 # noqa: E501
# add an alias here to prevent breaking existing databases
diff --git a/superset/commands/explore/parameters.py b/superset/commands/explore/parameters.py
index 1aa5418d62..d5457e102e 100644
--- a/superset/commands/explore/parameters.py
+++ b/superset/commands/explore/parameters.py
@@ -15,13 +15,13 @@
# specific language governing permissions and limitations
# under the License.
from dataclasses import dataclass
-from typing import Optional
+from typing import Optional, Union
@dataclass
class CommandParameters:
permalink_key: Optional[str]
form_data_key: Optional[str]
- datasource_id: Optional[int]
+ datasource_id: Optional[Union[int, str]]
datasource_type: Optional[str]
slice_id: Optional[int]
diff --git a/superset/daos/datasource.py b/superset/daos/datasource.py
index 308785f625..4bacaa3229 100644
--- a/superset/daos/datasource.py
+++ b/superset/daos/datasource.py
@@ -28,6 +28,7 @@ from superset.daos.exceptions import (
DatasourceValueIsIncorrect,
)
from superset.models.sql_lab import Query, SavedQuery
+from superset.semantic_layers.models import SemanticView
from superset.utils.core import DatasourceType
logger = logging.getLogger(__name__)
@@ -40,6 +41,7 @@ class DatasourceDAO(BaseDAO[Datasource]):
DatasourceType.TABLE: SqlaTable,
DatasourceType.QUERY: Query,
DatasourceType.SAVEDQUERY: SavedQuery,
+ DatasourceType.SEMANTIC_VIEW: SemanticView,
}
@classmethod
diff --git a/superset/explore/api.py b/superset/explore/api.py
index e16b083feb..a9b66d468c 100644
--- a/superset/explore/api.py
+++ b/superset/explore/api.py
@@ -109,7 +109,7 @@ class ExploreRestApi(BaseSupersetApi):
params = CommandParameters(
permalink_key=request.args.get("permalink_key", type=str),
form_data_key=request.args.get("form_data_key", type=str),
- datasource_id=request.args.get("datasource_id", type=int),
+ datasource_id=request.args.get("datasource_id"),
datasource_type=request.args.get("datasource_type", type=str),
slice_id=request.args.get("slice_id", type=int),
)
diff --git a/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py b/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
index 6cac30f3fd..c8e73e2cee 100644
--- a/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
+++ b/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
@@ -25,6 +25,7 @@ Create Date: 2025-11-04 11:26:00.000000
import uuid
import sqlalchemy as sa
+from sqlalchemy.types import JSON
from sqlalchemy_utils import UUIDType
from superset.extensions import encrypted_field_factory
@@ -51,7 +52,7 @@ def upgrade():
sa.Column("type", sa.String(length=250), nullable=False),
sa.Column(
"configuration",
- encrypted_field_factory.create(sa.Text),
+ encrypted_field_factory.create(JSON),
nullable=True,
),
sa.Column("cache_timeout", sa.Integer(), nullable=True),
@@ -84,9 +85,10 @@ def upgrade():
sa.Column("created_on", sa.DateTime(), nullable=True),
sa.Column("changed_on", sa.DateTime(), nullable=True),
sa.Column("name", sa.String(length=250), nullable=False),
+ sa.Column("description", sa.Text(), nullable=True),
sa.Column(
"configuration",
- encrypted_field_factory.create(sa.Text),
+ encrypted_field_factory.create(JSON),
nullable=True,
),
sa.Column("cache_timeout", sa.Integer(), nullable=True),
diff --git a/superset/semantic_layers/mapper.py b/superset/semantic_layers/mapper.py
index a9ee68f8f8..6fe6ffc16e 100644
--- a/superset/semantic_layers/mapper.py
+++ b/superset/semantic_layers/mapper.py
@@ -38,20 +38,27 @@ from superset.models.helpers import QueryResult
from superset.semantic_layers.types import (
AdhocExpression,
AdhocFilter,
- DateGrain,
+ Day,
Dimension,
Filter,
FilterValues,
+ Grain,
GroupLimit,
+ Hour,
Metric,
+ Minute,
+ Month,
Operator,
OrderDirection,
OrderTuple,
PredicateType,
+ Quarter,
+ Second,
SemanticQuery,
SemanticResult,
SemanticViewFeature,
- TimeGrain,
+ Week,
+ Year,
)
from superset.utils.core import (
FilterOperator,
@@ -696,17 +703,25 @@ def _get_group_limit_filters(
return filters if filters else None
-def _convert_time_grain(time_grain: str) -> TimeGrain | DateGrain | None:
+def _convert_time_grain(time_grain: str) -> Grain | None:
"""
- Convert a time grain string from the query object to a TimeGrain or DateGrain enum.
+ Convert a time grain string from the query object to a Grain enum.
"""
- if time_grain in TimeGrain.__members__:
- return TimeGrain[time_grain]
-
- if time_grain in DateGrain.__members__:
- return DateGrain[time_grain]
+ mapping = {
+ grain.representation: grain
+ for grain in [
+ Second,
+ Minute,
+ Hour,
+ Day,
+ Week,
+ Month,
+ Quarter,
+ Year,
+ ]
+ }
- return None
+ return mapping.get(time_grain)
def validate_query_object(
diff --git a/superset/semantic_layers/models.py b/superset/semantic_layers/models.py
index 0beba9c026..ecf7f67854 100644
--- a/superset/semantic_layers/models.py
+++ b/superset/semantic_layers/models.py
@@ -117,7 +117,7 @@ class SemanticLayer(AuditMixinNullable, Model):
# Core fields
name = Column(String(250), nullable=False)
description = Column(Text, nullable=True)
- type = Column(String(250), nullable=False)
+ type = Column(String(250), nullable=False) # snowflake, etc
configuration = Column(encrypted_field_factory.create(Text), default="{}")
cache_timeout = Column(Integer, nullable=True)
@@ -172,6 +172,7 @@ class SemanticView(AuditMixinNullable, Model):
# Core fields
name = Column(String(250), nullable=False)
+ description = Column(Text, nullable=True)
configuration = Column(encrypted_field_factory.create(Text), default="{}")
cache_timeout = Column(Integer, nullable=True)
@@ -304,7 +305,7 @@ class SemanticView(AuditMixinNullable, Model):
"sql": None,
"select_star": None,
"owners": [owner.id for owner in self.owners],
- "description": None,
+ "description": self.description,
"table_name": self.name,
"column_types": [
get_column_type(dimension.type)
diff --git a/superset/semantic_layers/snowflake/semantic_layer.py b/superset/semantic_layers/snowflake/semantic_layer.py
index 8f99f6706f..4446c28662 100644
--- a/superset/semantic_layers/snowflake/semantic_layer.py
+++ b/superset/semantic_layers/snowflake/semantic_layer.py
@@ -233,4 +233,38 @@ class SnowflakeSemanticLayer(
SnowflakeSemanticView(row[0], configuration)
for row in cursor.execute(query)
}
- return views
+
+ return views
+
+ def get_semantic_view(
+ self,
+ name: str,
+ additional_configuration: dict[str, Any],
+ ) -> SnowflakeSemanticView:
+ """
+ Get a specific semantic view by name.
+ """
+ # Avoid circular import
+ from superset.semantic_layers.snowflake.semantic_view import (
+ SnowflakeSemanticView,
+ )
+
+ # create a new configuration with the additional parameters
+ configuration = self.configuration.model_copy(update=additional_configuration)
+
+ # check that the semantic view exists
+ connection_parameters = get_connection_parameters(configuration)
+ with connect(**connection_parameters) as connection:
+ cursor = connection.cursor()
+ query = dedent(
+ """
+ SHOW SEMANTIC VIEWS
+ ->> SELECT "name" FROM $1 WHERE "name" = ?;
+ """
+ ).strip()
+ cursor.execute(query, (name,))
+ row = cursor.fetchone()
+ if not row:
+ raise ValueError(f'Semantic view "{name}" does not exist.')
+
+ return SnowflakeSemanticView(name, configuration)
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 1dbdb52ca6..762e3dd47a 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -200,6 +200,7 @@ class DatasourceType(StrEnum):
QUERY = "query"
SAVEDQUERY = "saved_query"
VIEW = "view"
+ SEMANTIC_VIEW = "semantic_view"
class LoggerLevel(StrEnum):
diff --git a/superset/views/utils.py b/superset/views/utils.py
index e61f131cad..da4e991c03 100644
--- a/superset/views/utils.py
+++ b/superset/views/utils.py
@@ -20,6 +20,7 @@ from collections import defaultdict
from functools import wraps
from typing import Any, Callable, DefaultDict, Optional, Union
from urllib import parse
+from uuid import UUID
import msgpack
import pyarrow as pa
@@ -272,8 +273,10 @@ def add_sqllab_custom_filters(form_data: dict[Any, Any]) -> Any:
def get_datasource_info(
- datasource_id: Optional[int], datasource_type: Optional[str], form_data: FormData
-) -> tuple[int, Optional[str]]:
+ datasource_id: Optional[int],
+ datasource_type: Optional[str],
+ form_data: FormData,
+) -> tuple[int | UUID, Optional[str]]:
"""
Compatibility layer for handling of datasource info
@@ -300,7 +303,11 @@ def get_datasource_info(
_("The dataset associated with this chart no longer exists")
)
- datasource_id = int(datasource_id)
+ if datasource_id.isdigit():
+ datasource_id = int(datasource_id)
+ else:
+ datasource_id = UUID(datasource_id)
+
return datasource_id, datasource_type
(superset) 02/02: WIP
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch semantic-layer-implementation
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 2833b69ca054b3c272b6bf96784536974148b515
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Wed Dec 10 16:26:45 2025 -0500
WIP
---
superset/commands/explore/get.py | 6 +++++-
...-26_33d7e0e21daa_add_semantic_layers_and_views.py | 6 +++---
superset/semantic_layers/models.py | 20 +++++++++++---------
superset/semantic_layers/snowflake/semantic_layer.py | 16 ++++++++++------
4 files changed, 29 insertions(+), 19 deletions(-)
diff --git a/superset/commands/explore/get.py b/superset/commands/explore/get.py
index 78142eb5ec..7954aff33d 100644
--- a/superset/commands/explore/get.py
+++ b/superset/commands/explore/get.py
@@ -124,7 +124,11 @@ class GetExploreCommand(BaseCommand, ABC):
security_manager.raise_for_access(datasource=datasource)
viz_type = form_data.get("viz_type")
- if not viz_type and datasource and datasource.default_endpoint:
+ if (
+ not viz_type
+ and datasource
+ and getattr(datasource, "default_endpoint", None)
+ ):
raise WrongEndpointError(redirect=datasource.default_endpoint)
form_data["datasource"] = (
diff --git a/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py b/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
index c8e73e2cee..605a005c8e 100644
--- a/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
+++ b/superset/migrations/versions/2025-11-04_11-26_33d7e0e21daa_add_semantic_layers_and_views.py
@@ -25,8 +25,8 @@ Create Date: 2025-11-04 11:26:00.000000
import uuid
import sqlalchemy as sa
-from sqlalchemy.types import JSON
from sqlalchemy_utils import UUIDType
+from sqlalchemy_utils.types.json import JSONType
from superset.extensions import encrypted_field_factory
from superset.migrations.shared.utils import (
@@ -52,7 +52,7 @@ def upgrade():
sa.Column("type", sa.String(length=250), nullable=False),
sa.Column(
"configuration",
- encrypted_field_factory.create(JSON),
+ encrypted_field_factory.create(JSONType),
nullable=True,
),
sa.Column("cache_timeout", sa.Integer(), nullable=True),
@@ -88,7 +88,7 @@ def upgrade():
sa.Column("description", sa.Text(), nullable=True),
sa.Column(
"configuration",
- encrypted_field_factory.create(JSON),
+ encrypted_field_factory.create(JSONType),
nullable=True,
),
sa.Column("cache_timeout", sa.Integer(), nullable=True),
diff --git a/superset/semantic_layers/models.py b/superset/semantic_layers/models.py
index ecf7f67854..dc09e5fe6e 100644
--- a/superset/semantic_layers/models.py
+++ b/superset/semantic_layers/models.py
@@ -30,6 +30,7 @@ from flask_appbuilder import Model
from sqlalchemy import Column, ForeignKey, Integer, String, Text
from sqlalchemy.orm import relationship
from sqlalchemy_utils import UUIDType
+from sqlalchemy_utils.types.json import JSONType
from superset.common.query_object import QueryObject
from superset.explorables.base import TimeGrainDict
@@ -119,7 +120,7 @@ class SemanticLayer(AuditMixinNullable, Model):
description = Column(Text, nullable=True)
type = Column(String(250), nullable=False) # snowflake, etc
- configuration = Column(encrypted_field_factory.create(Text), default="{}")
+ configuration = Column(encrypted_field_factory.create(JSONType), default=dict)
cache_timeout = Column(Integer, nullable=True)
# Semantic views relationship
@@ -156,7 +157,7 @@ class SemanticLayer(AuditMixinNullable, Model):
"must be a subclass of SemanticLayerImplementation"
)
- return implementation_class.from_configuration(self.configuration)
+ return implementation_class.from_configuration(json.loads(self.configuration))
class SemanticView(AuditMixinNullable, Model):
@@ -174,7 +175,7 @@ class SemanticView(AuditMixinNullable, Model):
name = Column(String(250), nullable=False)
description = Column(Text, nullable=True)
- configuration = Column(encrypted_field_factory.create(Text), default="{}")
+ configuration = Column(encrypted_field_factory.create(JSONType), default=dict)
cache_timeout = Column(Integer, nullable=True)
# Semantic layer relationship
@@ -199,7 +200,7 @@ class SemanticView(AuditMixinNullable, Model):
"""
return self.semantic_layer.implementation.get_semantic_view(
self.name,
- self.configuration,
+ json.loads(self.configuration),
)
# =========================================================================
@@ -268,7 +269,7 @@ class SemanticView(AuditMixinNullable, Model):
"filterable": True,
"groupby": True,
"id": None,
- "uuid": dimension.uuid.hex,
+ "uuid": None,
"is_certified": False,
"is_dttm": dimension.type in {DATE, TIME, DATETIME},
"python_date_format": None,
@@ -287,7 +288,7 @@ class SemanticView(AuditMixinNullable, Model):
"description": metric.description,
"expression": metric.definition,
"id": None,
- "uuid": metric.uuid.hex,
+ "uuid": None,
"is_certified": False,
"metric_name": metric.name,
"warning_markdown": None,
@@ -304,16 +305,16 @@ class SemanticView(AuditMixinNullable, Model):
"filter_select_enabled": True,
"sql": None,
"select_star": None,
- "owners": [owner.id for owner in self.owners],
+ "owners": [],
"description": self.description,
"table_name": self.name,
"column_types": [
get_column_type(dimension.type)
for dimension in self.implementation.dimensions
],
- "column_names": {
+ "column_names": [
dimension.name for dimension in self.implementation.dimensions
- },
+ ],
# rare
"column_formats": {},
"datasource_name": self.name,
@@ -334,6 +335,7 @@ class SemanticView(AuditMixinNullable, Model):
"always_filter_main_dttm": False,
"normalize_columns": False,
# TODO XXX
+ # "owners": [owner.id for owner in self.owners],
"edit_url": "",
"default_endpoint": None,
"folders": [],
diff --git a/superset/semantic_layers/snowflake/semantic_layer.py b/superset/semantic_layers/snowflake/semantic_layer.py
index 4446c28662..185f57a2d4 100644
--- a/superset/semantic_layers/snowflake/semantic_layer.py
+++ b/superset/semantic_layers/snowflake/semantic_layer.py
@@ -18,21 +18,19 @@
from __future__ import annotations
from textwrap import dedent
-from typing import Any, Literal, TYPE_CHECKING
+from typing import Any, Literal
from pydantic import create_model, Field
from snowflake.connector import connect
from snowflake.connector.connection import SnowflakeConnection
from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration
+from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView
from superset.semantic_layers.snowflake.utils import get_connection_parameters
from superset.semantic_layers.types import (
SemanticLayerImplementation,
)
-if TYPE_CHECKING:
- from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView
-
class SnowflakeSemanticLayer(
SemanticLayerImplementation[SnowflakeConfiguration, SnowflakeSemanticView]
@@ -251,9 +249,12 @@ class SnowflakeSemanticLayer(
# create a new configuration with the additional parameters
configuration = self.configuration.model_copy(update=additional_configuration)
+ return SnowflakeSemanticView(name, configuration)
# check that the semantic view exists
connection_parameters = get_connection_parameters(configuration)
+ print("PARAMS")
+ print(connection_parameters)
with connect(**connection_parameters) as connection:
cursor = connection.cursor()
query = dedent(
@@ -262,9 +263,12 @@ class SnowflakeSemanticLayer(
->> SELECT "name" FROM $1 WHERE "name" = ?;
"""
).strip()
+ print("BETO Q")
+ print(repr(query))
+ print(repr(name))
cursor.execute(query, (name,))
- row = cursor.fetchone()
- if not row:
+ rows = cursor.fetchall()
+ if not rows:
raise ValueError(f'Semantic view "{name}" does not exist.')
return SnowflakeSemanticView(name, configuration)