You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2024/05/24 18:09:21 UTC
(superset) branch rm_sl created (now 81725e3ee5)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a change to branch rm_sl
in repository https://gitbox.apache.org/repos/asf/superset.git
at 81725e3ee5 chore: remove sl_ tables
This branch includes the following new commits:
new 81725e3ee5 chore: remove sl_ tables
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(superset) 01/01: chore: remove sl_ tables
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch rm_sl
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 81725e3ee5ded51828bd6d82b98af5125fd1dc5b
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Fri May 24 11:06:53 2024 -0700
chore: remove sl_ tables
---
superset/columns/models.py | 115 ------------------------
superset/daos/datasource.py | 6 +-
superset/datasets/models.py | 118 -------------------------
superset/datasets/schemas.py | 13 ---
superset/tables/models.py | 206 -------------------------------------------
superset/utils/core.py | 1 -
6 files changed, 1 insertion(+), 458 deletions(-)
diff --git a/superset/columns/models.py b/superset/columns/models.py
deleted file mode 100644
index 142e3d6723..0000000000
--- a/superset/columns/models.py
+++ /dev/null
@@ -1,115 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Column model.
-
-This model was introduced in SIP-68 (https://github.com/apache/superset/issues/14909),
-and represents a "column" in a table or dataset. In addition to a column, new models for
-tables, metrics, and datasets were also introduced.
-
-These models are not fully implemented, and shouldn't be used yet.
-"""
-
-import sqlalchemy as sa
-from flask_appbuilder import Model
-
-from superset.models.helpers import (
- AuditMixinNullable,
- ExtraJSONMixin,
- ImportExportMixin,
-)
-
-UNKNOWN_TYPE = "UNKNOWN"
-
-
-class Column(
- Model,
- AuditMixinNullable,
- ExtraJSONMixin,
- ImportExportMixin,
-):
- """
- A "column".
-
- The definition of column here is overloaded: it can represent a physical column in a
- database relation (table or view); a computed/derived column on a dataset; or an
- aggregation expression representing a metric.
- """
-
- __tablename__ = "sl_columns"
-
- id = sa.Column(sa.Integer, primary_key=True)
-
- # Assuming the column is an aggregation, is it additive? Useful for determining which
- # aggregations can be done on the metric. Eg, ``COUNT(DISTINCT user_id)`` is not
- # additive, so it shouldn't be used in a ``SUM``.
- is_additive = sa.Column(sa.Boolean, default=False)
-
- # Is this column an aggregation (metric)?
- is_aggregation = sa.Column(sa.Boolean, default=False)
-
- is_filterable = sa.Column(sa.Boolean, nullable=False, default=True)
- is_dimensional = sa.Column(sa.Boolean, nullable=False, default=False)
-
- # Is an increase desired? Useful for displaying the results of A/B tests, or setting
- # up alerts. Eg, this is true for "revenue", but false for "latency".
- is_increase_desired = sa.Column(sa.Boolean, default=True)
-
- # Column is managed externally and should be read-only inside Superset
- is_managed_externally = sa.Column(sa.Boolean, nullable=False, default=False)
-
- # Is this column a partition? Useful for scheduling queries and previewing the latest
- # data.
- is_partition = sa.Column(sa.Boolean, default=False)
-
- # Does the expression point directly to a physical column?
- is_physical = sa.Column(sa.Boolean, default=True)
-
- # Is this a spatial column? This could be leveraged in the future for spatial
- # visualizations.
- is_spatial = sa.Column(sa.Boolean, default=False)
-
- # Is this a time column? Useful for plotting time series.
- is_temporal = sa.Column(sa.Boolean, default=False)
-
- # We use ``sa.Text`` for these attributes because (1) in modern databases the
- # performance is the same as ``VARCHAR``[1] and (2) because some table names can be
- # **really** long (eg, Google Sheets URLs).
- #
- # [1] https://www.postgresql.org/docs/9.1/datatype-character.html
- name = sa.Column(sa.Text)
- # Raw type as returned and used by db engine.
- type = sa.Column(sa.Text, default=UNKNOWN_TYPE)
-
- # Assigns column advanced type to determine custom behavior
- # does nothing unless feature flag ENABLE_ADVANCED_DATA_TYPES in true
- advanced_data_type = sa.Column(sa.Text)
-
- # Columns are defined by expressions. For tables, these are the actual columns names,
- # and should match the ``name`` attribute. For datasets, these can be any valid SQL
- # expression. If the SQL expression is an aggregation the column is a metric,
- # otherwise it's a computed column.
- expression = sa.Column(sa.Text)
- unit = sa.Column(sa.Text)
-
- # Additional metadata describing the column.
- description = sa.Column(sa.Text)
- warning_text = sa.Column(sa.Text)
- external_url = sa.Column(sa.Text, nullable=True)
-
- def __repr__(self) -> str:
- return f"<Column id={self.id}>"
diff --git a/superset/daos/datasource.py b/superset/daos/datasource.py
index 0e6058d6ab..9a48bb86b7 100644
--- a/superset/daos/datasource.py
+++ b/superset/daos/datasource.py
@@ -22,14 +22,12 @@ from superset import db
from superset.connectors.sqla.models import SqlaTable
from superset.daos.base import BaseDAO
from superset.daos.exceptions import DatasourceNotFound, DatasourceTypeNotSupportedError
-from superset.datasets.models import Dataset
from superset.models.sql_lab import Query, SavedQuery
-from superset.tables.models import Table
from superset.utils.core import DatasourceType
logger = logging.getLogger(__name__)
-Datasource = Union[Dataset, SqlaTable, Table, Query, SavedQuery]
+Datasource = Union[SqlaTable, Query, SavedQuery]
class DatasourceDAO(BaseDAO[Datasource]):
@@ -37,8 +35,6 @@ class DatasourceDAO(BaseDAO[Datasource]):
DatasourceType.TABLE: SqlaTable,
DatasourceType.QUERY: Query,
DatasourceType.SAVEDQUERY: SavedQuery,
- DatasourceType.DATASET: Dataset,
- DatasourceType.SLTABLE: Table,
}
@classmethod
diff --git a/superset/datasets/models.py b/superset/datasets/models.py
deleted file mode 100644
index 76e2156e61..0000000000
--- a/superset/datasets/models.py
+++ /dev/null
@@ -1,118 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Dataset model.
-
-This model was introduced in SIP-68 (https://github.com/apache/superset/issues/14909),
-and represents a "dataset" -- either a physical table or a virtual. In addition to a
-dataset, new models for columns, metrics, and tables were also introduced.
-
-These models are not fully implemented, and shouldn't be used yet.
-"""
-
-import sqlalchemy as sa
-from flask_appbuilder import Model
-from sqlalchemy.orm import backref, relationship
-
-from superset import security_manager
-from superset.columns.models import Column
-from superset.models.core import Database
-from superset.models.helpers import (
- AuditMixinNullable,
- ExtraJSONMixin,
- ImportExportMixin,
-)
-from superset.tables.models import Table
-
-dataset_column_association_table = sa.Table(
- "sl_dataset_columns",
- Model.metadata, # pylint: disable=no-member
- sa.Column(
- "dataset_id",
- sa.ForeignKey("sl_datasets.id"),
- primary_key=True,
- ),
- sa.Column(
- "column_id",
- sa.ForeignKey("sl_columns.id"),
- primary_key=True,
- ),
-)
-
-dataset_table_association_table = sa.Table(
- "sl_dataset_tables",
- Model.metadata, # pylint: disable=no-member
- sa.Column("dataset_id", sa.ForeignKey("sl_datasets.id"), primary_key=True),
- sa.Column("table_id", sa.ForeignKey("sl_tables.id"), primary_key=True),
-)
-
-dataset_user_association_table = sa.Table(
- "sl_dataset_users",
- Model.metadata, # pylint: disable=no-member
- sa.Column("dataset_id", sa.ForeignKey("sl_datasets.id"), primary_key=True),
- sa.Column("user_id", sa.ForeignKey("ab_user.id"), primary_key=True),
-)
-
-
-class Dataset(AuditMixinNullable, ExtraJSONMixin, ImportExportMixin, Model):
- """
- A table/view in a database.
- """
-
- __tablename__ = "sl_datasets"
-
- id = sa.Column(sa.Integer, primary_key=True)
- database_id = sa.Column(sa.Integer, sa.ForeignKey("dbs.id"), nullable=False)
- database: Database = relationship(
- "Database",
- backref=backref("datasets", cascade="all, delete-orphan"),
- foreign_keys=[database_id],
- )
- # The relationship between datasets and columns is 1:n, but we use a
- # many-to-many association table to avoid adding two mutually exclusive
- # columns(dataset_id and table_id) to Column
- columns: list[Column] = relationship(
- "Column",
- secondary=dataset_column_association_table,
- cascade="all, delete-orphan",
- single_parent=True,
- backref="datasets",
- )
- owners = relationship(
- security_manager.user_model, secondary=dataset_user_association_table
- )
- tables: list[Table] = relationship(
- "Table", secondary=dataset_table_association_table, backref="datasets"
- )
-
- # Does the dataset point directly to a ``Table``?
- is_physical = sa.Column(sa.Boolean, default=False)
-
- # Column is managed externally and should be read-only inside Superset
- is_managed_externally = sa.Column(sa.Boolean, nullable=False, default=False)
-
- # We use ``sa.Text`` for these attributes because (1) in modern databases the
- # performance is the same as ``VARCHAR``[1] and (2) because some table names can be
- # **really** long (eg, Google Sheets URLs).
- #
- # [1] https://www.postgresql.org/docs/9.1/datatype-character.html
- name = sa.Column(sa.Text)
- expression = sa.Column(sa.Text)
- external_url = sa.Column(sa.Text, nullable=True)
-
- def __repr__(self) -> str:
- return f"<Dataset id={self.id} database_id={self.database_id} {self.name}>"
diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py
index 5ce0621675..426cde01b5 100644
--- a/superset/datasets/schemas.py
+++ b/superset/datasets/schemas.py
@@ -22,9 +22,7 @@ from dateutil.parser import isoparse
from flask_babel import lazy_gettext as _
from marshmallow import fields, pre_load, Schema, ValidationError
from marshmallow.validate import Length
-from marshmallow_sqlalchemy import SQLAlchemyAutoSchema
-from superset.datasets.models import Dataset
from superset.exceptions import SupersetMarshmallowValidationError
get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}}
@@ -291,17 +289,6 @@ class GetOrCreateDatasetSchema(Schema):
always_filter_main_dttm = fields.Boolean(load_default=False)
-class DatasetSchema(SQLAlchemyAutoSchema):
- """
- Schema for the ``Dataset`` model.
- """
-
- class Meta: # pylint: disable=too-few-public-methods
- model = Dataset
- load_instance = True
- include_relationships = True
-
-
class DatasetCacheWarmUpRequestSchema(Schema):
db_name = fields.String(
required=True,
diff --git a/superset/tables/models.py b/superset/tables/models.py
deleted file mode 100644
index 2616aaf90f..0000000000
--- a/superset/tables/models.py
+++ /dev/null
@@ -1,206 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""
-Table model.
-
-This model was introduced in SIP-68 (https://github.com/apache/superset/issues/14909),
-and represents a "table" in a given database -- either a physical table or a view. In
-addition to a table, new models for columns, metrics, and datasets were also introduced.
-
-These models are not fully implemented, and shouldn't be used yet.
-"""
-
-from collections.abc import Iterable
-from typing import Any, Optional, TYPE_CHECKING
-
-import sqlalchemy as sa
-from flask_appbuilder import Model
-from sqlalchemy import inspect
-from sqlalchemy.orm import backref, relationship, Session
-from sqlalchemy.schema import UniqueConstraint
-from sqlalchemy.sql import and_, or_
-
-from superset.columns.models import Column
-from superset.connectors.sqla.utils import get_physical_table_metadata
-from superset.models.core import Database
-from superset.models.helpers import (
- AuditMixinNullable,
- ExtraJSONMixin,
- ImportExportMixin,
-)
-from superset.sql_parse import Table as TableName
-from superset.superset_typing import ResultSetColumnType
-
-if TYPE_CHECKING:
- from superset.datasets.models import Dataset
-
-table_column_association_table = sa.Table(
- "sl_table_columns",
- Model.metadata, # pylint: disable=no-member
- sa.Column(
- "table_id",
- sa.ForeignKey("sl_tables.id", ondelete="CASCADE"),
- primary_key=True,
- ),
- sa.Column(
- "column_id",
- sa.ForeignKey("sl_columns.id", ondelete="CASCADE"),
- primary_key=True,
- ),
-)
-
-
-class Table(AuditMixinNullable, ExtraJSONMixin, ImportExportMixin, Model):
- """
- A table/view in a database.
- """
-
- __tablename__ = "sl_tables"
-
- # Note this uniqueness constraint is not part of the physical schema, i.e., it does
- # not exist in the migrations. The reason it does not physically exist is MySQL,
- # PostgreSQL, etc. have a different interpretation of uniqueness when it comes to NULL
- # which is problematic given the catalog and schema are optional.
- __table_args__ = (UniqueConstraint("database_id", "catalog", "schema", "name"),)
-
- id = sa.Column(sa.Integer, primary_key=True)
- database_id = sa.Column(sa.Integer, sa.ForeignKey("dbs.id"), nullable=False)
- database: Database = relationship(
- "Database",
- # TODO (betodealmeida): rename the backref to ``tables`` once we get rid of the
- # old models.
- backref=backref("new_tables", cascade="all, delete-orphan"),
- foreign_keys=[database_id],
- )
- # The relationship between datasets and columns is 1:n, but we use a
- # many-to-many association table to avoid adding two mutually exclusive
- # columns(dataset_id and table_id) to Column
- columns: list[Column] = relationship(
- "Column",
- secondary=table_column_association_table,
- cascade="all, delete-orphan",
- single_parent=True,
- # backref is needed for session to skip detaching `dataset` if only `column`
- # is loaded.
- backref="tables",
- )
- datasets: list["Dataset"] # will be populated by Dataset.tables backref
-
- # We use ``sa.Text`` for these attributes because (1) in modern databases the
- # performance is the same as ``VARCHAR``[1] and (2) because some table names can be
- # **really** long (eg, Google Sheets URLs).
- #
- # [1] https://www.postgresql.org/docs/9.1/datatype-character.html
- catalog = sa.Column(sa.Text)
- schema = sa.Column(sa.Text)
- name = sa.Column(sa.Text)
-
- # Column is managed externally and should be read-only inside Superset
- is_managed_externally = sa.Column(sa.Boolean, nullable=False, default=False)
- external_url = sa.Column(sa.Text, nullable=True)
-
- @property
- def fullname(self) -> str:
- return str(TableName(table=self.name, schema=self.schema, catalog=self.catalog))
-
- def __repr__(self) -> str:
- return f"<Table id={self.id} database_id={self.database_id} {self.fullname}>"
-
- def sync_columns(self) -> None:
- """Sync table columns with the database. Keep metadata for existing columns"""
- try:
- column_metadata = get_physical_table_metadata(
- self.database, self.name, self.schema
- )
- except Exception: # pylint: disable=broad-except
- column_metadata = []
-
- existing_columns = {column.name: column for column in self.columns}
- quote_identifier = self.database.quote_identifier
-
- def update_or_create_column(column_meta: ResultSetColumnType) -> Column:
- column_name: str = column_meta["column_name"]
- if column_name in existing_columns:
- column = existing_columns[column_name]
- else:
- column = Column(name=column_name)
- column.type = column_meta["type"]
- column.is_temporal = column_meta["is_dttm"]
- column.expression = quote_identifier(column_name)
- column.is_aggregation = False
- column.is_physical = True
- column.is_spatial = False
- column.is_partition = False # TODO: update with accurate is_partition
- return column
-
- self.columns = [update_or_create_column(col) for col in column_metadata]
-
- @staticmethod
- def bulk_load_or_create(
- database: Database,
- table_names: Iterable[TableName],
- default_schema: Optional[str] = None,
- sync_columns: Optional[bool] = False,
- default_props: Optional[dict[str, Any]] = None,
- ) -> list["Table"]:
- """
- Load or create multiple Table instances.
- """
- if not table_names:
- return []
-
- if not database.id:
- raise Exception( # pylint: disable=broad-exception-raised
- "Database must be already saved to metastore"
- )
-
- default_props = default_props or {}
- session: Session = inspect(database).session # pylint: disable=disallowed-name
- # load existing tables
- predicate = or_(
- *[
- and_(
- Table.database_id == database.id,
- Table.schema == (table.schema or default_schema),
- Table.name == table.table,
- )
- for table in table_names
- ]
- )
- all_tables = session.query(Table).filter(predicate).order_by(Table.id).all()
-
- # add missing tables and pull its columns
- existing = {(table.schema, table.name) for table in all_tables}
- for table in table_names:
- schema = table.schema or default_schema
- name = table.table
- if (schema, name) not in existing:
- new_table = Table(
- database=database,
- database_id=database.id,
- name=name,
- schema=schema,
- catalog=None,
- **default_props,
- )
- if sync_columns:
- new_table.sync_columns()
- all_tables.append(new_table)
- existing.add((schema, name))
- session.add(new_table)
-
- return all_tables
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 37a11ea7ff..18e7e60d78 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -147,7 +147,6 @@ class GenericDataType(IntEnum):
class DatasourceType(StrEnum):
- SLTABLE = "sl_table"
TABLE = "table"
DATASET = "dataset"
QUERY = "query"