You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2022/03/16 17:33:54 UTC
[superset] 01/01: feat: import/export assets commands
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch export_import_assets
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 0b5981fa101ab4e467576272960e691f29533edb
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Wed Mar 16 10:31:00 2022 -0700
feat: import/export assets commands
---
superset/commands/export/assets.py | 64 ++++++++
superset/commands/importers/v1/assets.py | 163 +++++++++++++++++++++
tests/integration_tests/commands_test.py | 112 ++++++++++++++
.../commands/__init__.py} | 19 ---
tests/unit_tests/commands/export_test.py | 94 ++++++++++++
5 files changed, 433 insertions(+), 19 deletions(-)
diff --git a/superset/commands/export/assets.py b/superset/commands/export/assets.py
new file mode 100644
index 0000000..8711cac
--- /dev/null
+++ b/superset/commands/export/assets.py
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from datetime import datetime, timezone
+from typing import Iterator, Tuple
+
+import yaml
+
+from superset.charts.commands.export import ExportChartsCommand
+from superset.commands.base import BaseCommand
+from superset.dashboards.commands.export import ExportDashboardsCommand
+from superset.databases.commands.export import ExportDatabasesCommand
+from superset.datasets.commands.export import ExportDatasetsCommand
+from superset.queries.saved_queries.commands.export import ExportSavedQueriesCommand
+from superset.utils.dict_import_export import EXPORT_VERSION
+
+METADATA_FILE_NAME = "metadata.yaml"
+
+
+class ExportAssetsCommand(BaseCommand):
+ """
+ Command that exports all databases, datasets, charts, dashboards and saved queries.
+ """
+
+ def run(self) -> Iterator[Tuple[str, str]]:
+
+ metadata = {
+ "version": EXPORT_VERSION,
+ "type": "assets",
+ "timestamp": datetime.now(tz=timezone.utc).isoformat(),
+ }
+ yield METADATA_FILE_NAME, yaml.safe_dump(metadata, sort_keys=False)
+ seen = {METADATA_FILE_NAME}
+
+ commands = [
+ ExportDatabasesCommand,
+ ExportDatasetsCommand,
+ ExportChartsCommand,
+ ExportDashboardsCommand,
+ ExportSavedQueriesCommand,
+ ]
+ for command in commands:
+ ids = [model.id for model in command.dao.find_all()]
+ for file_name, file_content in command(ids, export_related=False).run():
+ if file_name not in seen:
+ yield file_name, file_content
+ seen.add(file_name)
+
+ def validate(self) -> None:
+ pass
diff --git a/superset/commands/importers/v1/assets.py b/superset/commands/importers/v1/assets.py
new file mode 100644
index 0000000..6a54438
--- /dev/null
+++ b/superset/commands/importers/v1/assets.py
@@ -0,0 +1,163 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import Any, Dict, List, Optional, Tuple
+
+from marshmallow import Schema
+from marshmallow.exceptions import ValidationError
+from sqlalchemy.orm import Session
+from sqlalchemy.sql import select
+
+from superset import db
+from superset.charts.commands.importers.v1.utils import import_chart
+from superset.charts.schemas import ImportV1ChartSchema
+from superset.commands.base import BaseCommand
+from superset.commands.exceptions import CommandInvalidError, ImportFailedError
+from superset.commands.importers.v1.utils import (
+ load_configs,
+ load_metadata,
+ validate_metadata_type,
+)
+from superset.dashboards.commands.importers.v1.utils import (
+ find_chart_uuids,
+ import_dashboard,
+ update_id_refs,
+)
+from superset.dashboards.schemas import ImportV1DashboardSchema
+from superset.databases.commands.importers.v1.utils import import_database
+from superset.databases.schemas import ImportV1DatabaseSchema
+from superset.datasets.commands.importers.v1.utils import import_dataset
+from superset.datasets.schemas import ImportV1DatasetSchema
+from superset.models.dashboard import dashboard_slices
+from superset.queries.saved_queries.commands.importers.v1.utils import (
+ import_saved_query,
+)
+from superset.queries.saved_queries.schemas import ImportV1SavedQuerySchema
+
+
+class ImportAssetsCommand(BaseCommand):
+ """
+ Command for importing databases, datasets, charts, dashboards and saved queries.
+
+ This command is used for managing Superset assets externally under source control,
+ and will overwrite everything.
+ """
+
+ schemas: Dict[str, Schema] = {
+ "charts/": ImportV1ChartSchema(),
+ "dashboards/": ImportV1DashboardSchema(),
+ "datasets/": ImportV1DatasetSchema(),
+ "databases/": ImportV1DatabaseSchema(),
+ "queries/": ImportV1SavedQuerySchema(),
+ }
+
+ # pylint: disable=unused-argument
+ def __init__(self, contents: Dict[str, str], *args: Any, **kwargs: Any):
+ self.contents = contents
+ self.passwords: Dict[str, str] = kwargs.get("passwords") or {}
+ self._configs: Dict[str, Any] = {}
+
+ # pylint: disable=too-many-locals
+ @staticmethod
+ def _import(session: Session, configs: Dict[str, Any]) -> None:
+ # import databases first
+ database_ids: Dict[str, int] = {}
+ for file_name, config in configs.items():
+ if file_name.startswith("databases/"):
+ database = import_database(session, config, overwrite=True)
+ database_ids[str(database.uuid)] = database.id
+
+ # import saved queries
+ for file_name, config in configs.items():
+ if file_name.startswith("queries/"):
+ config["db_id"] = database_ids[config["database_uuid"]]
+ import_saved_query(session, config, overwrite=True)
+
+ # import datasets
+ dataset_info: Dict[str, Dict[str, Any]] = {}
+ for file_name, config in configs.items():
+ if file_name.startswith("datasets/"):
+ config["database_id"] = database_ids[config["database_uuid"]]
+ dataset = import_dataset(session, config, overwrite=True)
+ dataset_info[str(dataset.uuid)] = {
+ "datasource_id": dataset.id,
+ "datasource_type": dataset.datasource_type,
+ "datasource_name": dataset.table_name,
+ }
+
+ # import charts
+ chart_ids: Dict[str, int] = {}
+ for file_name, config in configs.items():
+ config.update(dataset_info[config["dataset_uuid"]])
+ chart = import_chart(session, config, overwrite=True)
+ chart_ids[str(chart.uuid)] = chart.id
+
+ # store the existing relationship between dashboards and charts
+ existing_relationships = session.execute(
+ select([dashboard_slices.c.dashboard_id, dashboard_slices.c.slice_id])
+ ).fetchall()
+
+ # import dashboards
+ dashboard_chart_ids: List[Tuple[int, int]] = []
+ for file_name, config in configs.items():
+ if file_name.startswith("dashboards/"):
+ config = update_id_refs(config, chart_ids, dataset_info)
+ dashboard = import_dashboard(session, config, overwrite=True)
+ for uuid in find_chart_uuids(config["position"]):
+ if uuid not in chart_ids:
+ break
+ chart_id = chart_ids[uuid]
+ if (dashboard.id, chart_id) not in existing_relationships:
+ dashboard_chart_ids.append((dashboard.id, chart_id))
+
+ # set ref in the dashboard_slices table
+ values = [
+ {"dashboard_id": dashboard_id, "slice_id": chart_id}
+ for (dashboard_id, chart_id) in dashboard_chart_ids
+ ]
+ # pylint: disable=no-value-for-parameter # sqlalchemy/issues/4656
+ session.execute(dashboard_slices.insert(), values)
+
+ def run(self) -> None:
+ self.validate()
+
+ # rollback to prevent partial imports
+ try:
+ self._import(db.session, self._configs)
+ db.session.commit()
+ except Exception as ex:
+ db.session.rollback()
+ raise ImportFailedError() from ex
+
+ def validate(self) -> None:
+ exceptions: List[ValidationError] = []
+
+ # verify that the metadata file is present and valid
+ try:
+ metadata: Optional[Dict[str, str]] = load_metadata(self.contents)
+ except ValidationError as exc:
+ exceptions.append(exc)
+ metadata = None
+ validate_metadata_type(metadata, "assets", exceptions)
+
+ self._configs = load_configs(
+ self.contents, self.schemas, self.passwords, exceptions
+ )
+
+ if exceptions:
+ exception = CommandInvalidError("Error importing assets")
+ exception.add_list(exceptions)
+ raise exception
diff --git a/tests/integration_tests/commands_test.py b/tests/integration_tests/commands_test.py
index 1adf5bd..2b0dd3a 100644
--- a/tests/integration_tests/commands_test.py
+++ b/tests/integration_tests/commands_test.py
@@ -14,9 +14,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import json
+from unittest.mock import patch
+
+import yaml
+
+from superset import db, security_manager
from superset.commands.exceptions import CommandInvalidError
+from superset.commands.importers.v1.assets import ImportAssetsCommand
from superset.commands.importers.v1.utils import is_valid_config
+from superset.models.dashboard import Dashboard
from tests.integration_tests.base_tests import SupersetTestCase
+from tests.integration_tests.fixtures.importexport import (
+ chart_config,
+ dashboard_config,
+ dashboard_metadata_config,
+ database_config,
+ dataset_config,
+)
class TestCommandsExceptions(SupersetTestCase):
@@ -33,3 +48,100 @@ class TestImportersV1Utils(SupersetTestCase):
assert not is_valid_config(
"__MACOSX/chart_export_20210111T145253/databases/._examples.yaml"
)
+
+
+class TestImportAssetsCommand(SupersetTestCase):
+ @patch("superset.dashboards.commands.importers.v1.utils.g")
+ def test_import_v1_dashboard(self, mock_g):
+ """Test that we can import a dashboard"""
+ mock_g.user = security_manager.find_user("admin")
+ contents = {
+ "metadata.yaml": yaml.safe_dump(dashboard_metadata_config),
+ "databases/imported_database.yaml": yaml.safe_dump(database_config),
+ "datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
+ "charts/imported_chart.yaml": yaml.safe_dump(chart_config),
+ "dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
+ }
+ command = ImportAssetsCommand(contents)
+ command.run()
+
+ dashboard = (
+ db.session.query(Dashboard).filter_by(uuid=dashboard_config["uuid"]).one()
+ )
+
+ assert len(dashboard.slices) == 1
+ chart = dashboard.slices[0]
+ assert str(chart.uuid) == chart_config["uuid"]
+ new_chart_id = chart.id
+
+ assert dashboard.dashboard_title == "Test dash"
+ assert dashboard.description is None
+ assert dashboard.css == ""
+ assert dashboard.slug is None
+ assert json.loads(dashboard.position_json) == {
+ "CHART-SVAlICPOSJ": {
+ "children": [],
+ "id": "CHART-SVAlICPOSJ",
+ "meta": {
+ "chartId": new_chart_id,
+ "height": 50,
+ "sliceName": "Number of California Births",
+ "uuid": "0c23747a-6528-4629-97bf-e4b78d3b9df1",
+ "width": 4,
+ },
+ "parents": ["ROOT_ID", "GRID_ID", "ROW-dP_CHaK2q"],
+ "type": "CHART",
+ },
+ "DASHBOARD_VERSION_KEY": "v2",
+ "GRID_ID": {
+ "children": ["ROW-dP_CHaK2q"],
+ "id": "GRID_ID",
+ "parents": ["ROOT_ID"],
+ "type": "GRID",
+ },
+ "HEADER_ID": {
+ "id": "HEADER_ID",
+ "meta": {"text": "Test dash"},
+ "type": "HEADER",
+ },
+ "ROOT_ID": {"children": ["GRID_ID"], "id": "ROOT_ID", "type": "ROOT"},
+ "ROW-dP_CHaK2q": {
+ "children": ["CHART-SVAlICPOSJ"],
+ "id": "ROW-dP_CHaK2q",
+ "meta": {"0": "ROOT_ID", "background": "BACKGROUND_TRANSPARENT"},
+ "parents": ["ROOT_ID", "GRID_ID"],
+ "type": "ROW",
+ },
+ }
+ assert json.loads(dashboard.json_metadata) == {
+ "color_scheme": None,
+ "default_filters": "{}",
+ "expanded_slices": {str(new_chart_id): True},
+ "filter_scopes": {
+ str(new_chart_id): {
+ "region": {"scope": ["ROOT_ID"], "immune": [new_chart_id]}
+ },
+ },
+ "import_time": 1604342885,
+ "refresh_frequency": 0,
+ "remote_id": 7,
+ "timed_refresh_immune_slices": [new_chart_id],
+ }
+
+ dataset = chart.table
+ assert str(dataset.uuid) == dataset_config["uuid"]
+
+ database = dataset.database
+ assert str(database.uuid) == database_config["uuid"]
+
+ assert dashboard.owners == [mock_g.user]
+
+ dashboard.owners = []
+ chart.owners = []
+ dataset.owners = []
+ database.owners = []
+ db.session.delete(dashboard)
+ db.session.delete(chart)
+ db.session.delete(dataset)
+ db.session.delete(database)
+ db.session.commit()
diff --git a/tests/integration_tests/commands_test.py b/tests/unit_tests/commands/__init__.py
similarity index 51%
copy from tests/integration_tests/commands_test.py
copy to tests/unit_tests/commands/__init__.py
index 1adf5bd..13a8339 100644
--- a/tests/integration_tests/commands_test.py
+++ b/tests/unit_tests/commands/__init__.py
@@ -14,22 +14,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from superset.commands.exceptions import CommandInvalidError
-from superset.commands.importers.v1.utils import is_valid_config
-from tests.integration_tests.base_tests import SupersetTestCase
-
-
-class TestCommandsExceptions(SupersetTestCase):
- def test_command_invalid_error(self):
- exception = CommandInvalidError("A test")
- assert str(exception) == "A test"
-
-
-class TestImportersV1Utils(SupersetTestCase):
- def test_is_valid_config(self):
- assert is_valid_config("metadata.yaml")
- assert is_valid_config("databases/examples.yaml")
- assert not is_valid_config(".DS_Store")
- assert not is_valid_config(
- "__MACOSX/chart_export_20210111T145253/databases/._examples.yaml"
- )
diff --git a/tests/unit_tests/commands/export_test.py b/tests/unit_tests/commands/export_test.py
new file mode 100644
index 0000000..91aebf1
--- /dev/null
+++ b/tests/unit_tests/commands/export_test.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# pylint: disable=invalid-name, unused-argument, import-outside-toplevel
+
+from freezegun import freeze_time
+from pytest_mock import MockFixture
+
+
+def test_export_assets_command(mocker: MockFixture, app_context: None) -> None:
+ """
+ Test that all assets are exported correctly.
+ """
+ from superset.commands.export.assets import ExportAssetsCommand
+
+ ExportDatabasesCommand = mocker.patch(
+ "superset.commands.export.assets.ExportDatabasesCommand"
+ )
+ ExportDatabasesCommand.return_value.run.return_value = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: Database\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("databases/example.yaml", "<DATABASE CONTENTS>"),
+ ]
+ ExportDatasetsCommand = mocker.patch(
+ "superset.commands.export.assets.ExportDatasetsCommand"
+ )
+ ExportDatasetsCommand.return_value.run.return_value = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: Dataset\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("datasets/example/dataset.yaml", "<DATASET CONTENTS>"),
+ ]
+ ExportChartsCommand = mocker.patch(
+ "superset.commands.export.assets.ExportChartsCommand"
+ )
+ ExportChartsCommand.return_value.run.return_value = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: Slice\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("charts/pie.yaml", "<CHART CONTENTS>"),
+ ]
+ ExportDashboardsCommand = mocker.patch(
+ "superset.commands.export.assets.ExportDashboardsCommand"
+ )
+ ExportDashboardsCommand.return_value.run.return_value = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: Dashboard\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("dashboards/sales.yaml", "<DASHBOARD CONTENTS>"),
+ ]
+ ExportSavedQueriesCommand = mocker.patch(
+ "superset.commands.export.assets.ExportSavedQueriesCommand"
+ )
+ ExportSavedQueriesCommand.return_value.run.return_value = [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: SavedQuery\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("queries/example/metric.yaml", "<SAVED QUERY CONTENTS>"),
+ ]
+
+ with freeze_time("2022-01-01T00:00:00Z"):
+ command = ExportAssetsCommand()
+ output = list(command.run())
+
+ assert output == [
+ (
+ "metadata.yaml",
+ "version: 1.0.0\ntype: assets\ntimestamp: '2022-01-01T00:00:00+00:00'\n",
+ ),
+ ("databases/example.yaml", "<DATABASE CONTENTS>"),
+ ("datasets/example/dataset.yaml", "<DATASET CONTENTS>"),
+ ("charts/pie.yaml", "<CHART CONTENTS>"),
+ ("dashboards/sales.yaml", "<DASHBOARD CONTENTS>"),
+ ("queries/example/metric.yaml", "<SAVED QUERY CONTENTS>"),
+ ]