You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by vi...@apache.org on 2021/08/02 13:12:23 UTC
[superset] branch master updated: fix(native-filters): add support
for versioned import/export (#16017)
This is an automated email from the ASF dual-hosted git repository.
villebro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new c77bf26 fix(native-filters): add support for versioned import/export (#16017)
c77bf26 is described below
commit c77bf264d18ed26a85ad29903019716ec979b002
Author: Ville Brofeldt <33...@users.noreply.github.com>
AuthorDate: Mon Aug 2 16:10:57 2021 +0300
fix(native-filters): add support for versioned import/export (#16017)
* fix(native-filters): add support for versioned import/export
* fix lint
* address CR
* make query_context nullable
---
superset/charts/commands/export.py | 2 +-
superset/charts/schemas.py | 2 +-
superset/commands/importers/v1/examples.py | 2 +-
superset/dashboards/commands/export.py | 16 ++++++++++++-
.../dashboards/commands/importers/v1/__init__.py | 10 ++++----
superset/dashboards/commands/importers/v1/utils.py | 28 +++++++++++++++++++++-
6 files changed, 51 insertions(+), 9 deletions(-)
diff --git a/superset/charts/commands/export.py b/superset/charts/commands/export.py
index 7fc778f..b0dbc9f 100644
--- a/superset/charts/commands/export.py
+++ b/superset/charts/commands/export.py
@@ -54,7 +54,7 @@ class ExportChartsCommand(ExportModelsCommand):
export_uuids=True,
)
# TODO (betodealmeida): move this logic to export_to_dict once this
- # becomes the default export endpoint
+ # becomes the default export endpoint
for key in REMOVE_KEYS:
del payload[key]
if payload.get("params"):
diff --git a/superset/charts/schemas.py b/superset/charts/schemas.py
index 795bb63..bef857d 100644
--- a/superset/charts/schemas.py
+++ b/superset/charts/schemas.py
@@ -1205,7 +1205,7 @@ class ImportV1ChartSchema(Schema):
slice_name = fields.String(required=True)
viz_type = fields.String(required=True)
params = fields.Dict()
- query_context = fields.Dict()
+ query_context = fields.String(allow_none=True, validate=utils.validate_json)
cache_timeout = fields.Integer(allow_none=True)
uuid = fields.UUID(required=True)
version = fields.String(required=True)
diff --git a/superset/commands/importers/v1/examples.py b/superset/commands/importers/v1/examples.py
index f84f529..b7e6caf 100644
--- a/superset/commands/importers/v1/examples.py
+++ b/superset/commands/importers/v1/examples.py
@@ -130,7 +130,7 @@ class ImportExamplesCommand(ImportModelsCommand):
dashboard_chart_ids: List[Tuple[int, int]] = []
for file_name, config in configs.items():
if file_name.startswith("dashboards/"):
- config = update_id_refs(config, chart_ids)
+ config = update_id_refs(config, chart_ids, dataset_info)
dashboard = import_dashboard(session, config, overwrite=overwrite)
dashboard.published = True
diff --git a/superset/dashboards/commands/export.py b/superset/dashboards/commands/export.py
index 09ddb75..a046797 100644
--- a/superset/dashboards/commands/export.py
+++ b/superset/dashboards/commands/export.py
@@ -30,6 +30,8 @@ from superset.dashboards.commands.exceptions import DashboardNotFoundError
from superset.dashboards.commands.importers.v1.utils import find_chart_uuids
from superset.dashboards.dao import DashboardDAO
from superset.commands.export import ExportModelsCommand
+from superset.datasets.commands.export import ExportDatasetsCommand
+from superset.datasets.dao import DatasetDAO
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from superset.utils.dict_import_export import EXPORT_VERSION
@@ -116,7 +118,7 @@ class ExportDashboardsCommand(ExportModelsCommand):
export_uuids=True,
)
# TODO (betodealmeida): move this logic to export_to_dict once this
- # becomes the default export endpoint
+ # becomes the default export endpoint
for key, new_name in JSON_KEYS.items():
value: Optional[str] = payload.pop(key, None)
if value:
@@ -126,6 +128,18 @@ class ExportDashboardsCommand(ExportModelsCommand):
logger.info("Unable to decode `%s` field: %s", key, value)
payload[new_name] = {}
+ # Extract all native filter datasets and replace native
+ # filter dataset references with uuid
+ for native_filter in payload.get("metadata", {}).get(
+ "native_filter_configuration", []
+ ):
+ for target in native_filter.get("targets", []):
+ dataset_id = target.pop("datasetId", None)
+ if dataset_id is not None:
+ dataset = DatasetDAO.find_by_id(dataset_id)
+ target["datasetUuid"] = str(dataset.uuid)
+ yield from ExportDatasetsCommand([dataset_id]).run()
+
# the mapping between dashboard -> charts is inferred from the position
# attribute, so if it's not present we need to add a default config
if not payload.get("position"):
diff --git a/superset/dashboards/commands/importers/v1/__init__.py b/superset/dashboards/commands/importers/v1/__init__.py
index 57c5087..6e17aef 100644
--- a/superset/dashboards/commands/importers/v1/__init__.py
+++ b/superset/dashboards/commands/importers/v1/__init__.py
@@ -27,6 +27,7 @@ from superset.commands.importers.v1 import ImportModelsCommand
from superset.dashboards.commands.exceptions import DashboardImportError
from superset.dashboards.commands.importers.v1.utils import (
find_chart_uuids,
+ find_native_filter_datasets,
import_dashboard,
update_id_refs,
)
@@ -60,14 +61,15 @@ class ImportDashboardsCommand(ImportModelsCommand):
def _import(
session: Session, configs: Dict[str, Any], overwrite: bool = False
) -> None:
- # discover charts associated with dashboards
+ # discover charts and datasets associated with dashboards
chart_uuids: Set[str] = set()
+ dataset_uuids: Set[str] = set()
for file_name, config in configs.items():
if file_name.startswith("dashboards/"):
chart_uuids.update(find_chart_uuids(config["position"]))
+ dataset_uuids.update(find_native_filter_datasets(config["metadata"]))
# discover datasets associated with charts
- dataset_uuids: Set[str] = set()
for file_name, config in configs.items():
if file_name.startswith("charts/") and config["uuid"] in chart_uuids:
dataset_uuids.add(config["dataset_uuid"])
@@ -96,7 +98,7 @@ class ImportDashboardsCommand(ImportModelsCommand):
dataset = import_dataset(session, config, overwrite=False)
dataset_info[str(dataset.uuid)] = {
"datasource_id": dataset.id,
- "datasource_type": "view" if dataset.is_sqllab_view else "table",
+ "datasource_type": dataset.datasource_type,
"datasource_name": dataset.table_name,
}
@@ -121,7 +123,7 @@ class ImportDashboardsCommand(ImportModelsCommand):
dashboard_chart_ids: List[Tuple[int, int]] = []
for file_name, config in configs.items():
if file_name.startswith("dashboards/"):
- config = update_id_refs(config, chart_ids)
+ config = update_id_refs(config, chart_ids, dataset_info)
dashboard = import_dashboard(session, config, overwrite=overwrite)
for uuid in find_chart_uuids(config["position"]):
if uuid not in chart_ids:
diff --git a/superset/dashboards/commands/importers/v1/utils.py b/superset/dashboards/commands/importers/v1/utils.py
index 7b2ad91..c7bd15e 100644
--- a/superset/dashboards/commands/importers/v1/utils.py
+++ b/superset/dashboards/commands/importers/v1/utils.py
@@ -33,6 +33,17 @@ def find_chart_uuids(position: Dict[str, Any]) -> Set[str]:
return set(build_uuid_to_id_map(position))
+def find_native_filter_datasets(metadata: Dict[str, Any]) -> Set[str]:
+ uuids: Set[str] = set()
+ for native_filter in metadata.get("native_filter_configuration", []):
+ targets = native_filter.get("targets", [])
+ for target in targets:
+ dataset_uuid = target.get("datasetUuid")
+ if dataset_uuid:
+ uuids.add(dataset_uuid)
+ return uuids
+
+
def build_uuid_to_id_map(position: Dict[str, Any]) -> Dict[str, int]:
return {
child["meta"]["uuid"]: child["meta"]["chartId"]
@@ -45,7 +56,11 @@ def build_uuid_to_id_map(position: Dict[str, Any]) -> Dict[str, int]:
}
-def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[str, Any]:
+def update_id_refs(
+ config: Dict[str, Any],
+ chart_ids: Dict[str, int],
+ dataset_info: Dict[str, Dict[str, Any]],
+) -> Dict[str, Any]:
"""Update dashboard metadata to use new IDs"""
fixed = config.copy()
@@ -103,6 +118,17 @@ def update_id_refs(config: Dict[str, Any], chart_ids: Dict[str, int]) -> Dict[st
):
child["meta"]["chartId"] = chart_ids[child["meta"]["uuid"]]
+ # fix native filter references
+ native_filter_configuration = fixed["metadata"].get(
+ "native_filter_configuration", []
+ )
+ for native_filter in native_filter_configuration:
+ targets = native_filter.get("targets", [])
+ for target in targets:
+ dataset_uuid = target.pop("datasetUuid", None)
+ if dataset_uuid:
+ target["datasetId"] = dataset_info[dataset_uuid]["datasource_id"]
+
return fixed