You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by gr...@apache.org on 2020/08/16 21:44:02 UTC
[incubator-superset] branch master updated: refactor: [migration]
convert iframe chart into dashboard markdown component (#10590)
This is an automated email from the ASF dual-hosted git repository.
graceguo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new ca9ca99 refactor: [migration] convert iframe chart into dashboard markdown component (#10590)
ca9ca99 is described below
commit ca9ca995109ee5251918b9df60980bffaf9b9a24
Author: Grace Guo <gr...@airbnb.com>
AuthorDate: Sun Aug 16 14:43:30 2020 -0700
refactor: [migration] convert iframe chart into dashboard markdown component (#10590)
* refactor: [migration] convert iframe chart into dashboard markdown component
* remove 3 viz_types
* fix comments
---
UPDATING.md | 2 +
.../src/dashboard/reducers/getInitialState.js | 202 ++++++++++-----------
.../src/explore/components/SaveModal.jsx | 8 +-
.../explore/components/controls/VizTypeControl.jsx | 3 -
superset-frontend/src/explore/constants.js | 2 -
.../src/visualizations/presets/MainPreset.js | 5 -
...978245563a02_migrate_iframe_to_dash_markdown.py | 198 ++++++++++++++++++++
superset/migrations/versions/f80a3b88324b_.py | 38 ++++
superset/tasks/slack_util.py | 2 +-
superset/viz.py | 19 --
10 files changed, 340 insertions(+), 139 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 82c36f5..054afaf 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -23,6 +23,8 @@ assists people when migrating to a new version.
## Next
+* [10590](https://github.com/apache/incubator-superset/pull/10590): Breaking change: this PR will convert iframe chart into dashboard markdown component, and remove all `iframe`, `separator`, and `markup` slices (and support) from Superset. If you have important data in those slices, please backup manually.
+
* [10562](https://github.com/apache/incubator-superset/pull/10562): EMAIL_REPORTS_WEBDRIVER is deprecated use WEBDRIVER_TYPE instead.
* [10567](https://github.com/apache/incubator-superset/pull/10567): Default WEBDRIVER_OPTION_ARGS are Chrome-specific. If you're using FF, should be `--headless` only
diff --git a/superset-frontend/src/dashboard/reducers/getInitialState.js b/superset-frontend/src/dashboard/reducers/getInitialState.js
index 770e249..2e419df 100644
--- a/superset-frontend/src/dashboard/reducers/getInitialState.js
+++ b/superset-frontend/src/dashboard/reducers/getInitialState.js
@@ -108,118 +108,116 @@ export default function (bootstrapData) {
const sliceIds = new Set();
dashboard.slices.forEach(slice => {
const key = slice.slice_id;
- if (['separator', 'markup'].indexOf(slice.form_data.viz_type) === -1) {
- const form_data = {
- ...slice.form_data,
- url_params: {
- ...slice.form_data.url_params,
- ...urlParams,
- },
- };
- chartQueries[key] = {
- ...chart,
- id: key,
- form_data,
- formData: applyDefaultFormData(form_data),
- };
-
- slices[key] = {
- slice_id: key,
- slice_url: slice.slice_url,
- slice_name: slice.slice_name,
- form_data: slice.form_data,
- edit_url: slice.edit_url,
- viz_type: slice.form_data.viz_type,
- datasource: slice.form_data.datasource,
- description: slice.description,
- description_markeddown: slice.description_markeddown,
- owners: slice.owners,
- modified: slice.modified,
- changed_on: new Date(slice.changed_on).getTime(),
- };
+ const form_data = {
+ ...slice.form_data,
+ url_params: {
+ ...slice.form_data.url_params,
+ ...urlParams,
+ },
+ };
+ chartQueries[key] = {
+ ...chart,
+ id: key,
+ form_data,
+ formData: applyDefaultFormData(form_data),
+ };
- sliceIds.add(key);
+ slices[key] = {
+ slice_id: key,
+ slice_url: slice.slice_url,
+ slice_name: slice.slice_name,
+ form_data: slice.form_data,
+ edit_url: slice.edit_url,
+ viz_type: slice.form_data.viz_type,
+ datasource: slice.form_data.datasource,
+ description: slice.description,
+ description_markeddown: slice.description_markeddown,
+ owners: slice.owners,
+ modified: slice.modified,
+ changed_on: new Date(slice.changed_on).getTime(),
+ };
- // if there are newly added slices from explore view, fill slices into 1 or more rows
- if (!chartIdToLayoutId[key] && layout[parentId]) {
- if (
- newSlicesContainerWidth === 0 ||
- newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
- ) {
- newSlicesContainer = newComponentFactory(
- ROW_TYPE,
- (parent.parents || []).slice(),
- );
- layout[newSlicesContainer.id] = newSlicesContainer;
- parent.children.push(newSlicesContainer.id);
- newSlicesContainerWidth = 0;
- }
+ sliceIds.add(key);
- const chartHolder = newComponentFactory(
- CHART_TYPE,
- {
- chartId: slice.slice_id,
- },
- (newSlicesContainer.parents || []).slice(),
+ // if there are newly added slices from explore view, fill slices into 1 or more rows
+ if (!chartIdToLayoutId[key] && layout[parentId]) {
+ if (
+ newSlicesContainerWidth === 0 ||
+ newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
+ ) {
+ newSlicesContainer = newComponentFactory(
+ ROW_TYPE,
+ (parent.parents || []).slice(),
);
-
- layout[chartHolder.id] = chartHolder;
- newSlicesContainer.children.push(chartHolder.id);
- chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
- newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
+ layout[newSlicesContainer.id] = newSlicesContainer;
+ parent.children.push(newSlicesContainer.id);
+ newSlicesContainerWidth = 0;
}
- // build DashboardFilters for interactive filter features
- if (slice.form_data.viz_type === 'filter_box') {
- const configs = getFilterConfigsFromFormdata(slice.form_data);
- let columns = configs.columns;
- const labels = configs.labels;
- if (preselectFilters[key]) {
- Object.keys(columns).forEach(col => {
- if (preselectFilters[key][col]) {
- columns = {
- ...columns,
- [col]: preselectFilters[key][col],
- };
- }
- });
- }
+ const chartHolder = newComponentFactory(
+ CHART_TYPE,
+ {
+ chartId: slice.slice_id,
+ },
+ (newSlicesContainer.parents || []).slice(),
+ );
- const scopesByChartId = Object.keys(columns).reduce((map, column) => {
- const scopeSettings = {
- ...filterScopes[key],
- };
- const { scope, immune } = {
- ...DASHBOARD_FILTER_SCOPE_GLOBAL,
- ...scopeSettings[column],
- };
+ layout[chartHolder.id] = chartHolder;
+ newSlicesContainer.children.push(chartHolder.id);
+ chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
+ newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
+ }
- return {
- ...map,
- [column]: {
- scope,
- immune,
- },
- };
- }, {});
+ // build DashboardFilters for interactive filter features
+ if (slice.form_data.viz_type === 'filter_box') {
+ const configs = getFilterConfigsFromFormdata(slice.form_data);
+ let columns = configs.columns;
+ const labels = configs.labels;
+ if (preselectFilters[key]) {
+ Object.keys(columns).forEach(col => {
+ if (preselectFilters[key][col]) {
+ columns = {
+ ...columns,
+ [col]: preselectFilters[key][col],
+ };
+ }
+ });
+ }
- const componentId = chartIdToLayoutId[key];
- const directPathToFilter = (layout[componentId].parents || []).slice();
- directPathToFilter.push(componentId);
- dashboardFilters[key] = {
- ...dashboardFilter,
- chartId: key,
- componentId,
- datasourceId: slice.form_data.datasource,
- filterName: slice.slice_name,
- directPathToFilter,
- columns,
- labels,
- scopes: scopesByChartId,
- isInstantFilter: !!slice.form_data.instant_filtering,
- isDateFilter: Object.keys(columns).includes(TIME_RANGE),
+ const scopesByChartId = Object.keys(columns).reduce((map, column) => {
+ const scopeSettings = {
+ ...filterScopes[key],
};
- }
+ const { scope, immune } = {
+ ...DASHBOARD_FILTER_SCOPE_GLOBAL,
+ ...scopeSettings[column],
+ };
+
+ return {
+ ...map,
+ [column]: {
+ scope,
+ immune,
+ },
+ };
+ }, {});
+
+ const componentId = chartIdToLayoutId[key];
+ const directPathToFilter = (layout[componentId].parents || []).slice();
+ directPathToFilter.push(componentId);
+ dashboardFilters[key] = {
+ ...dashboardFilter,
+ chartId: key,
+ componentId,
+ datasourceId: slice.form_data.datasource,
+ filterName: slice.slice_name,
+ directPathToFilter,
+ columns,
+ labels,
+ scopes: scopesByChartId,
+ isInstantFilter: !!slice.form_data.instant_filtering,
+ isDateFilter: Object.keys(columns).includes(TIME_RANGE),
+ };
}
// sync layout names with current slice names in case a slice was edited
diff --git a/superset-frontend/src/explore/components/SaveModal.jsx b/superset-frontend/src/explore/components/SaveModal.jsx
index 45b96ff..804ebc2 100644
--- a/superset-frontend/src/explore/components/SaveModal.jsx
+++ b/superset-frontend/src/explore/components/SaveModal.jsx
@@ -33,8 +33,6 @@ import { CreatableSelect } from 'src/components/Select/SupersetStyledSelect';
import { t } from '@superset-ui/translation';
import ReactMarkdown from 'react-markdown';
-import { EXPLORE_ONLY_VIZ_TYPE } from '../constants';
-
const propTypes = {
can_overwrite: PropTypes.bool,
onHide: PropTypes.func.isRequired,
@@ -134,8 +132,6 @@ class SaveModal extends React.Component {
this.setState({ alert: null });
}
render() {
- const canNotSaveToDash =
- EXPLORE_ONLY_VIZ_TYPE.indexOf(this.state.vizType) > -1;
return (
<Modal show onHide={this.props.onHide}>
<Modal.Header closeButton>
@@ -225,9 +221,7 @@ class SaveModal extends React.Component {
id="btn_modal_save_goto_dash"
bsSize="sm"
disabled={
- canNotSaveToDash ||
- !this.state.newSliceName ||
- !this.state.newDashboardName
+ !this.state.newSliceName || !this.state.newDashboardName
}
onClick={this.saveOrOverwrite.bind(this, true)}
>
diff --git a/superset-frontend/src/explore/components/controls/VizTypeControl.jsx b/superset-frontend/src/explore/components/controls/VizTypeControl.jsx
index 2c67f68..cf5ad64 100644
--- a/superset-frontend/src/explore/components/controls/VizTypeControl.jsx
+++ b/superset-frontend/src/explore/components/controls/VizTypeControl.jsx
@@ -74,7 +74,6 @@ const DEFAULT_ORDER = [
'line_multi',
'treemap',
'box_plot',
- 'separator',
'sunburst',
'sankey',
'word_cloud',
@@ -85,7 +84,6 @@ const DEFAULT_ORDER = [
'bubble',
'deck_geojson',
'horizon',
- 'markup',
'deck_multi',
'compare',
'partition',
@@ -95,7 +93,6 @@ const DEFAULT_ORDER = [
'world_map',
'paired_ttest',
'para',
- 'iframe',
'country_map',
];
diff --git a/superset-frontend/src/explore/constants.js b/superset-frontend/src/explore/constants.js
index 9ef5c9e..b3d3a80 100644
--- a/superset-frontend/src/explore/constants.js
+++ b/superset-frontend/src/explore/constants.js
@@ -71,8 +71,6 @@ export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_d
export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
export const druidAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
-export const EXPLORE_ONLY_VIZ_TYPE = ['separator', 'markup'];
-
export const TIME_FILTER_LABELS = {
time_range: t('Time Range'),
granularity_sqla: t('Time Column'),
diff --git a/superset-frontend/src/visualizations/presets/MainPreset.js b/superset-frontend/src/visualizations/presets/MainPreset.js
index cf582cb..3fcad1a 100644
--- a/superset-frontend/src/visualizations/presets/MainPreset.js
+++ b/superset-frontend/src/visualizations/presets/MainPreset.js
@@ -29,9 +29,7 @@ import ForceDirectedChartPlugin from '@superset-ui/legacy-plugin-chart-force-dir
import HeatmapChartPlugin from '@superset-ui/legacy-plugin-chart-heatmap';
import HistogramChartPlugin from '@superset-ui/legacy-plugin-chart-histogram';
import HorizonChartPlugin from '@superset-ui/legacy-plugin-chart-horizon';
-import IframeChartPlugin from '@superset-ui/legacy-plugin-chart-iframe';
import MapBoxChartPlugin from '@superset-ui/legacy-plugin-chart-map-box';
-import MarkupChartPlugin from '@superset-ui/legacy-plugin-chart-markup';
import PairedTTestChartPlugin from '@superset-ui/legacy-plugin-chart-paired-t-test';
import ParallelCoordinatesChartPlugin from '@superset-ui/legacy-plugin-chart-parallel-coordinates';
import PartitionChartPlugin from '@superset-ui/legacy-plugin-chart-partition';
@@ -87,12 +85,9 @@ export default class MainPreset extends Preset {
new HeatmapChartPlugin().configure({ key: 'heatmap' }),
new HistogramChartPlugin().configure({ key: 'histogram' }),
new HorizonChartPlugin().configure({ key: 'horizon' }),
- new IframeChartPlugin().configure({ key: 'iframe' }),
new LineChartPlugin().configure({ key: 'line' }),
new LineMultiChartPlugin().configure({ key: 'line_multi' }),
new MapBoxChartPlugin().configure({ key: 'mapbox' }),
- new MarkupChartPlugin().configure({ key: 'markup' }),
- new MarkupChartPlugin().configure({ key: 'separator' }),
new PairedTTestChartPlugin().configure({ key: 'paired_ttest' }),
new ParallelCoordinatesChartPlugin().configure({ key: 'para' }),
new PartitionChartPlugin().configure({ key: 'partition' }),
diff --git a/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py b/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py
new file mode 100644
index 0000000..3d03ebe
--- /dev/null
+++ b/superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py
@@ -0,0 +1,198 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Migrate iframe in dashboard to markdown component
+
+Revision ID: 978245563a02
+Revises: f2672aa8350a
+Create Date: 2020-08-12 00:24:39.617899
+
+"""
+import collections
+import json
+import logging
+import uuid
+from collections import defaultdict
+
+from alembic import op
+from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+
+# revision identifiers, used by Alembic.
+revision = "978245563a02"
+down_revision = "f2672aa8350a"
+
+Base = declarative_base()
+
+
+class Slice(Base):
+ """Declarative class to do query in upgrade"""
+
+ __tablename__ = "slices"
+ id = Column(Integer, primary_key=True)
+ params = Column(Text)
+ viz_type = Column(String(250))
+
+
+dashboard_slices = Table(
+ "dashboard_slices",
+ Base.metadata,
+ Column("id", Integer, primary_key=True),
+ Column("dashboard_id", Integer, ForeignKey("dashboards.id")),
+ Column("slice_id", Integer, ForeignKey("slices.id")),
+)
+
+slice_user = Table(
+ "slice_user",
+ Base.metadata,
+ Column("id", Integer, primary_key=True),
+ Column("user_id", Integer, ForeignKey("ab_user.id")),
+ Column("slice_id", Integer, ForeignKey("slices.id")),
+)
+
+
+class Dashboard(Base):
+ __tablename__ = "dashboards"
+ id = Column(Integer, primary_key=True)
+ position_json = Column(Text)
+ slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards")
+
+
+def create_new_markdown_component(chart_position, url):
+ return {
+ "type": "MARKDOWN",
+ "id": "MARKDOWN-{}".format(uuid.uuid4().hex[:8]),
+ "children": [],
+ "parents": chart_position["parents"],
+ "meta": {
+ "width": chart_position["meta"]["width"],
+ "height": chart_position["meta"]["height"],
+ "code": f'<iframe src="{url}" width="100%" height="100%"></iframe>',
+ },
+ }
+
+
+def upgrade():
+ bind = op.get_bind()
+ session = db.Session(bind=bind)
+
+ dash_to_migrate = defaultdict(list)
+ iframe_urls = defaultdict(list)
+
+ try:
+ # find iframe viz_type and its url
+ iframes = session.query(Slice).filter_by(viz_type="iframe").all()
+ iframe_ids = [slc.id for slc in iframes]
+
+ for iframe in iframes:
+ iframe_params = json.loads(iframe.params or "{}")
+ url = iframe_params.get("url")
+ iframe_urls[iframe.id] = url
+
+ # find iframe viz_type that used in dashboard
+ dash_slc = (
+ session.query(dashboard_slices)
+ .filter(dashboard_slices.c.slice_id.in_(iframe_ids))
+ .all()
+ )
+ for entry in dash_slc:
+ dash_to_migrate[entry.dashboard_id].append(entry.slice_id)
+ dashboard_ids = list(dash_to_migrate.keys())
+
+ # replace iframe in dashboard metadata
+ dashboards = (
+ session.query(Dashboard).filter(Dashboard.id.in_(dashboard_ids)).all()
+ )
+ for i, dashboard in enumerate(dashboards):
+ print(
+ f"scanning dashboard ({i + 1}/{len(dashboards)}) dashboard: {dashboard.id} >>>>"
+ )
+
+ # remove iframe slices from dashboard
+ dashboard.slices = [
+ slc for slc in dashboard.slices if slc.id not in iframe_ids
+ ]
+
+ # find iframe chart position in metadata
+ # and replace it with markdown component
+ position_dict = json.loads(dashboard.position_json or "{}")
+ for key, chart_position in position_dict.items():
+ if (
+ chart_position
+ and isinstance(chart_position, dict)
+ and chart_position["type"] == "CHART"
+ and chart_position["meta"]
+ and chart_position["meta"]["chartId"] in iframe_ids
+ ):
+ iframe_id = chart_position["meta"]["chartId"]
+ # make new markdown component
+ markdown = create_new_markdown_component(
+ chart_position, iframe_urls[iframe_id]
+ )
+ position_dict.pop(key)
+ position_dict[markdown["id"]] = markdown
+
+ # add markdown to layout tree
+ parent_id = markdown["parents"][-1]
+ children = position_dict[parent_id]["children"]
+ children.remove(key)
+ children.append(markdown["id"])
+
+ dashboard.position_json = json.dumps(
+ position_dict,
+ indent=None,
+ separators=(",", ":"),
+ sort_keys=True,
+ )
+ session.merge(dashboard)
+
+ # remove iframe, separator and markup charts
+ slices_to_remove = (
+ session.query(Slice)
+ .filter(Slice.viz_type.in_(["iframe", "separator", "markup"]))
+ .all()
+ )
+ slices_ids = [slc.id for slc in slices_to_remove]
+
+ # remove dependencies first
+ session.query(dashboard_slices).filter(
+ dashboard_slices.c.slice_id.in_(slices_ids)
+ ).delete(synchronize_session=False)
+
+ session.query(slice_user).filter(slice_user.c.slice_id.in_(slices_ids)).delete(
+ synchronize_session=False
+ )
+
+ # remove slices
+ session.query(Slice).filter(Slice.id.in_(slices_ids)).delete(
+ synchronize_session=False
+ )
+
+ except Exception as ex:
+ logging.exception(f"dashboard {dashboard.id} has error: {ex}")
+
+ session.commit()
+ session.close()
+
+
+def downgrade():
+ # note: this upgrade is irreversible.
+ # this migration removed all iframe, separator, and markup type slices,
+ # and Superset will not support these 3 viz_type anymore.
+ pass
diff --git a/superset/migrations/versions/f80a3b88324b_.py b/superset/migrations/versions/f80a3b88324b_.py
new file mode 100644
index 0000000..7defdc5
--- /dev/null
+++ b/superset/migrations/versions/f80a3b88324b_.py
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""empty message
+
+Revision ID: f80a3b88324b
+Revises: ('978245563a02', 'f120347acb39')
+Create Date: 2020-08-12 15:47:56.580191
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "f80a3b88324b"
+down_revision = ("978245563a02", "f120347acb39")
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+ pass
+
+
+def downgrade():
+ pass
diff --git a/superset/tasks/slack_util.py b/superset/tasks/slack_util.py
index 865aa59..64e9923 100644
--- a/superset/tasks/slack_util.py
+++ b/superset/tasks/slack_util.py
@@ -26,7 +26,7 @@ from slack.web.slack_response import SlackResponse
from superset import app
# Globals
-config = app.config # type: ignore
+config = app.config
logger = logging.getLogger("tasks.slack_util")
diff --git a/superset/viz.py b/superset/viz.py
index 14eedf0..5dc05ed 100644
--- a/superset/viz.py
+++ b/superset/viz.py
@@ -2066,25 +2066,6 @@ class FilterBoxViz(BaseViz):
return d
-class IFrameViz(BaseViz):
-
- """You can squeeze just about anything in this iFrame component"""
-
- viz_type = "iframe"
- verbose_name = _("iFrame")
- credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
- is_timeseries = False
-
- def query_obj(self) -> QueryObjectDict:
- return {}
-
- def get_df(self, query_obj: Optional[QueryObjectDict] = None) -> pd.DataFrame:
- return pd.DataFrame()
-
- def get_data(self, df: pd.DataFrame) -> VizData:
- return {"iframe": True}
-
-
class ParallelCoordinatesViz(BaseViz):
"""Interactive parallel coordinate implementation