You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by gr...@apache.org on 2018/08/03 17:55:11 UTC
[incubator-superset] branch master updated: Reduce dashboard
position_json data size (#5543)
This is an automated email from the ASF dual-hosted git repository.
graceguo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 2e2c980 Reduce dashboard position_json data size (#5543)
2e2c980 is described below
commit 2e2c9806b21b97f349b8180b5dbf72b0ad5c4e12
Author: Grace Guo <gr...@airbnb.com>
AuthorDate: Fri Aug 3 10:55:08 2018 -0700
Reduce dashboard position_json data size (#5543)
---
.../util/findFirstParentContainer_spec.js | 151 ++++++++++-----------
.../assets/src/dashboard/components/Header.jsx | 26 +++-
.../src/dashboard/containers/DashboardHeader.jsx | 7 +-
.../assets/src/dashboard/util/componentTypes.js | 24 ++--
superset/assets/src/dashboard/util/constants.js | 10 +-
superset/config.py | 1 +
superset/migrations/versions/7fcdcde0761c_.py | 69 ++++++++++
superset/views/base.py | 1 +
superset/views/core.py | 3 +
9 files changed, 197 insertions(+), 95 deletions(-)
diff --git a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
index 4ab29bd..ecaca67 100644
--- a/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
+++ b/superset/assets/spec/javascripts/dashboard/util/findFirstParentContainer_spec.js
@@ -10,94 +10,91 @@ import {
describe('findFirstParentContainer', () => {
const mockGridLayout = {
DASHBOARD_VERSION_KEY: 'v2',
- DASHBOARD_ROOT_ID: {
- type: 'DASHBOARD_ROOT_TYPE',
- id: 'DASHBOARD_ROOT_ID',
- children: ['DASHBOARD_GRID_ID'],
- },
- DASHBOARD_GRID_ID: {
- type: 'DASHBOARD_GRID_TYPE',
- id: 'DASHBOARD_GRID_ID',
- children: ['DASHBOARD_ROW_TYPE-Bk45URrlQ'],
- },
- 'DASHBOARD_ROW_TYPE-Bk45URrlQ': {
- type: 'DASHBOARD_ROW_TYPE',
- id: 'DASHBOARD_ROW_TYPE-Bk45URrlQ',
- children: ['DASHBOARD_CHART_TYPE-ryxVc8RHlX'],
- },
- 'DASHBOARD_CHART_TYPE-ryxVc8RHlX': {
- type: 'DASHBOARD_CHART_TYPE',
- id: 'DASHBOARD_CHART_TYPE-ryxVc8RHlX',
+ ROOT_ID: {
+ type: 'ROOT',
+ id: 'ROOT_ID',
+ children: ['GRID_ID'],
+ },
+ GRID_ID: {
+ type: 'GRID',
+ id: 'GRID_ID',
+ children: ['ROW-Bk45URrlQ'],
+ },
+ 'ROW-Bk45URrlQ': {
+ type: 'ROW',
+ id: 'ROW-Bk45URrlQ',
+ children: ['CHART-ryxVc8RHlX'],
+ },
+ 'CHART-ryxVc8RHlX': {
+ type: 'CHART',
+ id: 'CHART-ryxVc8RHlX',
children: [],
},
- DASHBOARD_HEADER_ID: {
- id: 'DASHBOARD_HEADER_ID',
- type: 'DASHBOARD_HEADER_TYPE',
+ HEADER_ID: {
+ id: 'HEADER_ID',
+ type: 'HEADER',
},
};
const mockTabsLayout = {
- 'DASHBOARD_CHART_TYPE-S1gilYABe7': {
+ 'CHART-S1gilYABe7': {
children: [],
- id: 'DASHBOARD_CHART_TYPE-S1gilYABe7',
- type: 'DASHBOARD_CHART_TYPE',
+ id: 'CHART-S1gilYABe7',
+ type: 'CHART',
},
- 'DASHBOARD_CHART_TYPE-SJli5K0HlQ': {
+ 'CHART-SJli5K0HlQ': {
children: [],
- id: 'DASHBOARD_CHART_TYPE-SJli5K0HlQ',
- type: 'DASHBOARD_CHART_TYPE',
+ id: 'CHART-SJli5K0HlQ',
+ type: 'CHART',
},
- DASHBOARD_GRID_ID: {
+ GRID_ID: {
children: [],
- id: 'DASHBOARD_GRID_ID',
- type: 'DASHBOARD_GRID_TYPE',
- },
- DASHBOARD_HEADER_ID: {
- id: 'DASHBOARD_HEADER_ID',
- type: 'DASHBOARD_HEADER_TYPE',
- },
- DASHBOARD_ROOT_ID: {
- children: ['DASHBOARD_TABS_TYPE-SkgJ5t0Bem'],
- id: 'DASHBOARD_ROOT_ID',
- type: 'DASHBOARD_ROOT_TYPE',
- },
- 'DASHBOARD_ROW_TYPE-S1B8-JLgX': {
- children: ['DASHBOARD_CHART_TYPE-SJli5K0HlQ'],
- id: 'DASHBOARD_ROW_TYPE-S1B8-JLgX',
- type: 'DASHBOARD_ROW_TYPE',
- },
- 'DASHBOARD_ROW_TYPE-S1bUb1Ilm': {
- children: ['DASHBOARD_CHART_TYPE-S1gilYABe7'],
- id: 'DASHBOARD_ROW_TYPE-S1bUb1Ilm',
- type: 'DASHBOARD_ROW_TYPE',
- },
- 'DASHBOARD_TABS_TYPE-ByeLSWyLe7': {
- children: ['DASHBOARD_TAB_TYPE-BJbLSZ1UeQ'],
- id: 'DASHBOARD_TABS_TYPE-ByeLSWyLe7',
- type: 'DASHBOARD_TABS_TYPE',
- },
- 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem': {
- children: [
- 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
- 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
- ],
- id: 'DASHBOARD_TABS_TYPE-SkgJ5t0Bem',
+ id: 'GRID_ID',
+ type: 'GRID',
+ },
+ HEADER_ID: {
+ id: 'HEADER_ID',
+ type: 'HEADER',
+ },
+ ROOT_ID: {
+ children: ['TABS-SkgJ5t0Bem'],
+ id: 'ROOT_ID',
+ type: 'ROOT',
+ },
+ 'ROW-S1B8-JLgX': {
+ children: ['CHART-SJli5K0HlQ'],
+ id: 'ROW-S1B8-JLgX',
+ type: 'ROW',
+ },
+ 'ROW-S1bUb1Ilm': {
+ children: ['CHART-S1gilYABe7'],
+ id: 'ROW-S1bUb1Ilm',
+ type: 'ROW',
+ },
+ 'TABS-ByeLSWyLe7': {
+ children: ['TAB-BJbLSZ1UeQ'],
+ id: 'TABS-ByeLSWyLe7',
+ type: 'TABS',
+ },
+ 'TABS-SkgJ5t0Bem': {
+ children: ['TAB-HkWJcFCHxQ', 'TAB-ByDBbkLlQ'],
+ id: 'TABS-SkgJ5t0Bem',
meta: {},
- type: 'DASHBOARD_TABS_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ': {
- children: ['DASHBOARD_ROW_TYPE-S1bUb1Ilm'],
- id: 'DASHBOARD_TAB_TYPE-BJbLSZ1UeQ',
- type: 'DASHBOARD_TAB_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-ByDBbkLlQ': {
- children: ['DASHBOARD_ROW_TYPE-S1B8-JLgX'],
- id: 'DASHBOARD_TAB_TYPE-ByDBbkLlQ',
- type: 'DASHBOARD_TAB_TYPE',
- },
- 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ': {
- children: ['DASHBOARD_TABS_TYPE-ByeLSWyLe7'],
- id: 'DASHBOARD_TAB_TYPE-HkWJcFCHxQ',
- type: 'DASHBOARD_TAB_TYPE',
+ type: 'TABS',
+ },
+ 'TAB-BJbLSZ1UeQ': {
+ children: ['ROW-S1bUb1Ilm'],
+ id: 'TAB-BJbLSZ1UeQ',
+ type: 'TAB',
+ },
+ 'TAB-ByDBbkLlQ': {
+ children: ['ROW-S1B8-JLgX'],
+ id: 'TAB-ByDBbkLlQ',
+ type: 'TAB',
+ },
+ 'TAB-HkWJcFCHxQ': {
+ children: ['TABS-ByeLSWyLe7'],
+ id: 'TAB-HkWJcFCHxQ',
+ type: 'TAB',
},
DASHBOARD_VERSION_KEY: 'v2',
};
diff --git a/superset/assets/src/dashboard/components/Header.jsx b/superset/assets/src/dashboard/components/Header.jsx
index 9f976cb..bc66a01 100644
--- a/superset/assets/src/dashboard/components/Header.jsx
+++ b/superset/assets/src/dashboard/components/Header.jsx
@@ -10,11 +10,16 @@ import UndoRedoKeylisteners from './UndoRedoKeylisteners';
import { chartPropShape } from '../util/propShapes';
import { t } from '../../locales';
-import { UNDO_LIMIT, SAVE_TYPE_OVERWRITE } from '../util/constants';
+import {
+ UNDO_LIMIT,
+ SAVE_TYPE_OVERWRITE,
+ DASHBOARD_POSITION_DATA_LIMIT,
+} from '../util/constants';
const propTypes = {
addSuccessToast: PropTypes.func.isRequired,
addDangerToast: PropTypes.func.isRequired,
+ addWarningToast: PropTypes.func.isRequired,
dashboardInfo: PropTypes.object.isRequired,
dashboardTitle: PropTypes.string.isRequired,
charts: PropTypes.objectOf(chartPropShape).isRequired,
@@ -143,7 +148,24 @@ class Header extends React.PureComponent {
default_filters: JSON.stringify(filters),
};
- this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+ // make sure positions data less than DB storage limitation:
+ const positionJSONLength = JSON.stringify(positions).length;
+ const limit =
+ dashboardInfo.common.conf.SUPERSET_DASHBOARD_POSITION_DATA_LIMIT ||
+ DASHBOARD_POSITION_DATA_LIMIT;
+ if (positionJSONLength >= limit) {
+ this.props.addDangerToast(
+ t(
+ 'Your dashboard is too large. Please reduce the size before save it.',
+ ),
+ );
+ } else {
+ if (positionJSONLength >= limit * 0.9) {
+ this.props.addWarningToast('Your dashboard is near the size limit.');
+ }
+
+ this.props.onSave(data, dashboardInfo.id, SAVE_TYPE_OVERWRITE);
+ }
}
render() {
diff --git a/superset/assets/src/dashboard/containers/DashboardHeader.jsx b/superset/assets/src/dashboard/containers/DashboardHeader.jsx
index 629b916..3e54c62 100644
--- a/superset/assets/src/dashboard/containers/DashboardHeader.jsx
+++ b/superset/assets/src/dashboard/containers/DashboardHeader.jsx
@@ -23,7 +23,11 @@ import {
updateDashboardTitle,
} from '../actions/dashboardLayout';
-import { addSuccessToast, addDangerToast } from '../../messageToasts/actions';
+import {
+ addSuccessToast,
+ addDangerToast,
+ addWarningToast,
+} from '../../messageToasts/actions';
import { DASHBOARD_HEADER_ID } from '../util/constants';
@@ -59,6 +63,7 @@ function mapDispatchToProps(dispatch) {
{
addSuccessToast,
addDangerToast,
+ addWarningToast,
onUndo: undoLayoutAction,
onRedo: redoLayoutAction,
setEditMode,
diff --git a/superset/assets/src/dashboard/util/componentTypes.js b/superset/assets/src/dashboard/util/componentTypes.js
index b773417..47478e6 100644
--- a/superset/assets/src/dashboard/util/componentTypes.js
+++ b/superset/assets/src/dashboard/util/componentTypes.js
@@ -1,15 +1,15 @@
-export const CHART_TYPE = 'DASHBOARD_CHART_TYPE';
-export const COLUMN_TYPE = 'DASHBOARD_COLUMN_TYPE';
-export const DASHBOARD_HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const DASHBOARD_GRID_TYPE = 'DASHBOARD_GRID_TYPE';
-export const DASHBOARD_ROOT_TYPE = 'DASHBOARD_ROOT_TYPE';
-export const DIVIDER_TYPE = 'DASHBOARD_DIVIDER_TYPE';
-export const HEADER_TYPE = 'DASHBOARD_HEADER_TYPE';
-export const MARKDOWN_TYPE = 'DASHBOARD_MARKDOWN_TYPE';
-export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE_TYPE';
-export const ROW_TYPE = 'DASHBOARD_ROW_TYPE';
-export const TABS_TYPE = 'DASHBOARD_TABS_TYPE';
-export const TAB_TYPE = 'DASHBOARD_TAB_TYPE';
+export const CHART_TYPE = 'CHART';
+export const COLUMN_TYPE = 'COLUMN';
+export const DASHBOARD_HEADER_TYPE = 'HEADER';
+export const DASHBOARD_GRID_TYPE = 'GRID';
+export const DASHBOARD_ROOT_TYPE = 'ROOT';
+export const DIVIDER_TYPE = 'DIVIDER';
+export const HEADER_TYPE = 'HEADER';
+export const MARKDOWN_TYPE = 'MARKDOWN';
+export const NEW_COMPONENT_SOURCE_TYPE = 'NEW_COMPONENT_SOURCE';
+export const ROW_TYPE = 'ROW';
+export const TABS_TYPE = 'TABS';
+export const TAB_TYPE = 'TAB';
export default {
CHART_TYPE,
diff --git a/superset/assets/src/dashboard/util/constants.js b/superset/assets/src/dashboard/util/constants.js
index 4fd5e40..b26cbff 100644
--- a/superset/assets/src/dashboard/util/constants.js
+++ b/superset/assets/src/dashboard/util/constants.js
@@ -1,7 +1,7 @@
// Ids
-export const DASHBOARD_GRID_ID = 'DASHBOARD_GRID_ID';
-export const DASHBOARD_HEADER_ID = 'DASHBOARD_HEADER_ID';
-export const DASHBOARD_ROOT_ID = 'DASHBOARD_ROOT_ID';
+export const DASHBOARD_GRID_ID = 'GRID_ID';
+export const DASHBOARD_HEADER_ID = 'HEADER_ID';
+export const DASHBOARD_ROOT_ID = 'ROOT_ID';
export const DASHBOARD_VERSION_KEY = 'DASHBOARD_VERSION_KEY';
export const NEW_COMPONENTS_SOURCE_ID = 'NEW_COMPONENTS_SOURCE_ID';
@@ -40,3 +40,7 @@ export const UNDO_LIMIT = 50;
// save dash options
export const SAVE_TYPE_OVERWRITE = 'overwrite';
export const SAVE_TYPE_NEWDASHBOARD = 'newDashboard';
+
+// default dashboard layout data size limit
+// could be overwritten by server-side config
+export const DASHBOARD_POSITION_DATA_LIMIT = 65535;
diff --git a/superset/config.py b/superset/config.py
index 4e31358..ca9fcbd 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -49,6 +49,7 @@ SUPERSET_CELERY_WORKERS = 32 # deprecated
SUPERSET_WEBSERVER_ADDRESS = '0.0.0.0'
SUPERSET_WEBSERVER_PORT = 8088
SUPERSET_WEBSERVER_TIMEOUT = 60 # deprecated
+SUPERSET_DASHBOARD_POSITION_DATA_LIMIT = 65535
EMAIL_NOTIFICATIONS = False
CUSTOM_SECURITY_MANAGER = None
SQLALCHEMY_TRACK_MODIFICATIONS = False
diff --git a/superset/migrations/versions/7fcdcde0761c_.py b/superset/migrations/versions/7fcdcde0761c_.py
new file mode 100644
index 0000000..793e0fb
--- /dev/null
+++ b/superset/migrations/versions/7fcdcde0761c_.py
@@ -0,0 +1,69 @@
+"""Reduce position_json size by remove extra space and component id prefix
+
+Revision ID: 7fcdcde0761c
+Revises: c18bd4186f15
+Create Date: 2018-08-01 11:47:02.233971
+
+"""
+
+# revision identifiers, used by Alembic.
+import json
+import re
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import (
+ Table, Column,
+ Integer, String, Text, ForeignKey,
+)
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+
+revision = '7fcdcde0761c'
+down_revision = 'c18bd4186f15'
+
+Base = declarative_base()
+
+
+class Dashboard(Base):
+ """Declarative class to do query in upgrade"""
+ __tablename__ = 'dashboards'
+ id = sa.Column(sa.Integer, primary_key=True)
+ dashboard_title = sa.Column(String(500))
+ position_json = sa.Column(sa.Text)
+
+
+def is_v2_dash(positions):
+ return (
+ isinstance(positions, dict) and
+ positions.get('DASHBOARD_VERSION_KEY') == 'v2'
+ )
+
+
+def upgrade():
+ bind = op.get_bind()
+ session = db.Session(bind=bind)
+
+ dashboards = session.query(Dashboard).all()
+ for i, dashboard in enumerate(dashboards):
+ original_text = dashboard.position_json or ''
+ position_json = json.loads(original_text or '{}')
+ if is_v2_dash(position_json):
+ # re-dump the json data and remove leading and trailing white spaces
+ text = json.dumps(
+ position_json, indent=None, separators=(',', ':'), sort_keys=True)
+ # remove DASHBOARD_ and _TYPE prefix/suffix in all the component ids
+ text = re.sub(r'DASHBOARD_(?!VERSION)', '', text)
+ text = text.replace('_TYPE', '')
+
+ dashboard.position_json = text
+ print('dash id:{} position_json size from {} to {}'
+ .format(dashboard.id, len(original_text), len(text)))
+ session.merge(dashboard)
+ session.commit()
+
+
+def downgrade():
+ pass
diff --git a/superset/views/base.py b/superset/views/base.py
index 5d90284..8bcdee4 100644
--- a/superset/views/base.py
+++ b/superset/views/base.py
@@ -26,6 +26,7 @@ from superset.translations.utils import get_language_pack
FRONTEND_CONF_KEYS = (
'SUPERSET_WEBSERVER_TIMEOUT',
+ 'SUPERSET_DASHBOARD_POSITION_DATA_LIMIT',
'ENABLE_JAVASCRIPT_CONTROLS',
)
diff --git a/superset/views/core.py b/superset/views/core.py
index 93e79f3..667bfca 100755
--- a/superset/views/core.py
+++ b/superset/views/core.py
@@ -1643,6 +1643,9 @@ class Superset(BaseSupersetView):
session.merge(slc)
session.flush()
+ # remove leading and trailing white spaces in the dumped json
+ dashboard.position_json = json.dumps(
+ positions, indent=None, separators=(',', ':'), sort_keys=True)
dashboard.position_json = json.dumps(positions, sort_keys=True)
md = dashboard.params_dict
dashboard.css = data.get('css')