You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by gr...@apache.org on 2018/11/30 18:30:12 UTC
[incubator-superset] branch master updated: allow domain sharding
in frontend (#5039)
This is an automated email from the ASF dual-hosted git repository.
graceguo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 8e14e0b allow domain sharding in frontend (#5039)
8e14e0b is described below
commit 8e14e0bd67d439f1f5d33971713594baaedf6b62
Author: Grace Guo <gr...@airbnb.com>
AuthorDate: Fri Nov 30 10:30:04 2018 -0800
allow domain sharding in frontend (#5039)
---
docs/installation.rst | 12 ++++
.../assets/spec/javascripts/explore/utils_spec.jsx | 68 ++++++++++++++++++++++
superset/assets/src/chart/chartAction.js | 14 ++++-
superset/assets/src/explore/exploreUtils.js | 27 ++++++++-
superset/assets/src/utils/hostNamesConfig.js | 23 ++++++++
superset/config.py | 7 +++
superset/views/base.py | 1 +
7 files changed, 148 insertions(+), 4 deletions(-)
diff --git a/docs/installation.rst b/docs/installation.rst
index 295a20d..7224cd2 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -559,6 +559,18 @@ The following keys in `superset_config.py` can be specified to configure CORS:
* ``CORS_OPTIONS``: options passed to Flask-CORS (`documentation <http://flask-cors.corydolphin.com/en/latest/api.html#extension>`)
+DOMAIN SHARDING
+---------------
+
+Chrome allows up to 6 open connections per domain at a time. When there are more
+than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for
+next available socket. PR (`#5039 <https://github.com/apache/incubator-superset/pull/5039>`) adds domain sharding to Superset,
+and this feature will be enabled by configuration only (by default Superset
+doesn't allow cross-domain request).
+
+*``SUPERSET_WEBSERVER_DOMAINS``: list of allowed hostnames for domain sharding feature. default `None`
+
+
MIDDLEWARE
----------
diff --git a/superset/assets/spec/javascripts/explore/utils_spec.jsx b/superset/assets/spec/javascripts/explore/utils_spec.jsx
index 633f7ce..323655d 100644
--- a/superset/assets/spec/javascripts/explore/utils_spec.jsx
+++ b/superset/assets/spec/javascripts/explore/utils_spec.jsx
@@ -1,5 +1,8 @@
+import sinon from 'sinon';
+
import URI from 'urijs';
import { getExploreUrlAndPayload, getExploreLongUrl } from '../../../src/explore/exploreUtils';
+import * as hostNamesConfig from '../../../src/utils/hostNamesConfig';
describe('exploreUtils', () => {
const location = window.location;
@@ -128,6 +131,71 @@ describe('exploreUtils', () => {
});
});
+ describe('domain sharding', () => {
+ let stub;
+ const availableDomains = [
+ 'http://localhost/',
+ 'domain1.com', 'domain2.com', 'domain3.com',
+ ];
+ beforeEach(() => {
+ stub = sinon.stub(hostNamesConfig, 'availableDomains').value(availableDomains);
+ });
+ afterEach(() => {
+ stub.restore();
+ });
+
+ it('generate url to different domains', () => {
+ let url = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'json',
+ allowDomainSharding: true,
+ }).url;
+ expect(url).toMatch(availableDomains[0]);
+
+ url = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'json',
+ allowDomainSharding: true,
+ }).url;
+ expect(url).toMatch(availableDomains[1]);
+
+ url = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'json',
+ allowDomainSharding: true,
+ }).url;
+ expect(url).toMatch(availableDomains[2]);
+
+ url = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'json',
+ allowDomainSharding: true,
+ }).url;
+ expect(url).toMatch(availableDomains[3]);
+
+ // circle back to first available domain
+ url = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'json',
+ allowDomainSharding: true,
+ }).url;
+ expect(url).toMatch(availableDomains[0]);
+ });
+ it('not generate url to different domains without flag', () => {
+ let csvURL = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'csv',
+ }).url;
+ expect(csvURL).toMatch(availableDomains[0]);
+
+ csvURL = getExploreUrlAndPayload({
+ formData,
+ endpointType: 'csv',
+ }).url;
+ expect(csvURL).toMatch(availableDomains[0]);
+ });
+ });
+
describe('getExploreLongUrl', () => {
it('generates proper base url with form_data', () => {
compareURI(
diff --git a/superset/assets/src/chart/chartAction.js b/superset/assets/src/chart/chartAction.js
index b816d87..6c6ddab 100644
--- a/superset/assets/src/chart/chartAction.js
+++ b/superset/assets/src/chart/chartAction.js
@@ -8,6 +8,7 @@ import { requiresQuery, ANNOTATION_SOURCE_TYPES } from '../modules/AnnotationTyp
import { addDangerToast } from '../messageToasts/actions';
import { Logger, LOG_ACTIONS_LOAD_CHART } from '../logger';
import getClientErrorObject from '../utils/getClientErrorObject';
+import { allowCrossDomain } from '../utils/hostNamesConfig';
export const CHART_UPDATE_STARTED = 'CHART_UPDATE_STARTED';
export function chartUpdateStarted(queryController, latestQueryFormData, key) {
@@ -145,6 +146,7 @@ export function runQuery(formData, force = false, timeout = 60, key) {
formData,
endpointType: 'json',
force,
+ allowDomainSharding: true,
});
const logStart = Logger.getTimestamp();
const controller = new AbortController();
@@ -152,12 +154,20 @@ export function runQuery(formData, force = false, timeout = 60, key) {
dispatch(chartUpdateStarted(controller, payload, key));
- const queryPromise = SupersetClient.post({
+ let querySettings = {
url,
postPayload: { form_data: payload },
signal,
timeout: timeout * 1000,
- })
+ };
+ if (allowCrossDomain) {
+ querySettings = {
+ ...querySettings,
+ mode: 'cors',
+ credentials: 'include',
+ };
+ }
+ const queryPromise = SupersetClient.post(querySettings)
.then(({ json }) => {
Logger.append(LOG_ACTIONS_LOAD_CHART, {
slice_id: key,
diff --git a/superset/assets/src/explore/exploreUtils.js b/superset/assets/src/explore/exploreUtils.js
index dcf562d..4eac3ab 100644
--- a/superset/assets/src/explore/exploreUtils.js
+++ b/superset/assets/src/explore/exploreUtils.js
@@ -1,11 +1,23 @@
/* eslint camelcase: 0 */
import URI from 'urijs';
+import { availableDomains } from '../utils/hostNamesConfig';
export function getChartKey(explore) {
const slice = explore.slice;
return slice ? (slice.slice_id) : 0;
}
+let requestCounter = 0;
+function getHostName(allowDomainSharding = false) {
+ let currentIndex = 0;
+ if (allowDomainSharding) {
+ currentIndex = requestCounter % availableDomains.length;
+ requestCounter += 1;
+ }
+
+ return availableDomains[currentIndex];
+}
+
export function getAnnotationJsonUrl(slice_id, form_data, isNative) {
if (slice_id === null || slice_id === undefined) {
return null;
@@ -49,6 +61,7 @@ export function getExploreUrlAndPayload({
force = false,
curUrl = null,
requestParams = {},
+ allowDomainSharding = false,
}) {
if (!formData.datasource) {
return null;
@@ -57,7 +70,13 @@ export function getExploreUrlAndPayload({
// The search params from the window.location are carried through,
// but can be specified with curUrl (used for unit tests to spoof
// the window.location).
- let uri = new URI([location.protocol, '//', location.host].join(''));
+ let uri = new URI({
+ protocol: location.protocol.slice(0, -1),
+ hostname: getHostName(allowDomainSharding),
+ port: location.port ? location.port : '',
+ path: '/',
+ });
+
if (curUrl) {
uri = URI(URI(curUrl).search());
}
@@ -105,7 +124,11 @@ export function getExploreUrlAndPayload({
}
export function exportChart(formData, endpointType) {
- const { url, payload } = getExploreUrlAndPayload({ formData, endpointType });
+ const { url, payload } = getExploreUrlAndPayload({
+ formData,
+ endpointType,
+ allowDomainSharding: false,
+ });
const exploreForm = document.createElement('form');
exploreForm.action = url;
diff --git a/superset/assets/src/utils/hostNamesConfig.js b/superset/assets/src/utils/hostNamesConfig.js
new file mode 100644
index 0000000..8f381bb
--- /dev/null
+++ b/superset/assets/src/utils/hostNamesConfig.js
@@ -0,0 +1,23 @@
+function getDomainsConfig() {
+ const appContainer = document.getElementById('app');
+ if (!appContainer) {
+ return [];
+ }
+
+ const bootstrapData = JSON.parse(appContainer.getAttribute('data-bootstrap'));
+ const availableDomains = new Set([location.hostname]);
+ if (bootstrapData &&
+ bootstrapData.common &&
+ bootstrapData.common.conf &&
+ bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS
+ ) {
+ bootstrapData.common.conf.SUPERSET_WEBSERVER_DOMAINS.forEach((hostName) => {
+ availableDomains.add(hostName);
+ });
+ }
+ return Array.from(availableDomains);
+}
+
+export const availableDomains = getDomainsConfig();
+
+export const allowCrossDomain = availableDomains.length > 1;
diff --git a/superset/config.py b/superset/config.py
index a34e5d9..a9622db 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -194,6 +194,13 @@ TABLE_NAMES_CACHE_CONFIG = {'CACHE_TYPE': 'null'}
ENABLE_CORS = False
CORS_OPTIONS = {}
+# Chrome allows up to 6 open connections per domain at a time. When there are more
+# than 6 slices in dashboard, a lot of time fetch requests are queued up and wait for
+# next available socket. PR #5039 is trying to allow domain sharding for Superset,
+# and this feature will be enabled by configuration only (by default Superset
+# doesn't allow cross-domain request).
+SUPERSET_WEBSERVER_DOMAINS = None
+
# Allowed format types for upload on Database view
# TODO: Add processing of other spreadsheet formats (xls, xlsx etc)
ALLOWED_EXTENSIONS = set(['csv'])
diff --git a/superset/views/base.py b/superset/views/base.py
index 31ab1ce..7057a98 100644
--- a/superset/views/base.py
+++ b/superset/views/base.py
@@ -26,6 +26,7 @@ FRONTEND_CONF_KEYS = (
'ENABLE_JAVASCRIPT_CONTROLS',
'DEFAULT_SQLLAB_LIMIT',
'SQL_MAX_ROW',
+ 'SUPERSET_WEBSERVER_DOMAINS',
)