You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by el...@apache.org on 2023/04/19 00:51:36 UTC
[superset] branch master updated: chore(api v1): Deprecate datasource/save and datasource/get endpoints (#23678)
This is an automated email from the ASF dual-hosted git repository.
elizabeth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/superset.git
The following commit(s) were added to refs/heads/master by this push:
new 44557f5a23 chore(api v1): Deprecate datasource/save and datasource/get endpoints (#23678)
44557f5a23 is described below
commit 44557f5a23d5a6b8f7d6cc267f0d43337c36cd76
Author: Jack Fragassi <jf...@gmail.com>
AuthorDate: Tue Apr 18 17:51:24 2023 -0700
chore(api v1): Deprecate datasource/save and datasource/get endpoints (#23678)
---
.../superset-ui-chart-controls/src/fixtures.ts | 2 +-
.../superset-ui-chart-controls/src/types.ts | 2 +-
.../test/utils/columnChoices.test.tsx | 2 +-
.../test/utils/defineSavedMetrics.test.tsx | 2 +-
.../Datasource/ChangeDatasourceModal.test.jsx | 4 +-
.../Datasource/ChangeDatasourceModal.tsx | 6 +-
.../components/Datasource/DatasourceModal.test.jsx | 12 ++-
.../src/components/Datasource/DatasourceModal.tsx | 98 +++++++++++++++-------
superset-frontend/src/dashboard/constants.ts | 2 +-
.../src/explore/actions/datasourcesActions.test.ts | 4 +-
.../DatasourceControl/DatasourceControl.test.tsx | 23 +++--
.../src/explore/controlUtils/controlUtils.test.tsx | 2 +-
...etControlValuesCompatibleWithDatasource.test.ts | 2 +-
superset-frontend/src/explore/fixtures.tsx | 4 +-
.../src/utils/getDatasourceUid.test.ts | 2 +-
superset/connectors/base/models.py | 40 ++++++---
superset/connectors/sqla/models.py | 14 +++-
superset/datasets/api.py | 8 ++
superset/datasets/commands/exceptions.py | 17 ++++
superset/datasets/commands/update.py | 11 +++
superset/datasets/schemas.py | 2 +-
superset/views/datasource/views.py | 3 +
tests/integration_tests/datasets/api_tests.py | 50 ++++++++++-
23 files changed, 242 insertions(+), 70 deletions(-)
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts b/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts
index 71c4dd3118..1e5152b2bd 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts
+++ b/superset-frontend/packages/superset-ui-chart-controls/src/fixtures.ts
@@ -20,7 +20,7 @@ import { DatasourceType } from '@superset-ui/core';
import { Dataset } from './types';
export const TestDataset: Dataset = {
- column_format: {},
+ column_formats: {},
columns: [
{
advanced_data_type: undefined,
diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts
index 67582523bc..c26f53b6a2 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/src/types.ts
+++ b/superset-frontend/packages/superset-ui-chart-controls/src/types.ts
@@ -67,7 +67,7 @@ export interface Dataset {
type: DatasourceType;
columns: ColumnMeta[];
metrics: Metric[];
- column_format: Record<string, string>;
+ column_formats: Record<string, string>;
verbose_map: Record<string, string>;
main_dttm_col: string;
// eg. ['["ds", true]', 'ds [asc]']
diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/columnChoices.test.tsx b/superset-frontend/packages/superset-ui-chart-controls/test/utils/columnChoices.test.tsx
index 59f4796a44..e27fa95120 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/columnChoices.test.tsx
+++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/columnChoices.test.tsx
@@ -42,7 +42,7 @@ describe('columnChoices()', () => {
},
],
verbose_map: {},
- column_format: { fiz: 'NUMERIC', about: 'STRING', foo: 'DATE' },
+ column_formats: { fiz: 'NUMERIC', about: 'STRING', foo: 'DATE' },
datasource_name: 'my_datasource',
description: 'this is my datasource',
}),
diff --git a/superset-frontend/packages/superset-ui-chart-controls/test/utils/defineSavedMetrics.test.tsx b/superset-frontend/packages/superset-ui-chart-controls/test/utils/defineSavedMetrics.test.tsx
index 48b000ed17..765412d592 100644
--- a/superset-frontend/packages/superset-ui-chart-controls/test/utils/defineSavedMetrics.test.tsx
+++ b/superset-frontend/packages/superset-ui-chart-controls/test/utils/defineSavedMetrics.test.tsx
@@ -39,7 +39,7 @@ describe('defineSavedMetrics', () => {
time_grain_sqla: 'P1D',
columns: [],
verbose_map: {},
- column_format: {},
+ column_formats: {},
datasource_name: 'my_datasource',
description: 'this is my datasource',
};
diff --git a/superset-frontend/src/components/Datasource/ChangeDatasourceModal.test.jsx b/superset-frontend/src/components/Datasource/ChangeDatasourceModal.test.jsx
index 419af6e3cf..5dbc52c38a 100644
--- a/superset-frontend/src/components/Datasource/ChangeDatasourceModal.test.jsx
+++ b/superset-frontend/src/components/Datasource/ChangeDatasourceModal.test.jsx
@@ -49,7 +49,7 @@ const datasourceData = {
const DATASOURCES_ENDPOINT =
'glob:*/api/v1/dataset/?q=(order_column:changed_on_delta_humanized,order_direction:desc,page:0,page_size:25)';
-const DATASOURCE_ENDPOINT = `glob:*/datasource/get/${datasourceData.type}/${datasourceData.id}`;
+const DATASOURCE_ENDPOINT = `glob:*/api/v1/dataset/${datasourceData.id}`;
const DATASOURCE_PAYLOAD = { new: 'data' };
const INFO_ENDPOINT = 'glob:*/api/v1/dataset/_info?*';
@@ -112,6 +112,6 @@ describe('ChangeDatasourceModal', () => {
});
await waitForComponentToPaint(wrapper);
- expect(fetchMock.calls(/datasource\/get\/table\/7/)).toHaveLength(1);
+ expect(fetchMock.calls(/api\/v1\/dataset\/7/)).toHaveLength(1);
});
});
diff --git a/superset-frontend/src/components/Datasource/ChangeDatasourceModal.tsx b/superset-frontend/src/components/Datasource/ChangeDatasourceModal.tsx
index b5b99e1089..5e837f0a26 100644
--- a/superset-frontend/src/components/Datasource/ChangeDatasourceModal.tsx
+++ b/superset-frontend/src/components/Datasource/ChangeDatasourceModal.tsx
@@ -173,10 +173,12 @@ const ChangeDatasourceModal: FunctionComponent<ChangeDatasourceModalProps> = ({
const handleChangeConfirm = () => {
SupersetClient.get({
- endpoint: `/datasource/get/${confirmedDataset?.type}/${confirmedDataset?.id}/`,
+ endpoint: `/api/v1/dataset/${confirmedDataset?.id}`,
})
.then(({ json }) => {
- onDatasourceSave(json);
+ // eslint-disable-next-line no-param-reassign
+ json.result.type = 'table';
+ onDatasourceSave(json.result);
onChange(`${confirmedDataset?.id}__table`);
})
.catch(response => {
diff --git a/superset-frontend/src/components/Datasource/DatasourceModal.test.jsx b/superset-frontend/src/components/Datasource/DatasourceModal.test.jsx
index 12be350521..7d378902e6 100644
--- a/superset-frontend/src/components/Datasource/DatasourceModal.test.jsx
+++ b/superset-frontend/src/components/Datasource/DatasourceModal.test.jsx
@@ -40,7 +40,8 @@ const datasource = mockDatasource['7__table'];
const SAVE_ENDPOINT = 'glob:*/api/v1/dataset/7';
const SAVE_PAYLOAD = { new: 'data' };
-const SAVE_DATASOURCE_ENDPOINT = 'glob:*/datasource/save/';
+const SAVE_DATASOURCE_ENDPOINT = 'glob:*/api/v1/dataset/7';
+const GET_DATASOURCE_ENDPOINT = SAVE_DATASOURCE_ENDPOINT;
const mockedProps = {
datasource,
@@ -96,7 +97,8 @@ describe('DatasourceModal', () => {
it('saves on confirm', async () => {
const callsP = fetchMock.post(SAVE_ENDPOINT, SAVE_PAYLOAD);
- fetchMock.post(SAVE_DATASOURCE_ENDPOINT, {});
+ fetchMock.put(SAVE_DATASOURCE_ENDPOINT, {});
+ fetchMock.get(GET_DATASOURCE_ENDPOINT, {});
act(() => {
wrapper
.find('button[data-test="datasource-modal-save"]')
@@ -111,7 +113,11 @@ describe('DatasourceModal', () => {
okButton.simulate('click');
});
await waitForComponentToPaint(wrapper);
- const expected = ['http://localhost/datasource/save/'];
+ // one call to PUT, then one to GET
+ const expected = [
+ 'http://localhost/api/v1/dataset/7',
+ 'http://localhost/api/v1/dataset/7',
+ ];
expect(callsP._calls.map(call => call[0])).toEqual(
expected,
); /* eslint no-underscore-dangle: 0 */
diff --git a/superset-frontend/src/components/Datasource/DatasourceModal.tsx b/superset-frontend/src/components/Datasource/DatasourceModal.tsx
index 90135f40f9..b5b698f831 100644
--- a/superset-frontend/src/components/Datasource/DatasourceModal.tsx
+++ b/superset-frontend/src/components/Datasource/DatasourceModal.tsx
@@ -96,39 +96,81 @@ const DatasourceModal: FunctionComponent<DatasourceModalProps> = ({
currentDatasource.schema;
setIsSaving(true);
- SupersetClient.post({
- endpoint: '/datasource/save/',
- postPayload: {
- data: {
- ...currentDatasource,
- cache_timeout:
- currentDatasource.cache_timeout === ''
- ? null
- : currentDatasource.cache_timeout,
- schema,
- metrics: currentDatasource?.metrics?.map(
- (metric: Record<string, unknown>) => ({
- ...metric,
+ SupersetClient.put({
+ endpoint: `/api/v1/dataset/${currentDatasource.id}`,
+ jsonPayload: {
+ table_name: currentDatasource.table_name,
+ database_id: currentDatasource.database?.id,
+ sql: currentDatasource.sql,
+ filter_select_enabled: currentDatasource.filter_select_enabled,
+ fetch_values_predicate: currentDatasource.fetch_values_predicate,
+ schema,
+ description: currentDatasource.description,
+ main_dttm_col: currentDatasource.main_dttm_col,
+ offset: currentDatasource.offset,
+ default_endpoint: currentDatasource.default_endpoint,
+ cache_timeout:
+ currentDatasource.cache_timeout === ''
+ ? null
+ : currentDatasource.cache_timeout,
+ is_sqllab_view: currentDatasource.is_sqllab_view,
+ template_params: currentDatasource.template_params,
+ extra: currentDatasource.extra,
+ is_managed_externally: currentDatasource.is_managed_externally,
+ external_url: currentDatasource.external_url,
+ metrics: currentDatasource?.metrics?.map(
+ (metric: Record<string, unknown>) => {
+ const metricBody: any = {
+ expression: metric.expression,
+ description: metric.description,
+ metric_name: metric.metric_name,
+ metric_type: metric.metric_type,
+ d3format: metric.d3format,
+ verbose_name: metric.verbose_name,
+ warning_text: metric.warning_text,
+ uuid: metric.uuid,
extra: buildExtraJsonObject(metric),
- }),
- ),
- columns: currentDatasource?.columns?.map(
- (column: Record<string, unknown>) => ({
- ...column,
- extra: buildExtraJsonObject(column),
- }),
- ),
- type: currentDatasource.type || currentDatasource.datasource_type,
- owners: currentDatasource.owners.map(
- (o: Record<string, number>) => o.value || o.id,
- ),
- },
+ };
+ if (!Number.isNaN(Number(metric.id))) {
+ metricBody.id = metric.id;
+ }
+ return metricBody;
+ },
+ ),
+ columns: currentDatasource?.columns?.map(
+ (column: Record<string, unknown>) => ({
+ id: column.id,
+ column_name: column.column_name,
+ type: column.type,
+ advanced_data_type: column.advanced_data_type,
+ verbose_name: column.verbose_name,
+ description: column.description,
+ expression: column.expression,
+ filterable: column.filterable,
+ groupby: column.groupby,
+ is_active: column.is_active,
+ is_dttm: column.is_dttm,
+ python_date_format: column.python_date_format,
+ uuid: column.uuid,
+ extra: buildExtraJsonObject(column),
+ }),
+ ),
+ owners: currentDatasource.owners.map(
+ (o: Record<string, number>) => o.value || o.id,
+ ),
},
})
- .then(({ json }) => {
+ .then(() => {
addSuccessToast(t('The dataset has been saved'));
+ return SupersetClient.get({
+ endpoint: `/api/v1/dataset/${currentDatasource?.id}`,
+ });
+ })
+ .then(({ json }) => {
+ // eslint-disable-next-line no-param-reassign
+ json.result.type = 'table';
onDatasourceSave({
- ...json,
+ ...json.result,
owners: currentDatasource.owners,
});
onHide();
diff --git a/superset-frontend/src/dashboard/constants.ts b/superset-frontend/src/dashboard/constants.ts
index 97753abe45..aa9f4d8bd3 100644
--- a/superset-frontend/src/dashboard/constants.ts
+++ b/superset-frontend/src/dashboard/constants.ts
@@ -28,7 +28,7 @@ export const PLACEHOLDER_DATASOURCE: Datasource = {
columns: [],
column_types: [],
metrics: [],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '',
description: '',
diff --git a/superset-frontend/src/explore/actions/datasourcesActions.test.ts b/superset-frontend/src/explore/actions/datasourcesActions.test.ts
index 6317f1f4a6..996758b262 100644
--- a/superset-frontend/src/explore/actions/datasourcesActions.test.ts
+++ b/superset-frontend/src/explore/actions/datasourcesActions.test.ts
@@ -34,7 +34,7 @@ const CURRENT_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']
@@ -47,7 +47,7 @@ const NEW_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']
diff --git a/superset-frontend/src/explore/components/controls/DatasourceControl/DatasourceControl.test.tsx b/superset-frontend/src/explore/components/controls/DatasourceControl/DatasourceControl.test.tsx
index 2c094e72af..80cb84ac8d 100644
--- a/superset-frontend/src/explore/components/controls/DatasourceControl/DatasourceControl.test.tsx
+++ b/superset-frontend/src/explore/components/controls/DatasourceControl/DatasourceControl.test.tsx
@@ -69,9 +69,20 @@ const createProps = (overrides: JsonObject = {}) => ({
});
async function openAndSaveChanges(datasource: any) {
- fetchMock.post('glob:*/datasource/save/', datasource, {
- overwriteRoutes: true,
- });
+ fetchMock.put(
+ 'glob:*/api/v1/dataset/*',
+ {},
+ {
+ overwriteRoutes: true,
+ },
+ );
+ fetchMock.get(
+ 'glob:*/api/v1/dataset/*',
+ { result: datasource },
+ {
+ overwriteRoutes: true,
+ },
+ );
userEvent.click(screen.getByTestId('datasource-menu-trigger'));
userEvent.click(await screen.findByTestId('edit-dataset'));
userEvent.click(await screen.findByTestId('datasource-modal-save'));
@@ -154,7 +165,7 @@ test('Should show SQL Lab for sql_lab role', async () => {
test('Click on Swap dataset option', async () => {
const props = createProps();
- SupersetClientGet.mockImplementation(
+ SupersetClientGet.mockImplementationOnce(
async ({ endpoint }: { endpoint: string }) => {
if (endpoint.includes('_info')) {
return {
@@ -182,7 +193,7 @@ test('Click on Swap dataset option', async () => {
test('Click on Edit dataset', async () => {
const props = createProps();
- SupersetClientGet.mockImplementation(
+ SupersetClientGet.mockImplementationOnce(
async () => ({ json: { result: [] } } as any),
);
render(<DatasourceControl {...props} />, {
@@ -206,7 +217,7 @@ test('Edit dataset should be disabled when user is not admin', async () => {
// @ts-expect-error
props.user.roles = {};
props.datasource.owners = [];
- SupersetClientGet.mockImplementation(
+ SupersetClientGet.mockImplementationOnce(
async () => ({ json: { result: [] } } as any),
);
diff --git a/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx b/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx
index 5feefc3481..e03aba06d1 100644
--- a/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx
+++ b/superset-frontend/src/explore/controlUtils/controlUtils.test.tsx
@@ -51,7 +51,7 @@ describe('controlUtils', () => {
type: DatasourceType.Table,
columns: [{ column_name: 'a' }],
metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: '1__table',
diff --git a/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.test.ts b/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.test.ts
index ad7ccc4801..c8d34f749d 100644
--- a/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.test.ts
+++ b/superset-frontend/src/explore/controlUtils/getControlValuesCompatibleWithDatasource.test.ts
@@ -34,7 +34,7 @@ const sampleDatasource: Dataset = {
{ column_name: 'sample_column_4' },
],
metrics: [{ metric_name: 'saved_metric_2' }],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: 'Sample Dataset',
diff --git a/superset-frontend/src/explore/fixtures.tsx b/superset-frontend/src/explore/fixtures.tsx
index 351e6f26b1..a0f3c112ec 100644
--- a/superset-frontend/src/explore/fixtures.tsx
+++ b/superset-frontend/src/explore/fixtures.tsx
@@ -135,7 +135,7 @@ export const exploreInitialData: ExplorePageInitialData = {
type: DatasourceType.Table,
columns: [{ column_name: 'a' }],
metrics: [{ metric_name: 'first' }, { metric_name: 'second' }],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '',
datasource_name: '8__table',
@@ -153,7 +153,7 @@ export const fallbackExploreInitialData: ExplorePageInitialData = {
type: DatasourceType.Table,
columns: [],
metrics: [],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '',
owners: [],
diff --git a/superset-frontend/src/utils/getDatasourceUid.test.ts b/superset-frontend/src/utils/getDatasourceUid.test.ts
index 2d14fde262..d3a629efcf 100644
--- a/superset-frontend/src/utils/getDatasourceUid.test.ts
+++ b/superset-frontend/src/utils/getDatasourceUid.test.ts
@@ -25,7 +25,7 @@ const TEST_DATASOURCE = {
type: DatasourceType.Table,
columns: [],
metrics: [],
- column_format: {},
+ column_formats: {},
verbose_map: {},
main_dttm_col: '__timestamp',
// eg. ['["ds", true]', 'ds [asc]']
diff --git a/superset/connectors/base/models.py b/superset/connectors/base/models.py
index 30cb5a4030..2cb0d54c51 100644
--- a/superset/connectors/base/models.py
+++ b/superset/connectors/base/models.py
@@ -19,7 +19,18 @@ from __future__ import annotations
import json
from datetime import datetime
from enum import Enum
-from typing import Any, Dict, Hashable, List, Optional, Set, Type, TYPE_CHECKING, Union
+from typing import (
+ Any,
+ Dict,
+ Hashable,
+ List,
+ Optional,
+ Set,
+ Tuple,
+ Type,
+ TYPE_CHECKING,
+ Union,
+)
from flask_appbuilder.security.sqla.models import User
from flask_babel import gettext as __
@@ -242,26 +253,33 @@ class BaseDatasource(
pass
@property
- def data(self) -> Dict[str, Any]:
- """Data representation of the datasource sent to the frontend"""
- order_by_choices = []
+ def order_by_choices(self) -> List[Tuple[str, str]]:
+ choices = []
# self.column_names return sorted column_names
for column_name in self.column_names:
column_name = str(column_name or "")
- order_by_choices.append(
+ choices.append(
(json.dumps([column_name, True]), f"{column_name} " + __("[asc]"))
)
- order_by_choices.append(
+ choices.append(
(json.dumps([column_name, False]), f"{column_name} " + __("[desc]"))
)
+ return choices
- verbose_map = {"__timestamp": "Time"}
- verbose_map.update(
+ @property
+ def verbose_map(self) -> Dict[str, str]:
+ verb_map = {"__timestamp": "Time"}
+ verb_map.update(
{o.metric_name: o.verbose_name or o.metric_name for o in self.metrics}
)
- verbose_map.update(
+ verb_map.update(
{o.column_name: o.verbose_name or o.column_name for o in self.columns}
)
+ return verb_map
+
+ @property
+ def data(self) -> Dict[str, Any]:
+ """Data representation of the datasource sent to the frontend"""
return {
# simple fields
"id": self.id,
@@ -288,9 +306,9 @@ class BaseDatasource(
"columns": [o.data for o in self.columns],
"metrics": [o.data for o in self.metrics],
# TODO deprecate, move logic to JS
- "order_by_choices": order_by_choices,
+ "order_by_choices": self.order_by_choices,
"owners": [owner.id for owner in self.owners],
- "verbose_map": verbose_map,
+ "verbose_map": self.verbose_map,
"select_star": self.select_star,
}
diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py
index 6180a546e7..1a5dd0037e 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -785,14 +785,20 @@ class SqlaTable(
check = config["DATASET_HEALTH_CHECK"]
return check(self) if check else None
+ @property
+ def granularity_sqla(self) -> List[Tuple[Any, Any]]:
+ return utils.choicify(self.dttm_cols)
+
+ @property
+ def time_grain_sqla(self) -> List[Tuple[Any, Any]]:
+ return [(g.duration, g.name) for g in self.database.grains() or []]
+
@property
def data(self) -> Dict[str, Any]:
data_ = super().data
if self.type == "table":
- data_["granularity_sqla"] = utils.choicify(self.dttm_cols)
- data_["time_grain_sqla"] = [
- (g.duration, g.name) for g in self.database.grains() or []
- ]
+ data_["granularity_sqla"] = self.granularity_sqla
+ data_["time_grain_sqla"] = self.time_grain_sqla
data_["main_dttm_col"] = self.main_dttm_col
data_["fetch_values_predicate"] = self.fetch_values_predicate
data_["template_params"] = self.template_params
diff --git a/superset/datasets/api.py b/superset/datasets/api.py
index 48c429d32d..12de82c780 100644
--- a/superset/datasets/api.py
+++ b/superset/datasets/api.py
@@ -195,6 +195,14 @@ class DatasetRestApi(BaseSupersetModelRestApi):
"database.backend",
"columns.advanced_data_type",
"is_managed_externally",
+ "uid",
+ "datasource_name",
+ "name",
+ "column_formats",
+ "granularity_sqla",
+ "time_grain_sqla",
+ "order_by_choices",
+ "verbose_map",
]
add_model_schema = DatasetPostSchema()
edit_model_schema = DatasetPutSchema()
diff --git a/superset/datasets/commands/exceptions.py b/superset/datasets/commands/exceptions.py
index 91af2fdde4..e06e92802f 100644
--- a/superset/datasets/commands/exceptions.py
+++ b/superset/datasets/commands/exceptions.py
@@ -61,6 +61,23 @@ class DatasetExistsValidationError(ValidationError):
)
+class DatasetEndpointUnsafeValidationError(ValidationError):
+ """
+ Marshmallow validation error for unsafe dataset default endpoint
+ """
+
+ def __init__(self) -> None:
+ super().__init__(
+ [
+ _(
+ "The submitted URL is not considered safe,"
+ " only use URLs with the same domain as Superset."
+ )
+ ],
+ field_name="default_endpoint",
+ )
+
+
class DatasetColumnNotFoundValidationError(ValidationError):
"""
Marshmallow validation error when dataset column for update does not exist
diff --git a/superset/datasets/commands/update.py b/superset/datasets/commands/update.py
index 483a98e76c..a2e483ba93 100644
--- a/superset/datasets/commands/update.py
+++ b/superset/datasets/commands/update.py
@@ -18,6 +18,7 @@ import logging
from collections import Counter
from typing import Any, Dict, List, Optional
+from flask import current_app
from flask_appbuilder.models.sqla import Model
from marshmallow import ValidationError
@@ -30,6 +31,7 @@ from superset.datasets.commands.exceptions import (
DatasetColumnNotFoundValidationError,
DatasetColumnsDuplicateValidationError,
DatasetColumnsExistsValidationError,
+ DatasetEndpointUnsafeValidationError,
DatasetExistsValidationError,
DatasetForbiddenError,
DatasetInvalidError,
@@ -41,6 +43,7 @@ from superset.datasets.commands.exceptions import (
)
from superset.datasets.dao import DatasetDAO
from superset.exceptions import SupersetSecurityException
+from superset.utils.urls import is_safe_url
logger = logging.getLogger(__name__)
@@ -101,6 +104,14 @@ class UpdateDatasetCommand(UpdateMixin, BaseCommand):
self._properties["owners"] = owners
except ValidationError as ex:
exceptions.append(ex)
+ # Validate default URL safety
+ default_endpoint = self._properties.get("default_endpoint")
+ if (
+ default_endpoint
+ and not is_safe_url(default_endpoint)
+ and current_app.config["PREVENT_UNSAFE_DEFAULT_URLS_ON_DATASET"]
+ ):
+ exceptions.append(DatasetEndpointUnsafeValidationError())
# Validate columns
columns = self._properties.get("columns")
diff --git a/superset/datasets/schemas.py b/superset/datasets/schemas.py
index 103359a2c3..1d49bc1cfb 100644
--- a/superset/datasets/schemas.py
+++ b/superset/datasets/schemas.py
@@ -55,7 +55,7 @@ class DatasetColumnsPutSchema(Schema):
extra = fields.String(allow_none=True)
filterable = fields.Boolean()
groupby = fields.Boolean()
- is_active = fields.Boolean()
+ is_active = fields.Boolean(allow_none=True)
is_dttm = fields.Boolean(default=False)
python_date_format = fields.String(
allow_none=True, validate=[Length(1, 255), validate_python_date_format]
diff --git a/superset/views/datasource/views.py b/superset/views/datasource/views.py
index 4f158e8369..77cb47d199 100644
--- a/superset/views/datasource/views.py
+++ b/superset/views/datasource/views.py
@@ -44,6 +44,7 @@ from superset.utils.urls import is_safe_url
from superset.views.base import (
api,
BaseSupersetView,
+ deprecated,
handle_api_exception,
json_error_response,
)
@@ -69,6 +70,7 @@ class Datasource(BaseSupersetView):
@has_access_api
@api
@handle_api_exception
+ @deprecated(new_target="/api/v1/dataset/<int:pk>")
def save(self) -> FlaskResponse:
data = request.form.get("data")
if not isinstance(data, str):
@@ -133,6 +135,7 @@ class Datasource(BaseSupersetView):
@has_access_api
@api
@handle_api_exception
+ @deprecated(new_target="/api/v1/dataset/<int:pk>")
def get(self, datasource_type: str, datasource_id: int) -> FlaskResponse:
datasource = DatasourceDAO.get_datasource(
db.session, DatasourceType(datasource_type), datasource_id
diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py
index 1762a46305..60a8ee58bd 100644
--- a/tests/integration_tests/datasets/api_tests.py
+++ b/tests/integration_tests/datasets/api_tests.py
@@ -19,7 +19,7 @@ import json
import unittest
from io import BytesIO
from typing import List, Optional
-from unittest.mock import patch
+from unittest.mock import ANY, patch
from zipfile import is_zipfile, ZipFile
import prison
@@ -347,6 +347,28 @@ class TestDatasetApi(SupersetTestCase):
"sql": None,
"table_name": "energy_usage",
"template_params": None,
+ "uid": "2__table",
+ "datasource_name": "energy_usage",
+ "name": f"{get_example_default_schema()}.energy_usage",
+ "column_formats": {},
+ "granularity_sqla": [],
+ "time_grain_sqla": ANY,
+ "order_by_choices": [
+ ['["source", true]', "source [asc]"],
+ ['["source", false]', "source [desc]"],
+ ['["target", true]', "target [asc]"],
+ ['["target", false]', "target [desc]"],
+ ['["value", true]', "value [asc]"],
+ ['["value", false]', "value [desc]"],
+ ],
+ "verbose_map": {
+ "__timestamp": "Time",
+ "count": "COUNT(*)",
+ "source": "source",
+ "sum__value": "sum__value",
+ "target": "target",
+ "value": "value",
+ },
}
if response["result"]["database"]["backend"] not in ("presto", "hive"):
assert {
@@ -1350,6 +1372,32 @@ class TestDatasetApi(SupersetTestCase):
db.session.delete(ab_user)
db.session.commit()
+ def test_update_dataset_unsafe_default_endpoint(self):
+ """
+ Dataset API: Test unsafe default endpoint
+ """
+ if backend() == "sqlite":
+ return
+
+ dataset = self.insert_default_dataset()
+ self.login(username="admin")
+ uri = f"api/v1/dataset/{dataset.id}"
+ table_data = {"default_endpoint": "http://www.google.com"}
+ rv = self.client.put(uri, json=table_data)
+ data = json.loads(rv.data.decode("utf-8"))
+ assert rv.status_code == 422
+ expected_response = {
+ "message": {
+ "default_endpoint": [
+ "The submitted URL is not considered safe,"
+ " only use URLs with the same domain as Superset."
+ ]
+ }
+ }
+ assert data == expected_response
+ db.session.delete(dataset)
+ db.session.commit()
+
@patch("superset.datasets.dao.DatasetDAO.update")
def test_update_dataset_sqlalchemy_error(self, mock_dao_update):
"""