You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by mi...@apache.org on 2023/09/25 13:30:22 UTC
[superset] branch 3.0 updated (40d9c4c81f -> 295b9e1811)
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a change to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
from 40d9c4c81f chore: Updates CHANGELOG.md
new 807a027a5f fix: is_select with UNION (#25290)
new 2d1f1e3d71 fix: Add explicit ON DELETE CASCADE for dashboard_roles (#25320)
new 8ca49d4e6f fix(chart): Supporting custom SQL as temporal x-axis column with filter (#25126)
new 61dcc70db4 fix: Use RLS clause instead of ID for cache key (#25229)
new 0d53446562 fix: Improve the reliability of alerts & reports (#25239)
new 5390e2d826 fix: DashboardRoles cascade operation (#25349)
new 295b9e1811 fix: datetime with timezone excel export (#25318)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/explore/actions/exploreActions.test.js | 13 +-
.../src/explore/reducers/exploreReducer.js | 18 +--
superset/common/query_context_factory.py | 7 +-
superset/config.py | 5 +
superset/connectors/sqla/models.py | 2 +
...3_add_on_delete_cascade_for_dashboard_slices.py | 2 +-
...on_delete_cascade_for_embedded_dashboards.py.py | 2 +-
...1_add_on_delete_cascade_for_dashboard_roles.py} | 20 +--
superset/models/dashboard.py | 14 +-
superset/security/manager.py | 26 ++--
superset/sql_parse.py | 10 +-
superset/tasks/cron_util.py | 14 +-
superset/tasks/scheduler.py | 9 +-
superset/utils/excel.py | 5 +
tests/integration_tests/charts/data/api_tests.py | 66 ++++++++++
.../security/row_level_security_tests.py | 15 +++
tests/unit_tests/sql_parse_tests.py | 10 ++
tests/unit_tests/tasks/test_cron_util.py | 145 ++++++++++++---------
.../unit_tests/utils/excel_tests.py | 18 +--
19 files changed, 281 insertions(+), 120 deletions(-)
copy superset/migrations/versions/{2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py => 2023-09-15_12-58_4b85906e5b91_add_on_delete_cascade_for_dashboard_roles.py} (79%)
copy superset/utils/excel.py => tests/unit_tests/utils/excel_tests.py (67%)
[superset] 07/07: fix: datetime with timezone excel export (#25318)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 295b9e1811c1dc42fc537703d6098528c2476693
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Thu Sep 21 13:11:59 2023 -0700
fix: datetime with timezone excel export (#25318)
Co-authored-by: Michael S. Molina <70...@users.noreply.github.com>
(cherry picked from commit 5ebcd2a5f69d2691f1e6c0ffc4a611c728cf4354)
---
superset/utils/excel.py | 5 +++++
.../excel.py => tests/unit_tests/utils/excel_tests.py | 18 ++++++++++--------
2 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/superset/utils/excel.py b/superset/utils/excel.py
index 1f68031b64..ccbeadee5e 100644
--- a/superset/utils/excel.py
+++ b/superset/utils/excel.py
@@ -22,6 +22,11 @@ import pandas as pd
def df_to_excel(df: pd.DataFrame, **kwargs: Any) -> Any:
output = io.BytesIO()
+
+ # timezones are not supported
+ for column in df.select_dtypes(include=["datetimetz"]).columns:
+ df[column] = df[column].astype(str)
+
# pylint: disable=abstract-class-instantiated
with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
df.to_excel(writer, **kwargs)
diff --git a/superset/utils/excel.py b/tests/unit_tests/utils/excel_tests.py
similarity index 67%
copy from superset/utils/excel.py
copy to tests/unit_tests/utils/excel_tests.py
index 1f68031b64..c15f69a0c6 100644
--- a/superset/utils/excel.py
+++ b/tests/unit_tests/utils/excel_tests.py
@@ -14,16 +14,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-import io
-from typing import Any
+
+from datetime import datetime, timezone
import pandas as pd
+from superset.utils.excel import df_to_excel
-def df_to_excel(df: pd.DataFrame, **kwargs: Any) -> Any:
- output = io.BytesIO()
- # pylint: disable=abstract-class-instantiated
- with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
- df.to_excel(writer, **kwargs)
- return output.getvalue()
+def test_timezone_conversion() -> None:
+ """
+ Test that columns with timezones are converted to a string.
+ """
+ df = pd.DataFrame({"dt": [datetime(2023, 1, 1, 0, 0, tzinfo=timezone.utc)]})
+ contents = df_to_excel(df)
+ assert pd.read_excel(contents)["dt"][0] == "2023-01-01 00:00:00+00:00"
[superset] 03/07: fix(chart): Supporting custom SQL as temporal x-axis column with filter (#25126)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 8ca49d4e6f07efd00c745178d66f117507a53c85
Author: Zef Lin <ze...@preset.io>
AuthorDate: Mon Sep 18 11:30:52 2023 -0700
fix(chart): Supporting custom SQL as temporal x-axis column with filter (#25126)
Co-authored-by: Kamil Gabryjelski <ka...@gmail.com>
---
.../src/explore/actions/exploreActions.test.js | 13 ++---
.../src/explore/reducers/exploreReducer.js | 18 +++---
superset/common/query_context_factory.py | 7 ++-
superset/connectors/sqla/models.py | 2 +
tests/integration_tests/charts/data/api_tests.py | 66 ++++++++++++++++++++++
5 files changed, 90 insertions(+), 16 deletions(-)
diff --git a/superset-frontend/src/explore/actions/exploreActions.test.js b/superset-frontend/src/explore/actions/exploreActions.test.js
index cdf76bdcc6..9dd5375680 100644
--- a/superset-frontend/src/explore/actions/exploreActions.test.js
+++ b/superset-frontend/src/explore/actions/exploreActions.test.js
@@ -22,20 +22,19 @@ import exploreReducer from 'src/explore/reducers/exploreReducer';
import * as actions from 'src/explore/actions/exploreActions';
describe('reducers', () => {
- it('sets correct control value given an arbitrary key and value', () => {
+ it('Does not set a control value if control does not exist', () => {
const newState = exploreReducer(
defaultState,
actions.setControlValue('NEW_FIELD', 'x', []),
);
- expect(newState.controls.NEW_FIELD.value).toBe('x');
- expect(newState.form_data.NEW_FIELD).toBe('x');
+ expect(newState.controls.NEW_FIELD).toBeUndefined();
});
- it('setControlValue works as expected with a checkbox', () => {
+ it('setControlValue works as expected with a Select control', () => {
const newState = exploreReducer(
defaultState,
- actions.setControlValue('show_legend', true, []),
+ actions.setControlValue('y_axis_format', '$,.2f', []),
);
- expect(newState.controls.show_legend.value).toBe(true);
- expect(newState.form_data.show_legend).toBe(true);
+ expect(newState.controls.y_axis_format.value).toBe('$,.2f');
+ expect(newState.form_data.y_axis_format).toBe('$,.2f');
});
});
diff --git a/superset-frontend/src/explore/reducers/exploreReducer.js b/superset-frontend/src/explore/reducers/exploreReducer.js
index ac451ade3d..d5565a0dad 100644
--- a/superset-frontend/src/explore/reducers/exploreReducer.js
+++ b/superset-frontend/src/explore/reducers/exploreReducer.js
@@ -112,7 +112,7 @@ export default function exploreReducer(state = {}, action) {
const vizType = new_form_data.viz_type;
// if the controlName is metrics, and the metric column name is updated,
- // need to update column config as well to keep the previou config.
+ // need to update column config as well to keep the previous config.
if (controlName === 'metrics' && old_metrics_data && new_column_config) {
value.forEach((item, index) => {
if (
@@ -129,11 +129,11 @@ export default function exploreReducer(state = {}, action) {
}
// Use the processed control config (with overrides and everything)
- // if `controlName` does not existing in current controls,
+ // if `controlName` does not exist in current controls,
const controlConfig =
state.controls[action.controlName] ||
getControlConfig(action.controlName, vizType) ||
- {};
+ null;
// will call validators again
const control = {
@@ -149,7 +149,7 @@ export default function exploreReducer(state = {}, action) {
...state,
controls: {
...state.controls,
- [controlName]: control,
+ ...(controlConfig && { [controlName]: control }),
...(controlName === 'metrics' && { column_config }),
},
};
@@ -196,10 +196,12 @@ export default function exploreReducer(state = {}, action) {
triggerRender: control.renderTrigger && !hasErrors,
controls: {
...currentControlsState,
- [action.controlName]: {
- ...control,
- validationErrors: errors,
- },
+ ...(controlConfig && {
+ [action.controlName]: {
+ ...control,
+ validationErrors: errors,
+ },
+ }),
...rerenderedControls,
},
};
diff --git a/superset/common/query_context_factory.py b/superset/common/query_context_factory.py
index a6fe549894..e4680ed5ed 100644
--- a/superset/common/query_context_factory.py
+++ b/superset/common/query_context_factory.py
@@ -26,7 +26,7 @@ from superset.common.query_object_factory import QueryObjectFactory
from superset.daos.chart import ChartDAO
from superset.daos.datasource import DatasourceDAO
from superset.models.slice import Slice
-from superset.utils.core import DatasourceDict, DatasourceType
+from superset.utils.core import DatasourceDict, DatasourceType, is_adhoc_column
if TYPE_CHECKING:
from superset.connectors.base.models import BaseDatasource
@@ -129,6 +129,8 @@ class QueryContextFactory: # pylint: disable=too-few-public-methods
if granularity := query_object.granularity:
filter_to_remove = None
+ if is_adhoc_column(x_axis): # type: ignore
+ x_axis = x_axis.get("sqlExpression")
if x_axis and x_axis in temporal_columns:
filter_to_remove = x_axis
x_axis_column = next(
@@ -176,6 +178,9 @@ class QueryContextFactory: # pylint: disable=too-few-public-methods
# another temporal filter. A new filter based on the value of
# the granularity will be added later in the code.
# In practice, this is replacing the previous default temporal filter.
+ if is_adhoc_column(filter_to_remove): # type: ignore
+ filter_to_remove = filter_to_remove.get("sqlExpression")
+
if filter_to_remove:
query_object.filter = [
filter
diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py
index c7ea336ded..79203256f1 100644
--- a/superset/connectors/sqla/models.py
+++ b/superset/connectors/sqla/models.py
@@ -1007,6 +1007,8 @@ class SqlaTable(
qry = sa.select([sqla_column]).limit(1).select_from(tbl)
sql = self.database.compile_sqla_query(qry)
col_desc = get_columns_description(self.database, sql)
+ if not col_desc:
+ raise SupersetGenericDBErrorException("Column not found")
is_dttm = col_desc[0]["is_dttm"] # type: ignore
except SupersetGenericDBErrorException as ex:
raise ColumnNotFoundException(message=str(ex)) from ex
diff --git a/tests/integration_tests/charts/data/api_tests.py b/tests/integration_tests/charts/data/api_tests.py
index dc82026986..ab91cce55e 100644
--- a/tests/integration_tests/charts/data/api_tests.py
+++ b/tests/integration_tests/charts/data/api_tests.py
@@ -51,6 +51,7 @@ from superset.models.slice import Slice
from superset.superset_typing import AdhocColumn
from superset.utils.core import (
AnnotationType,
+ backend,
get_example_default_schema,
AdhocMetricExpressionType,
ExtraFiltersReasonType,
@@ -943,6 +944,71 @@ class TestGetChartDataApi(BaseTestChartDataApi):
assert data["result"][0]["status"] == "success"
assert data["result"][0]["rowcount"] == 2
+ @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
+ def test_chart_data_get_with_x_axis_using_custom_sql(self):
+ """
+ Chart data API: Test GET endpoint
+ """
+ chart = db.session.query(Slice).filter_by(slice_name="Genders").one()
+ chart.query_context = json.dumps(
+ {
+ "datasource": {"id": chart.table.id, "type": "table"},
+ "force": False,
+ "queries": [
+ {
+ "time_range": "1900-01-01T00:00:00 : 2000-01-01T00:00:00",
+ "granularity": "ds",
+ "filters": [
+ {"col": "ds", "op": "TEMPORAL_RANGE", "val": "No filter"}
+ ],
+ "extras": {
+ "having": "",
+ "where": "",
+ },
+ "applied_time_extras": {},
+ "columns": [
+ {
+ "columnType": "BASE_AXIS",
+ "datasourceWarning": False,
+ "expressionType": "SQL",
+ "label": "My column",
+ "sqlExpression": "ds",
+ "timeGrain": "P1W",
+ }
+ ],
+ "metrics": ["sum__num"],
+ "orderby": [["sum__num", False]],
+ "annotation_layers": [],
+ "row_limit": 50000,
+ "timeseries_limit": 0,
+ "order_desc": True,
+ "url_params": {},
+ "custom_params": {},
+ "custom_form_data": {},
+ }
+ ],
+ "form_data": {
+ "x_axis": {
+ "datasourceWarning": False,
+ "expressionType": "SQL",
+ "label": "My column",
+ "sqlExpression": "ds",
+ }
+ },
+ "result_format": "json",
+ "result_type": "full",
+ }
+ )
+ rv = self.get_assert_metric(f"api/v1/chart/{chart.id}/data/", "get_data")
+ assert rv.mimetype == "application/json"
+ data = json.loads(rv.data.decode("utf-8"))
+ assert data["result"][0]["status"] == "success"
+
+ if backend() == "presto":
+ assert data["result"][0]["rowcount"] == 41
+ else:
+ assert data["result"][0]["rowcount"] == 40
+
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
def test_chart_data_get_forced(self):
"""
[superset] 04/07: fix: Use RLS clause instead of ID for cache key (#25229)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 61dcc70db442320b6b3ae5c7feda88cc9e1968a9
Author: Jack Fragassi <jf...@gmail.com>
AuthorDate: Mon Sep 18 11:37:35 2023 -0700
fix: Use RLS clause instead of ID for cache key (#25229)
(cherry picked from commit fba66c6250c38944639cfc1f95a67ef00c66629c)
---
superset/security/manager.py | 26 ++++++++++++----------
.../security/row_level_security_tests.py | 15 +++++++++++++
2 files changed, 29 insertions(+), 12 deletions(-)
diff --git a/superset/security/manager.py b/superset/security/manager.py
index 6c47c6e163..47e772d765 100644
--- a/superset/security/manager.py
+++ b/superset/security/manager.py
@@ -79,7 +79,7 @@ from superset.utils.urls import get_url_host
if TYPE_CHECKING:
from superset.common.query_context import QueryContext
from superset.connectors.base.models import BaseDatasource
- from superset.connectors.sqla.models import SqlaTable
+ from superset.connectors.sqla.models import RowLevelSecurityFilter, SqlaTable
from superset.models.core import Database
from superset.models.dashboard import Dashboard
from superset.models.sql_lab import Query
@@ -2091,28 +2091,30 @@ class SupersetSecurityManager( # pylint: disable=too-many-public-methods
)
return query.all()
- def get_rls_ids(self, table: "BaseDatasource") -> list[int]:
+ def get_rls_sorted(self, table: "BaseDatasource") -> list["RowLevelSecurityFilter"]:
"""
- Retrieves the appropriate row level security filters IDs for the current user
- and the passed table.
+ Retrieves a list RLS filters sorted by ID for
+ the current user and the passed table.
:param table: The table to check against
- :returns: A list of IDs
+ :returns: A list RLS filters
"""
- ids = [f.id for f in self.get_rls_filters(table)]
- ids.sort() # Combinations rather than permutations
- return ids
+ filters = self.get_rls_filters(table)
+ filters.sort(key=lambda f: f.id)
+ return filters
def get_guest_rls_filters_str(self, table: "BaseDatasource") -> list[str]:
return [f.get("clause", "") for f in self.get_guest_rls_filters(table)]
def get_rls_cache_key(self, datasource: "BaseDatasource") -> list[str]:
- rls_ids = []
+ rls_clauses_with_group_key = []
if datasource.is_rls_supported:
- rls_ids = self.get_rls_ids(datasource)
- rls_str = [str(rls_id) for rls_id in rls_ids]
+ rls_clauses_with_group_key = [
+ f"{f.clause}-{f.group_key or ''}"
+ for f in self.get_rls_sorted(datasource)
+ ]
guest_rls = self.get_guest_rls_filters_str(datasource)
- return guest_rls + rls_str
+ return guest_rls + rls_clauses_with_group_key
@staticmethod
def _get_current_epoch_time() -> float:
diff --git a/tests/integration_tests/security/row_level_security_tests.py b/tests/integration_tests/security/row_level_security_tests.py
index c29ebe9afe..41ca0d5e79 100644
--- a/tests/integration_tests/security/row_level_security_tests.py
+++ b/tests/integration_tests/security/row_level_security_tests.py
@@ -305,6 +305,21 @@ class TestRowLevelSecurity(SupersetTestCase):
assert not self.NAMES_Q_REGEX.search(sql)
assert not self.BASE_FILTER_REGEX.search(sql)
+ @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
+ def test_get_rls_cache_key(self):
+ g.user = self.get_user(username="admin")
+ tbl = self.get_table(name="birth_names")
+ clauses = security_manager.get_rls_cache_key(tbl)
+ assert clauses == []
+
+ g.user = self.get_user(username="gamma")
+ clauses = security_manager.get_rls_cache_key(tbl)
+ assert clauses == [
+ "name like 'A%' or name like 'B%'-name",
+ "name like 'Q%'-name",
+ "gender = 'boy'-gender",
+ ]
+
class TestRowLevelSecurityCreateAPI(SupersetTestCase):
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
[superset] 02/07: fix: Add explicit ON DELETE CASCADE for dashboard_roles (#25320)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 2d1f1e3d71adfebbdf4a6c09425f7279c60edcdf
Author: John Bodley <45...@users.noreply.github.com>
AuthorDate: Mon Sep 18 05:49:51 2023 -0700
fix: Add explicit ON DELETE CASCADE for dashboard_roles (#25320)
(cherry picked from commit d54e827bb9f8eab8a7734bf7bdb8a5fdb2ae0c79)
---
...6f3_add_on_delete_cascade_for_dashboard_slices.py | 2 +-
...d_on_delete_cascade_for_embedded_dashboards.py.py | 2 +-
...b91_add_on_delete_cascade_for_dashboard_roles.py} | 20 ++++++++++----------
superset/models/dashboard.py | 16 ++++++++++++++--
4 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py b/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py
index caac489bd1..42e3507f7c 100644
--- a/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py
+++ b/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""add on delete cascade for dashboard slices
+"""add on delete cascade for dashboard_slices
Revision ID: 8ace289026f3
Revises: 2e826adca42c
diff --git a/superset/migrations/versions/2023-08-09_15-39_4448fa6deeb1__dd_on_delete_cascade_for_embedded_dashboards.py.py b/superset/migrations/versions/2023-08-09_15-39_4448fa6deeb1__dd_on_delete_cascade_for_embedded_dashboards.py.py
index b50f637514..8d298863a6 100644
--- a/superset/migrations/versions/2023-08-09_15-39_4448fa6deeb1__dd_on_delete_cascade_for_embedded_dashboards.py.py
+++ b/superset/migrations/versions/2023-08-09_15-39_4448fa6deeb1__dd_on_delete_cascade_for_embedded_dashboards.py.py
@@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""add on delete cascade for embedded dashboards
+"""add on delete cascade for embedded_dashboards
Revision ID: 4448fa6deeb1
Revises: 8ace289026f3
diff --git a/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py b/superset/migrations/versions/2023-09-15_12-58_4b85906e5b91_add_on_delete_cascade_for_dashboard_roles.py
similarity index 79%
copy from superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py
copy to superset/migrations/versions/2023-09-15_12-58_4b85906e5b91_add_on_delete_cascade_for_dashboard_roles.py
index caac489bd1..b712aca682 100644
--- a/superset/migrations/versions/2023-08-09_14-17_8ace289026f3_add_on_delete_cascade_for_dashboard_slices.py
+++ b/superset/migrations/versions/2023-09-15_12-58_4b85906e5b91_add_on_delete_cascade_for_dashboard_roles.py
@@ -14,32 +14,32 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""add on delete cascade for dashboard slices
+"""add on delete cascade for dashboard_roles
-Revision ID: 8ace289026f3
-Revises: 2e826adca42c
-Create Date: 2023-08-09 14:17:53.326191
+Revision ID: 4b85906e5b91
+Revises: 317970b4400c
+Create Date: 2023-09-15 12:58:26.772759
"""
# revision identifiers, used by Alembic.
-revision = "8ace289026f3"
-down_revision = "2e826adca42c"
+revision = "4b85906e5b91"
+down_revision = "317970b4400c"
from superset.migrations.shared.constraints import ForeignKey, redefine
foreign_keys = [
ForeignKey(
- table="dashboard_slices",
+ table="dashboard_roles",
referent_table="dashboards",
local_cols=["dashboard_id"],
remote_cols=["id"],
),
ForeignKey(
- table="dashboard_slices",
- referent_table="slices",
- local_cols=["slice_id"],
+ table="dashboard_roles",
+ referent_table="ab_role",
+ local_cols=["role_id"],
remote_cols=["id"],
),
]
diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py
index 0848126012..a88d361a0e 100644
--- a/superset/models/dashboard.py
+++ b/superset/models/dashboard.py
@@ -127,8 +127,20 @@ DashboardRoles = Table(
"dashboard_roles",
metadata,
Column("id", Integer, primary_key=True),
- Column("dashboard_id", Integer, ForeignKey("dashboards.id"), nullable=False),
- Column("role_id", Integer, ForeignKey("ab_role.id"), nullable=False),
+ Column(
+ "dashboard_id",
+ Integer,
+ ForeignKey("dashboards.id"),
+ nullable=False,
+ on_delete="CASCADE",
+ ),
+ Column(
+ "role_id",
+ Integer,
+ ForeignKey("ab_role.id"),
+ nullable=False,
+ on_delete="CASCADE",
+ ),
)
[superset] 06/07: fix: DashboardRoles cascade operation (#25349)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 5390e2d82643b652c75a5560729fb6e142f8482e
Author: Michael S. Molina <70...@users.noreply.github.com>
AuthorDate: Wed Sep 20 13:08:07 2023 -0300
fix: DashboardRoles cascade operation (#25349)
(cherry picked from commit a971a28a3450b28151bbad3632ce2364c87df3fc)
---
superset/models/dashboard.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py
index a88d361a0e..c7bf15fa04 100644
--- a/superset/models/dashboard.py
+++ b/superset/models/dashboard.py
@@ -130,16 +130,14 @@ DashboardRoles = Table(
Column(
"dashboard_id",
Integer,
- ForeignKey("dashboards.id"),
+ ForeignKey("dashboards.id", on_delete="CASCADE"),
nullable=False,
- on_delete="CASCADE",
),
Column(
"role_id",
Integer,
- ForeignKey("ab_role.id"),
+ ForeignKey("ab_role.id", on_delete="CASCADE"),
nullable=False,
- on_delete="CASCADE",
),
)
[superset] 05/07: fix: Improve the reliability of alerts & reports (#25239)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 0d53446562acfaae6014c8f03af376072326e255
Author: Jack Fragassi <jf...@gmail.com>
AuthorDate: Tue Sep 19 09:40:13 2023 -0700
fix: Improve the reliability of alerts & reports (#25239)
(cherry picked from commit f672d5da5cb9390b83176bb12c27ce7eeea3e8ae)
---
superset/config.py | 5 ++
superset/tasks/cron_util.py | 14 +--
superset/tasks/scheduler.py | 9 +-
tests/unit_tests/tasks/test_cron_util.py | 145 ++++++++++++++++++-------------
4 files changed, 106 insertions(+), 67 deletions(-)
diff --git a/superset/config.py b/superset/config.py
index 98718e731e..3847555a05 100644
--- a/superset/config.py
+++ b/superset/config.py
@@ -904,6 +904,10 @@ DASHBOARD_AUTO_REFRESH_INTERVALS = [
[86400, "24 hours"],
]
+# This is used as a workaround for the alerts & reports scheduler task to get the time
+# celery beat triggered it, see https://github.com/celery/celery/issues/6974 for details
+CELERY_BEAT_SCHEDULER_EXPIRES = timedelta(weeks=1)
+
# Default celery config is to use SQLA as a broker, in a production setting
# you'll want to use a proper broker as specified here:
# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html
@@ -932,6 +936,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods
"reports.scheduler": {
"task": "reports.scheduler",
"schedule": crontab(minute="*", hour="*"),
+ "options": {"expires": int(CELERY_BEAT_SCHEDULER_EXPIRES.total_seconds())},
},
"reports.prune_log": {
"task": "reports.prune_log",
diff --git a/superset/tasks/cron_util.py b/superset/tasks/cron_util.py
index 19d342ebdc..329937fb82 100644
--- a/superset/tasks/cron_util.py
+++ b/superset/tasks/cron_util.py
@@ -17,7 +17,7 @@
import logging
from collections.abc import Iterator
-from datetime import datetime, timedelta, timezone as dt_timezone
+from datetime import datetime, timedelta
from croniter import croniter
from pytz import timezone as pytz_timezone, UnknownTimeZoneError
@@ -27,10 +27,10 @@ from superset import app
logger = logging.getLogger(__name__)
-def cron_schedule_window(cron: str, timezone: str) -> Iterator[datetime]:
+def cron_schedule_window(
+ triggered_at: datetime, cron: str, timezone: str
+) -> Iterator[datetime]:
window_size = app.config["ALERT_REPORTS_CRON_WINDOW_SIZE"]
- # create a time-aware datetime in utc
- time_now = datetime.now(tz=dt_timezone.utc)
try:
tz = pytz_timezone(timezone)
except UnknownTimeZoneError:
@@ -39,9 +39,9 @@ def cron_schedule_window(cron: str, timezone: str) -> Iterator[datetime]:
logger.warning("Timezone %s was invalid. Falling back to 'UTC'", timezone)
utc = pytz_timezone("UTC")
# convert the current time to the user's local time for comparison
- time_now = time_now.astimezone(tz)
- start_at = time_now - timedelta(seconds=1)
- stop_at = time_now + timedelta(seconds=window_size)
+ time_now = triggered_at.astimezone(tz)
+ start_at = time_now - timedelta(seconds=window_size / 2)
+ stop_at = time_now + timedelta(seconds=window_size / 2)
crons = croniter(cron, start_at)
for schedule in crons.all_next(datetime):
if schedule >= stop_at:
diff --git a/superset/tasks/scheduler.py b/superset/tasks/scheduler.py
index a84036c340..9c541605c7 100644
--- a/superset/tasks/scheduler.py
+++ b/superset/tasks/scheduler.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
import logging
+from datetime import datetime
from celery import Celery
from celery.exceptions import SoftTimeLimitExceeded
@@ -43,9 +44,15 @@ def scheduler() -> None:
return
with session_scope(nullpool=True) as session:
active_schedules = ReportScheduleDAO.find_active(session)
+ triggered_at = (
+ datetime.fromisoformat(scheduler.request.expires)
+ - app.config["CELERY_BEAT_SCHEDULER_EXPIRES"]
+ if scheduler.request.expires
+ else datetime.utcnow()
+ )
for active_schedule in active_schedules:
for schedule in cron_schedule_window(
- active_schedule.crontab, active_schedule.timezone
+ triggered_at, active_schedule.crontab, active_schedule.timezone
):
logger.info(
"Scheduling alert %s eta: %s", active_schedule.name, schedule
diff --git a/tests/unit_tests/tasks/test_cron_util.py b/tests/unit_tests/tasks/test_cron_util.py
index 5bc22273f5..56f1258e30 100644
--- a/tests/unit_tests/tasks/test_cron_util.py
+++ b/tests/unit_tests/tasks/test_cron_util.py
@@ -14,11 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from datetime import datetime
import pytest
-import pytz
-from dateutil import parser
-from freezegun import freeze_time
from freezegun.api import FakeDatetime
from superset.tasks.cron_util import cron_schedule_window
@@ -27,23 +25,28 @@ from superset.tasks.cron_util import cron_schedule_window
@pytest.mark.parametrize(
"current_dttm, cron, expected",
[
- ("2020-01-01T08:59:01Z", "0 1 * * *", []),
+ ("2020-01-01T08:59:01+00:00", "0 1 * * *", []),
(
- "2020-01-01T08:59:02Z",
+ "2020-01-01T08:59:32+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 9, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T08:59:59Z",
+ "2020-01-01T08:59:59+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 9, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T09:00:00Z",
+ "2020-01-01T09:00:00+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 9, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
- ("2020-01-01T09:00:01Z", "0 1 * * *", []),
+ (
+ "2020-01-01T09:00:01+00:00",
+ "0 1 * * *",
+ [FakeDatetime(2020, 1, 1, 9, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
+ ),
+ ("2020-01-01T09:00:30+00:00", "0 1 * * *", []),
],
)
def test_cron_schedule_window_los_angeles(
@@ -53,34 +56,40 @@ def test_cron_schedule_window_los_angeles(
Reports scheduler: Test cron schedule window for "America/Los_Angeles"
"""
- with freeze_time(current_dttm):
- datetimes = cron_schedule_window(cron, "America/Los_Angeles")
- assert (
- list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes)
- == expected
- )
+ datetimes = cron_schedule_window(
+ datetime.fromisoformat(current_dttm), cron, "America/Los_Angeles"
+ )
+ assert (
+ list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes) == expected
+ )
@pytest.mark.parametrize(
"current_dttm, cron, expected",
[
- ("2020-01-01T00:59:01Z", "0 1 * * *", []),
+ ("2020-01-01T00:59:01+00:00", "0 1 * * *", []),
+ ("2020-01-01T00:59:02+00:00", "0 1 * * *", []),
+ (
+ "2020-01-01T00:59:59+00:00",
+ "0 1 * * *",
+ [FakeDatetime(2020, 1, 1, 1, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
+ ),
(
- "2020-01-01T00:59:02Z",
+ "2020-01-01T01:00:00+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 1, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T00:59:59Z",
+ "2020-01-01T01:00:01+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 1, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T01:00:00Z",
+ "2020-01-01T01:00:29+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 1, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
- ("2020-01-01T01:00:01Z", "0 1 * * *", []),
+ ("2020-01-01T01:00:30+00:00", "0 1 * * *", []),
],
)
def test_cron_schedule_window_invalid_timezone(
@@ -90,35 +99,41 @@ def test_cron_schedule_window_invalid_timezone(
Reports scheduler: Test cron schedule window for "invalid timezone"
"""
- with freeze_time(current_dttm):
- datetimes = cron_schedule_window(cron, "invalid timezone")
- # it should default to UTC
- assert (
- list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes)
- == expected
- )
+ datetimes = cron_schedule_window(
+ datetime.fromisoformat(current_dttm), cron, "invalid timezone"
+ )
+ # it should default to UTC
+ assert (
+ list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes) == expected
+ )
@pytest.mark.parametrize(
"current_dttm, cron, expected",
[
- ("2020-01-01T05:59:01Z", "0 1 * * *", []),
+ ("2020-01-01T05:59:01+00:00", "0 1 * * *", []),
+ ("2020-01-01T05:59:02+00:00", "0 1 * * *", []),
+ (
+ "2020-01-01T05:59:59+00:00",
+ "0 1 * * *",
+ [FakeDatetime(2020, 1, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
+ ),
(
- "2020-01-01T05:59:02Z",
+ "2020-01-01T06:00:00+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T5:59:59Z",
+ "2020-01-01T06:00:01+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T6:00:00",
+ "2020-01-01T06:00:29+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
- ("2020-01-01T6:00:01Z", "0 1 * * *", []),
+ ("2020-01-01T06:00:30+00:00", "0 1 * * *", []),
],
)
def test_cron_schedule_window_new_york(
@@ -128,34 +143,40 @@ def test_cron_schedule_window_new_york(
Reports scheduler: Test cron schedule window for "America/New_York"
"""
- with freeze_time(current_dttm, tz_offset=0):
- datetimes = cron_schedule_window(cron, "America/New_York")
- assert (
- list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes)
- == expected
- )
+ datetimes = cron_schedule_window(
+ datetime.fromisoformat(current_dttm), cron, "America/New_York"
+ )
+ assert (
+ list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes) == expected
+ )
@pytest.mark.parametrize(
"current_dttm, cron, expected",
[
- ("2020-01-01T06:59:01Z", "0 1 * * *", []),
+ ("2020-01-01T06:59:01+00:00", "0 1 * * *", []),
+ ("2020-01-01T06:59:02+00:00", "0 1 * * *", []),
+ (
+ "2020-01-01T06:59:59+00:00",
+ "0 1 * * *",
+ [FakeDatetime(2020, 1, 1, 7, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
+ ),
(
- "2020-01-01T06:59:02Z",
+ "2020-01-01T07:00:00+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 7, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T06:59:59Z",
+ "2020-01-01T07:00:01+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 7, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-01-01T07:00:00",
+ "2020-01-01T07:00:29+00:00",
"0 1 * * *",
[FakeDatetime(2020, 1, 1, 7, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
- ("2020-01-01T07:00:01Z", "0 1 * * *", []),
+ ("2020-01-01T07:00:30+00:00", "0 1 * * *", []),
],
)
def test_cron_schedule_window_chicago(
@@ -165,34 +186,40 @@ def test_cron_schedule_window_chicago(
Reports scheduler: Test cron schedule window for "America/Chicago"
"""
- with freeze_time(current_dttm, tz_offset=0):
- datetimes = cron_schedule_window(cron, "America/Chicago")
- assert (
- list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes)
- == expected
- )
+ datetimes = cron_schedule_window(
+ datetime.fromisoformat(current_dttm), cron, "America/Chicago"
+ )
+ assert (
+ list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes) == expected
+ )
@pytest.mark.parametrize(
"current_dttm, cron, expected",
[
- ("2020-07-01T05:59:01Z", "0 1 * * *", []),
+ ("2020-07-01T05:59:01+00:00", "0 1 * * *", []),
+ ("2020-07-01T05:59:02+00:00", "0 1 * * *", []),
+ (
+ "2020-07-01T05:59:59+00:00",
+ "0 1 * * *",
+ [FakeDatetime(2020, 7, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
+ ),
(
- "2020-07-01T05:59:02Z",
+ "2020-07-01T06:00:00+00:00",
"0 1 * * *",
[FakeDatetime(2020, 7, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-07-01T05:59:59Z",
+ "2020-07-01T06:00:01+00:00",
"0 1 * * *",
[FakeDatetime(2020, 7, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
(
- "2020-07-01T06:00:00",
+ "2020-07-01T06:00:29+00:00",
"0 1 * * *",
[FakeDatetime(2020, 7, 1, 6, 0).strftime("%A, %d %B %Y, %H:%M:%S")],
),
- ("2020-07-01T06:00:01Z", "0 1 * * *", []),
+ ("2020-07-01T06:00:30+00:00", "0 1 * * *", []),
],
)
def test_cron_schedule_window_chicago_daylight(
@@ -202,9 +229,9 @@ def test_cron_schedule_window_chicago_daylight(
Reports scheduler: Test cron schedule window for "America/Chicago"
"""
- with freeze_time(current_dttm, tz_offset=0):
- datetimes = cron_schedule_window(cron, "America/Chicago")
- assert (
- list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes)
- == expected
- )
+ datetimes = cron_schedule_window(
+ datetime.fromisoformat(current_dttm), cron, "America/Chicago"
+ )
+ assert (
+ list(cron.strftime("%A, %d %B %Y, %H:%M:%S") for cron in datetimes) == expected
+ )
[superset] 01/07: fix: is_select with UNION (#25290)
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
michaelsmolina pushed a commit to branch 3.0
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 807a027a5ffb27c4335cc8af81dd31d24559e165
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Thu Sep 14 09:05:19 2023 -0700
fix: is_select with UNION (#25290)
(cherry picked from commit bb002d6147c82de692f6692e77f59bd729953495)
---
superset/sql_parse.py | 10 +++++++---
tests/unit_tests/sql_parse_tests.py | 10 ++++++++++
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/superset/sql_parse.py b/superset/sql_parse.py
index 2d467ec2b3..c196fdabfa 100644
--- a/superset/sql_parse.py
+++ b/superset/sql_parse.py
@@ -230,13 +230,17 @@ class ParsedQuery:
:param oxide_parse: parsed CTE
:return: True if CTE is a SELECT statement
"""
+
+ def is_body_select(body: dict[str, Any]) -> bool:
+ if op := body.get("SetOperation"):
+ return is_body_select(op["left"]) and is_body_select(op["right"])
+ return all(key == "Select" for key in body.keys())
+
for query in oxide_parse:
parsed_query = query["Query"]
cte_tables = self._get_cte_tables(parsed_query)
for cte_table in cte_tables:
- is_select = all(
- key == "Select" for key in cte_table["query"]["body"].keys()
- )
+ is_select = is_body_select(cte_table["query"]["body"])
if not is_select:
return False
return True
diff --git a/tests/unit_tests/sql_parse_tests.py b/tests/unit_tests/sql_parse_tests.py
index 73074d3df6..341ba9d789 100644
--- a/tests/unit_tests/sql_parse_tests.py
+++ b/tests/unit_tests/sql_parse_tests.py
@@ -1623,3 +1623,13 @@ def test_is_select() -> None:
Test `is_select`.
"""
assert not ParsedQuery("SELECT 1; DROP DATABASE superset").is_select()
+ assert ParsedQuery(
+ "with base as(select id from table1 union all select id from table2) select * from base"
+ ).is_select()
+ assert ParsedQuery(
+ """
+WITH t AS (
+ SELECT 1 UNION ALL SELECT 2
+)
+SELECT * FROM t"""
+ ).is_select()