You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ly...@apache.org on 2022/07/22 17:07:08 UTC
[superset] 02/39: Working POC > columns are loading into page
This is an automated email from the ASF dual-hosted git repository.
lyndsi pushed a commit to branch lyndsi/sql-lab-new-explore-button-functionality-and-move-save-dataset-to-split-save-button
in repository https://gitbox.apache.org/repos/asf/superset.git
commit fe143804a6cfc3ae912a3148cc147457fbaf95d7
Author: Hugh A. Miles II <hu...@gmail.com>
AuthorDate: Mon Jun 6 16:19:20 2022 +0000
Working POC
> columns are loading into page
---
.../superset-ui-core/src/query/DatasourceKey.ts | 3 +-
.../superset-ui-core/src/query/types/Datasource.ts | 4 +-
superset/common/query_context_factory.py | 2 +
superset/common/query_context_processor.py | 5 +-
superset/models/helpers.py | 50 ++++----
superset/models/sql_lab.py | 60 ++--------
superset/utils/core.py | 35 +++---
superset/views/core.py | 127 ++++++++++++++++++++-
8 files changed, 182 insertions(+), 104 deletions(-)
diff --git a/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts b/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts
index 38a38e10b1..6f40abb9e3 100644
--- a/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts
+++ b/superset-frontend/packages/superset-ui-core/src/query/DatasourceKey.ts
@@ -27,7 +27,8 @@ export default class DatasourceKey {
constructor(key: string) {
const [idStr, typeStr] = key.split('__');
this.id = parseInt(idStr, 10);
- this.type = DatasourceType.Table; // default to SqlaTable model
+ this.type =
+ typeStr === 'table' ? DatasourceType.Table : DatasourceType.Druid;
this.type = typeStr === 'query' ? DatasourceType.Query : this.type;
}
diff --git a/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts b/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts
index 9639a000d0..389d2dce44 100644
--- a/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts
+++ b/superset-frontend/packages/superset-ui-core/src/query/types/Datasource.ts
@@ -21,10 +21,8 @@ import { Metric } from './Metric';
export enum DatasourceType {
Table = 'table',
+ Druid = 'druid',
Query = 'query',
- Dataset = 'dataset',
- SlTable = 'sl_table',
- SavedQuery = 'saved_query',
}
/**
diff --git a/superset/common/query_context_factory.py b/superset/common/query_context_factory.py
index dc43d28de9..c5a5d7cd1b 100644
--- a/superset/common/query_context_factory.py
+++ b/superset/common/query_context_factory.py
@@ -82,6 +82,8 @@ class QueryContextFactory: # pylint: disable=too-few-public-methods
# pylint: disable=no-self-use
def _convert_to_model(self, datasource: DatasourceDict) -> BaseDatasource:
+ from superset.dao.datasource.dao import DatasourceDAO
+ from superset.utils.core import DatasourceType
return DatasourceDAO.get_datasource(
session=db.session,
diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py
index 3b174dc7a2..49977aaa55 100644
--- a/superset/common/query_context_processor.py
+++ b/superset/common/query_context_processor.py
@@ -116,7 +116,7 @@ class QueryContextProcessor:
and col != DTTM_ALIAS
)
]
-
+ breakpoint()
if invalid_columns:
raise QueryObjectValidationError(
_(
@@ -124,7 +124,7 @@ class QueryContextProcessor:
invalid_columns=invalid_columns,
)
)
-
+
query_result = self.get_query_result(query_obj)
annotation_data = self.get_annotation_data(query_obj)
cache.set_query_result(
@@ -185,7 +185,6 @@ class QueryContextProcessor:
# support multiple queries from different data sources.
# The datasource here can be different backend but the interface is common
- # pylint: disable=import-outside-toplevel
from superset.models.sql_lab import Query
query = ""
diff --git a/superset/models/helpers.py b/superset/models/helpers.py
index 08efb59b60..f7f9aeff33 100644
--- a/superset/models/helpers.py
+++ b/superset/models/helpers.py
@@ -677,11 +677,7 @@ class ExploreMixin:
@property
def column_names(self):
- return ["ethnic_minority", "gender"]
-
- @property
- def columns(self):
- return ["<col_name>"]
+ return [col.get('column_name') for col in self.columns]
@property
def offset(self):
@@ -767,6 +763,10 @@ class ExploreMixin:
# todo(hugh): apply filters for extended query
query_str_ext = self.get_query_str_extended(qry)
sql = query_str_ext.sql
+
+ print('*****' * 5)
+
+ # sql = "select count(*) from flights"
status = QueryStatus.SUCCESS
errors = None
error_message = None
@@ -1117,12 +1117,11 @@ class ExploreMixin:
if granularity not in self.dttm_cols and granularity is not None:
granularity = self.main_dttm_col
- # columns_by_name: Dict[str, sa.Table] = {
- # col.column_name: col for col in self.columns
- # }
# todo(hugh): fix this
- columns_by_name = {}
-
+ columns_by_name = {
+ col.get('column_name'): col for col in self.columns
+ }
+
# todo(hugh): how are we handling metrics
# metrics_by_name: Dict[str, Column] = { # todo column vs metric?
# m.metric_name: m for m in self.metrics
@@ -1218,26 +1217,27 @@ class ExploreMixin:
# template_processor=template_processor,
)
# if groupby field equals a selected column
- elif selected in columns_by_name:
- outer = columns_by_name[selected].get_sqla_col()
- else:
- outer = literal_column(f"({selected})")
- outer = self.make_sqla_column_compatible(outer, selected)
+ # elif selected in columns_by_name:
+ # outer = columns_by_name[selected].get_sqla_col()
+ # else:
+ # outer = literal_column(f"({selected})")
+ # outer = self.make_sqla_column_compatible(outer, selected)
else:
outer = self.adhoc_column_to_sqla(
col=selected, # template_processor=template_processor
)
- groupby_all_columns[outer.name] = outer
- if not series_column_names or outer.name in series_column_names:
- groupby_series_columns[outer.name] = outer
- select_exprs.append(outer)
+ # groupby_all_columns[outer.name] = outer
+ # if not series_column_names or outer.name in series_column_names:
+ # groupby_series_columns[outer.name] = outer
+ # select_exprs.append(outer)
elif columns:
for selected in columns:
- select_exprs.append(
- columns_by_name[selected].get_sqla_col()
- if selected in columns_by_name
- else self.make_sqla_column_compatible(literal_column(selected))
- )
+ # select_exprs.append(
+ # columns_by_name[selected].get_sqla_col()
+ # if selected in columns_by_name
+ # else self.make_sqla_column_compatible(literal_column(selected))
+ # )
+ select_exprs.append(selected)
metrics_exprs = []
# todo(hugh): fix this
@@ -1288,7 +1288,7 @@ class ExploreMixin:
if not db_engine_spec.allows_hidden_ordeby_agg:
select_exprs = utils.remove_duplicates(select_exprs + orderby_exprs)
- qry = sa.select(select_exprs)
+ qry = sa.select([sa.column("YEAR")])
# todo(hugh) fix templating
# tbl, cte = self.get_from_clause(template_processor)
diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py
index 22f565032c..b6e318c10e 100644
--- a/superset/models/sql_lab.py
+++ b/superset/models/sql_lab.py
@@ -17,7 +17,7 @@
"""A collection of ORM sqlalchemy models for SQL Lab"""
import re
from datetime import datetime
-from typing import Any, Dict, List, Optional, Type, TYPE_CHECKING
+from typing import Any, Dict, List, Type
import simplejson as json
import sqlalchemy as sqla
@@ -52,11 +52,9 @@ from superset.sqllab.limiting_factor import LimitingFactor
from superset.superset_typing import ResultSetColumnType
from superset.utils.core import GenericDataType, QueryStatus, user_label
-if TYPE_CHECKING:
- from superset.db_engine_specs import BaseEngineSpec
-
+from superset.superset_typing import ResultSetColumnType
-class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-method
+class Query(Model, ExtraJSONMixin, ExploreMixin):
"""ORM model for SQL query
Now that SQL Lab support multi-statement execution, an entry in this
@@ -174,6 +172,8 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me
@property
def columns(self) -> List[ResultSetColumnType]:
+ # todo(hughhh): move this logic into a base class
+ from superset.utils.core import GenericDataType
bool_types = ("BOOL",)
num_types = (
"DOUBLE",
@@ -189,11 +189,10 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me
)
date_types = ("DATE", "TIME")
str_types = ("VARCHAR", "STRING", "CHAR")
- columns = []
- col_type = ""
+ columns = []
for col in self.extra.get("columns", []):
computed_column = {**col}
- col_type = col.get("type")
+ col_type = col.get('type')
if col_type and any(map(lambda t: t in col_type.upper(), str_types)):
computed_column["type_generic"] = GenericDataType.STRING
@@ -204,7 +203,7 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me
if col_type and any(map(lambda t: t in col_type.upper(), date_types)):
computed_column["type_generic"] = GenericDataType.TEMPORAL
- computed_column["column_name"] = col.get("name")
+ computed_column["column_name"] = col.get('name')
computed_column["groupby"] = True
columns.append(computed_column)
return columns # type: ignore
@@ -236,49 +235,6 @@ class Query(Model, ExtraJSONMixin, ExploreMixin): # pylint: disable=abstract-me
def db_engine_spec(self) -> Type["BaseEngineSpec"]:
return self.database.db_engine_spec
- @property
- def owners_data(self) -> List[Dict[str, Any]]:
- return []
-
- @property
- def uid(self) -> str:
- return f"{self.id}__{self.type}"
-
- @property
- def is_rls_supported(self) -> bool:
- return False
-
- @property
- def cache_timeout(self) -> int:
- return 0
-
- @property
- def column_names(self) -> List[Any]:
- return [col.get("column_name") for col in self.columns]
-
- @property
- def offset(self) -> int:
- return 0
-
- @property
- def main_dttm_col(self) -> Optional[str]:
- for col in self.columns:
- if col.get("is_dttm"):
- return col.get("column_name") # type: ignore
- return None
-
- @property
- def dttm_cols(self) -> List[Any]:
- return [col.get("column_name") for col in self.columns if col.get("is_dttm")]
-
- @property
- def default_endpoint(self) -> str:
- return ""
-
- @staticmethod
- def get_extra_cache_keys(query_obj: Dict[str, Any]) -> List[str]:
- return []
-
class SavedQuery(Model, AuditMixinNullable, ExtraJSONMixin, ImportExportMixin):
"""ORM model for SQL query"""
diff --git a/superset/utils/core.py b/superset/utils/core.py
index 44d76ea533..ceb49268b1 100644
--- a/superset/utils/core.py
+++ b/superset/utils/core.py
@@ -1674,23 +1674,26 @@ def extract_dataframe_dtypes(
"date": GenericDataType.TEMPORAL,
}
- columns_by_name: Dict[str, Any] = {}
- if datasource:
- for column in datasource.columns:
- if isinstance(column, dict):
- columns_by_name[column.get("column_name")] = column
- else:
- columns_by_name[column.column_name] = column
-
+ # todo(hughhhh): can we make the column_object a Union
+ if datasource and datasource.type == "query":
+ columns_by_name = {column.get('column_name'): column for column in datasource.columns}
+ else:
+ columns_by_name = (
+ {column.column_name: column for column in datasource.columns}
+ if datasource
+ else {}
+ )
+
generic_types: List[GenericDataType] = []
for column in df.columns:
column_object = columns_by_name.get(column)
series = df[column]
inferred_type = infer_dtype(series)
- if isinstance(column_object, dict): # type: ignore
+ # todo(hughhhh): can we make the column_object a Union
+ if datasource.type == "query":
generic_type = (
GenericDataType.TEMPORAL
- if column_object and column_object.get("is_dttm")
+ if column_object and column_object.get('is_dttm')
else inferred_type_map.get(inferred_type, GenericDataType.STRING)
)
else:
@@ -1733,15 +1736,9 @@ def get_time_filter_status( # pylint: disable=too-many-branches
applied_time_extras: Dict[str, str],
) -> Tuple[List[Dict[str, str]], List[Dict[str, str]]]:
- temporal_columns: Set[Any]
- if datasource.type == "query":
- temporal_columns = {
- col.get("column_name") for col in datasource.columns if col.get("is_dttm")
- }
- else:
- temporal_columns = {
- col.column_name for col in datasource.columns if col.is_dttm
- }
+ # todo(hugh): fix this
+ # temporal_columns = {col.column_name for col in datasource.columns if col.is_dttm}
+ temporal_columns = {}
applied: List[Dict[str, str]] = []
rejected: List[Dict[str, str]] = []
time_column = applied_time_extras.get(ExtraFiltersTimeColumnType.TIME_COL)
diff --git a/superset/views/core.py b/superset/views/core.py
index 5236ebc494..75848a29c9 100755
--- a/superset/views/core.py
+++ b/superset/views/core.py
@@ -753,7 +753,6 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
self.__class__.__name__,
)
initial_form_data = {}
-
form_data_key = request.args.get("form_data_key")
if key is not None:
command = GetExplorePermalinkCommand(key)
@@ -777,6 +776,132 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
value = GetFormDataCommand(parameters).run()
initial_form_data = json.loads(value) if value else {}
+ from superset.dao.datasource.dao import DatasourceDAO
+ from superset.utils.core import DatasourceType
+ from superset.models.helpers import ExploreMixin
+
+ # Handle SIP-68 Models or explore view
+ # API will always use /explore/<datasource_type>/<int:datasource_id>/ to query
+ # new models to power any viz in explore
+ if datasource_id and datasource_type:
+ # 1. Query datasource object by type and id
+ datasource = DatasourceDAO.get_datasource(
+ session=db.session,
+ datasource_type=DatasourceType(datasource_type),
+ datasource_id=datasource_id,
+ )
+
+ # 2. Verify that it's an ExploreMixin
+ if isinstance(datasource, ExploreMixin):
+ # Handle Query object bootstrap
+ datasource_name = datasource.name if datasource else _("[Missing Dataset]")
+ form_data, slc = get_form_data(
+ use_slice_data=True, initial_form_data=initial_form_data
+ )
+
+ query_context = request.form.get("query_context")
+
+ viz_type = form_data.get("viz_type", "table")
+ if not viz_type and datasource and datasource.default_endpoint:
+ return redirect(datasource.default_endpoint)
+
+ # slc perms
+ slice_add_perm = security_manager.can_access("can_write", "Chart")
+ slice_overwrite_perm = is_owner(slc, g.user) if slc else False
+ slice_download_perm = security_manager.can_access("can_csv", "Superset")
+
+ form_data["datasource"] = (
+ str(datasource_id) + "__" + cast(str, datasource_type)
+ )
+
+ # On explore, merge legacy and extra filters into the form data
+ utils.convert_legacy_filters_into_adhoc(form_data)
+ utils.merge_extra_filters(form_data)
+
+ # merge request url params
+ if request.method == "GET":
+ utils.merge_request_params(form_data, request.args)
+
+ # handle save or overwrite
+ action = request.args.get("action")
+
+ if action == "overwrite" and not slice_overwrite_perm:
+ return json_error_response(
+ _("You don't have the rights to ") + _("alter this ") + _("chart"),
+ status=403,
+ )
+
+ if action == "saveas" and not slice_add_perm:
+ return json_error_response(
+ _("You don't have the rights to ") + _("create a ") + _("chart"),
+ status=403,
+ )
+
+ if action in ("saveas", "overwrite") and datasource:
+ return self.save_or_overwrite_slice(
+ slc,
+ slice_add_perm,
+ slice_overwrite_perm,
+ slice_download_perm,
+ datasource.id,
+ datasource.type,
+ datasource.name,
+ query_context,
+ )
+ standalone_mode = ReservedUrlParameters.is_standalone_mode()
+ force = request.args.get("force") in {"force", "1", "true"}
+ dummy_datasource_data: Dict[str, Any] = {
+ "type": datasource_type,
+ "name": datasource_name,
+ "columns": [],
+ "metrics": [],
+ "database": {"id": 0, "backend": ""},
+ }
+ try:
+ datasource_data = (
+ datasource.data if datasource else dummy_datasource_data
+ )
+ except (SupersetException, SQLAlchemyError):
+ datasource_data = dummy_datasource_data
+
+ columns: List[Dict[str, Any]] = []
+ if datasource:
+ datasource_data["owners"] = datasource.owners_data
+ if isinstance(datasource, Query):
+ # todo(hughhh): set is_dttm + name -> column_name
+ # datasource_data["data"] = datasource.data
+ # move all this logic into the class for the property data
+ datasource_data["columns"] = datasource.columns
+ datasource_data["metrics"] = datasource.extra.get("metrics", [])
+ datasource_data["id"] = datasource_id
+ datasource_data["type"] = datasource_type
+
+ bootstrap_data = {
+ "can_add": slice_add_perm,
+ "can_download": slice_download_perm,
+ "datasource": sanitize_datasource_data(datasource_data),
+ "form_data": form_data,
+ "datasource_id": datasource_id,
+ "datasource_type": datasource_type,
+ "slice": slc.data if slc else None,
+ "standalone": standalone_mode,
+ "force": force,
+ "user": bootstrap_user_data(g.user, include_perms=True),
+ "forced_height": request.args.get("height"),
+ "common": common_bootstrap_payload(),
+ }
+
+ title = _("Explore - %(name)s", name=datasource.name)
+ return self.render_template(
+ "superset/basic.html",
+ bootstrap_data=json.dumps(
+ bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
+ ),
+ entry="explore",
+ title=title.__str__(),
+ standalone_mode=standalone_mode,
+ )
+
if not initial_form_data:
slice_id = request.args.get("slice_id")
dataset_id = request.args.get("dataset_id")