You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by ma...@apache.org on 2020/11/25 16:45:27 UTC
[incubator-superset] branch master updated: chore: improve
analytics (#11714)
This is an automated email from the ASF dual-hosted git repository.
maximebeauchemin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push:
new 0504cf1 chore: improve analytics (#11714)
0504cf1 is described below
commit 0504cf1a00d7b2b8e96bbddc93fcc737c7602b73
Author: Maxime Beauchemin <ma...@gmail.com>
AuthorDate: Wed Nov 25 08:45:02 2020 -0800
chore: improve analytics (#11714)
* chore: improve analytics
* lint
* log more events, add note in UPDATING.md
* handling base class
* more events\!
* get ref through
* right before @expose
* fix context
* touchups
---
UPDATING.md | 5 +-
superset/annotation_layers/api.py | 5 ++
superset/cachekeys/api.py | 2 +-
superset/charts/api.py | 11 +++-
superset/css_templates/api.py | 2 +
superset/dashboards/api.py | 8 +++
superset/databases/api.py | 12 +++-
superset/datasets/api.py | 9 ++-
.../versions/a8173232b786_add_path_to_logs.py | 45 +++++++++++++
superset/models/core.py | 3 +
superset/utils/log.py | 75 ++++++++++++++++++----
superset/views/base_api.py | 51 +++++++++------
superset/views/core.py | 51 +++++++++++----
13 files changed, 229 insertions(+), 50 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 2b8385f..c1ae805 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -25,7 +25,10 @@ assists people when migrating to a new version.
## Next
- [11704](https://github.com/apache/incubator-superset/pull/11704) Breaking change: Jinja templating for SQL queries has been updated, removing default modules such as `datetime` and `random` and enforcing static template values. To restore or extend functionality, use `JINJA_CONTEXT_ADDONS` and `CUSTOM_TEMPLATE_PROCESSORS` in `superset_config.py`.
-
+- [11714](https://github.com/apache/incubator-superset/pull/11714): Logs
+ significantly more analytics events (roughly double?), and when
+ using DBEventLogger (default) could result in stressing the metadata
+ database more.
- [11509](https://github.com/apache/incubator-superset/pull/11509): Config value `TABLE_NAMES_CACHE_CONFIG` has been renamed to `DATA_CACHE_CONFIG`, which will now also hold query results cache from connected datasources (previously held in `CACHE_CONFIG`), in addition to the table names. If you will set `DATA_CACHE_CONFIG` to a new cache backend different than your previous `CACHE_CONFIG`, plan for additional cache warmup to avoid degrading charting performance for the end users.
- [11575](https://github.com/apache/incubator-superset/pull/11575) The Row Level Security (RLS) config flag has been moved to a feature flag. To migrate, add `ROW_LEVEL_SECURITY: True` to the `FEATURE_FLAGS` dict in `superset_config.py`.
diff --git a/superset/annotation_layers/api.py b/superset/annotation_layers/api.py
index 31fa7a0..c608e30 100644
--- a/superset/annotation_layers/api.py
+++ b/superset/annotation_layers/api.py
@@ -47,6 +47,7 @@ from superset.annotation_layers.schemas import (
openapi_spec_methods_override,
)
from superset.constants import RouteMethod
+from superset.extensions import event_logger
from superset.models.annotations import AnnotationLayer
from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics
@@ -110,6 +111,7 @@ class AnnotationLayerRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@permission_name("delete")
+ @event_logger.log_this_with_context(log_to_statsd=False)
def delete(self, pk: int) -> Response:
"""Delete an annotation layer
---
@@ -159,6 +161,7 @@ class AnnotationLayerRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@permission_name("post")
+ @event_logger.log_this_with_context(log_to_statsd=False)
def post(self) -> Response:
"""Creates a new Annotation Layer
---
@@ -218,6 +221,7 @@ class AnnotationLayerRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@permission_name("put")
+ @event_logger.log_this_with_context(log_to_statsd=False)
def put(self, pk: int) -> Response:
"""Updates an Annotation Layer
---
@@ -284,6 +288,7 @@ class AnnotationLayerRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_delete_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Delete bulk Annotation layers
---
diff --git a/superset/cachekeys/api.py b/superset/cachekeys/api.py
index 94a8c4b..8548898 100644
--- a/superset/cachekeys/api.py
+++ b/superset/cachekeys/api.py
@@ -45,10 +45,10 @@ class CacheRestApi(BaseSupersetModelRestApi):
openapi_spec_component_schemas = (CacheInvalidationRequestSchema,)
@expose("/invalidate", methods=["POST"])
- @event_logger.log_this
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def invalidate(self) -> Response:
"""
Takes a list of datasources, finds the associated cache records and
diff --git a/superset/charts/api.py b/superset/charts/api.py
index 8e8789d..cb6a33d 100644
--- a/superset/charts/api.py
+++ b/superset/charts/api.py
@@ -214,6 +214,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def post(self) -> Response:
"""Creates a new Chart
---
@@ -270,6 +271,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def put(self, pk: int) -> Response:
"""Changes a Chart
---
@@ -343,6 +345,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def delete(self, pk: int) -> Response:
"""Deletes a Chart
---
@@ -393,6 +396,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_delete_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Delete bulk Charts
---
@@ -444,10 +448,10 @@ class ChartRestApi(BaseSupersetModelRestApi):
return self.response_422(message=str(ex))
@expose("/data", methods=["POST"])
- @event_logger.log_this
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def data(self) -> Response:
"""
Takes a query context constructed in the client and returns payload
@@ -532,6 +536,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@rison(screenshot_query_schema)
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def cache_screenshot(self, pk: int, **kwargs: Dict[str, bool]) -> WerkzeugResponse:
"""
---
@@ -604,6 +609,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@rison(screenshot_query_schema)
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def screenshot(self, pk: int, digest: str) -> WerkzeugResponse:
"""Get Chart screenshot
---
@@ -657,6 +663,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@rison(thumbnail_query_schema)
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def thumbnail(
self, pk: int, digest: str, **kwargs: Dict[str, bool]
) -> WerkzeugResponse:
@@ -730,6 +737,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_export_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def export(self, **kwargs: Any) -> Response:
"""Export charts
---
@@ -787,6 +795,7 @@ class ChartRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_fav_star_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def favorite_status(self, **kwargs: Any) -> Response:
"""Favorite stars for Charts
---
diff --git a/superset/css_templates/api.py b/superset/css_templates/api.py
index e59fd51..8e6f07a 100644
--- a/superset/css_templates/api.py
+++ b/superset/css_templates/api.py
@@ -33,6 +33,7 @@ from superset.css_templates.schemas import (
get_delete_ids_schema,
openapi_spec_methods_override,
)
+from superset.extensions import event_logger
from superset.models.core import CssTemplate
from superset.views.base_api import BaseSupersetModelRestApi, statsd_metrics
@@ -87,6 +88,7 @@ class CssTemplateRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_delete_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Delete bulk CSS Templates
---
diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py
index f16f358..da8e2d1 100644
--- a/superset/dashboards/api.py
+++ b/superset/dashboards/api.py
@@ -63,6 +63,7 @@ from superset.dashboards.schemas import (
openapi_spec_methods_override,
thumbnail_query_schema,
)
+from superset.extensions import event_logger
from superset.models.dashboard import Dashboard
from superset.tasks.thumbnails import cache_dashboard_thumbnail
from superset.utils.screenshots import DashboardScreenshot
@@ -209,6 +210,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def post(self) -> Response:
"""Creates a new Dashboard
---
@@ -267,6 +269,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def put(self, pk: int) -> Response:
"""Changes a Dashboard
---
@@ -337,6 +340,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def delete(self, pk: int) -> Response:
"""Deletes a Dashboard
---
@@ -387,6 +391,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_delete_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Delete bulk Dashboards
---
@@ -444,6 +449,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_export_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def export(self, **kwargs: Any) -> Response:
"""Export dashboards
---
@@ -519,6 +525,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@rison(thumbnail_query_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def thumbnail(
self, pk: int, digest: str, **kwargs: Dict[str, bool]
) -> WerkzeugResponse:
@@ -606,6 +613,7 @@ class DashboardRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_fav_star_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def favorite_status(self, **kwargs: Any) -> Response:
"""Favorite Stars for Dashboards
---
diff --git a/superset/databases/api.py b/superset/databases/api.py
index aa11a62..51c2eda 100644
--- a/superset/databases/api.py
+++ b/superset/databases/api.py
@@ -185,6 +185,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def post(self) -> Response:
"""Creates a new Database
---
@@ -247,6 +248,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def put( # pylint: disable=too-many-return-statements, arguments-differ
self, pk: int
) -> Response:
@@ -320,6 +322,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def delete(self, pk: int) -> Response: # pylint: disable=arguments-differ
"""Deletes a Database
---
@@ -370,6 +373,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@safe
@rison(database_schemas_query_schema)
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def schemas(self, pk: int, **kwargs: Any) -> FlaskResponse:
"""Get all schemas from a database
---
@@ -423,8 +427,8 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@check_datasource_access
@safe
- @event_logger.log_this
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def table_metadata(
self, database: Database, table_name: str, schema_name: str
) -> FlaskResponse:
@@ -480,8 +484,8 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@check_datasource_access
@safe
- @event_logger.log_this
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def select_star(
self, database: Database, table_name: str, schema_name: Optional[str] = None
) -> FlaskResponse:
@@ -537,8 +541,8 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@expose("/test_connection", methods=["POST"])
@protect()
@safe
- @event_logger.log_this
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def test_connection( # pylint: disable=too-many-return-statements
self,
) -> FlaskResponse:
@@ -618,6 +622,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def related_objects(self, pk: int) -> Response:
"""Get charts and dashboards count associated to a database
---
@@ -676,6 +681,7 @@ class DatabaseRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_export_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def export(self, **kwargs: Any) -> Response:
"""Export database(s) with associated datasets
---
diff --git a/superset/datasets/api.py b/superset/datasets/api.py
index 3609f8f..cd2c594 100644
--- a/superset/datasets/api.py
+++ b/superset/datasets/api.py
@@ -27,7 +27,7 @@ from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import ngettext
from marshmallow import ValidationError
-from superset import is_feature_enabled
+from superset import event_logger, is_feature_enabled
from superset.commands.exceptions import CommandInvalidError
from superset.connectors.sqla.models import SqlaTable
from superset.constants import RouteMethod
@@ -182,6 +182,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def post(self) -> Response:
"""Creates a new Dataset
---
@@ -238,6 +239,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def put(self, pk: int) -> Response:
"""Changes a Dataset
---
@@ -308,6 +310,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def delete(self, pk: int) -> Response:
"""Deletes a Dataset
---
@@ -358,6 +361,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_export_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def export(self, **kwargs: Any) -> Response:
"""Export datasets
---
@@ -433,6 +437,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def refresh(self, pk: int) -> Response:
"""Refresh a Dataset
---
@@ -482,6 +487,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@protect()
@safe
@statsd_metrics
+ @event_logger.log_this_with_context(log_to_statsd=False)
def related_objects(self, pk: int) -> Response:
"""Get charts and dashboards count associated to a dataset
---
@@ -540,6 +546,7 @@ class DatasetRestApi(BaseSupersetModelRestApi):
@safe
@statsd_metrics
@rison(get_delete_ids_schema)
+ @event_logger.log_this_with_context(log_to_statsd=False)
def bulk_delete(self, **kwargs: Any) -> Response:
"""Delete bulk Datasets
---
diff --git a/superset/migrations/versions/a8173232b786_add_path_to_logs.py b/superset/migrations/versions/a8173232b786_add_path_to_logs.py
new file mode 100644
index 0000000..d88f324
--- /dev/null
+++ b/superset/migrations/versions/a8173232b786_add_path_to_logs.py
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Add path to logs
+
+Revision ID: a8173232b786
+Revises: 49b5a32daba5
+Create Date: 2020-11-15 16:08:24.580764
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "a8173232b786"
+down_revision = "49b5a32daba5"
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+
+def upgrade():
+ op.add_column("logs", sa.Column("path", sa.String(length=256), nullable=True))
+ op.add_column(
+ "logs", sa.Column("path_no_int", sa.String(length=256), nullable=True)
+ )
+ op.add_column("logs", sa.Column("ref", sa.String(length=256), nullable=True))
+
+
+def downgrade():
+ op.drop_column("logs", "path")
+ op.drop_column("logs", "path_no_int")
+ op.drop_column("logs", "ref")
diff --git a/superset/models/core.py b/superset/models/core.py
index c4a8369..2ceff54 100755
--- a/superset/models/core.py
+++ b/superset/models/core.py
@@ -715,6 +715,9 @@ class Log(Model): # pylint: disable=too-few-public-methods
dttm = Column(DateTime, default=datetime.utcnow)
duration_ms = Column(Integer)
referrer = Column(String(1024))
+ path = Column(String(256))
+ path_no_int = Column(String(256))
+ ref = Column(String(256))
class FavStarClassName(str, Enum):
diff --git a/superset/utils/log.py b/superset/utils/log.py
index 4380006..24c9161 100644
--- a/superset/utils/log.py
+++ b/superset/utils/log.py
@@ -30,15 +30,35 @@ from sqlalchemy.exc import SQLAlchemyError
from superset.stats_logger import BaseStatsLogger
+def strip_int_from_path(path: Optional[str]) -> str:
+ """Simple function to remove ints from '/' separated paths"""
+ if path:
+ return "/".join(["<int>" if s.isdigit() else s for s in path.split("/")])
+ return ""
+
+
class AbstractEventLogger(ABC):
@abstractmethod
- def log(
- self, user_id: Optional[int], action: str, *args: Any, **kwargs: Any
+ def log( # pylint: disable=too-many-arguments
+ self,
+ user_id: Optional[int],
+ action: str,
+ dashboard_id: Optional[int],
+ duration_ms: Optional[int],
+ slice_id: Optional[int],
+ path: Optional[str],
+ path_no_int: Optional[str],
+ ref: Optional[str],
+ referrer: Optional[str],
+ *args: Any,
+ **kwargs: Any,
) -> None:
pass
@contextmanager
- def log_context(self, action: str) -> Iterator[Callable[..., None]]:
+ def log_context(
+ self, action: str, ref: Optional[str] = None, log_to_statsd: bool = True,
+ ) -> Iterator[Callable[..., None]]:
"""
Log an event while reading information from the request context.
`kwargs` will be appended directly to the log payload.
@@ -69,7 +89,8 @@ class AbstractEventLogger(ABC):
except (TypeError, ValueError):
slice_id = 0
- self.stats_logger.incr(action)
+ if log_to_statsd:
+ self.stats_logger.incr(action)
# bulk insert
try:
@@ -86,18 +107,38 @@ class AbstractEventLogger(ABC):
slice_id=slice_id,
duration_ms=round((time.time() - start_time) * 1000),
referrer=referrer,
+ path=request.path,
+ path_no_int=strip_int_from_path(request.path),
+ ref=ref,
)
- def log_this(self, f: Callable[..., Any]) -> Callable[..., Any]:
+ def _wrapper(
+ self, f: Callable[..., Any], **wrapper_kwargs: Any
+ ) -> Callable[..., Any]:
+ action_str = wrapper_kwargs.get("action") or f.__name__
+ ref = f.__qualname__ if hasattr(f, "__qualname__") else None
+
@functools.wraps(f)
def wrapper(*args: Any, **kwargs: Any) -> Any:
- with self.log_context(f.__name__) as log:
+ with self.log_context(action_str, ref, **wrapper_kwargs) as log:
value = f(*args, **kwargs)
log(**kwargs)
return value
return wrapper
+ def log_this(self, f: Callable[..., Any]) -> Callable[..., Any]:
+ """Decorator that uses the function name as the action"""
+ return self._wrapper(f)
+
+ def log_this_with_context(self, **kwargs: Any) -> Callable[..., Any]:
+ """Decorator that can override kwargs of log_context"""
+
+ def func(f: Callable[..., Any]) -> Callable[..., Any]:
+ return self._wrapper(f, **kwargs)
+
+ return func
+
def log_manually(self, f: Callable[..., Any]) -> Callable[..., Any]:
"""Allow a function to manually update"""
@@ -162,16 +203,23 @@ def get_event_logger_from_cfg_value(cfg_value: Any) -> AbstractEventLogger:
class DBEventLogger(AbstractEventLogger):
"""Event logger that commits logs to Superset DB"""
- def log( # pylint: disable=too-many-locals
- self, user_id: Optional[int], action: str, *args: Any, **kwargs: Any
+ def log( # pylint: disable=too-many-arguments,too-many-locals
+ self,
+ user_id: Optional[int],
+ action: str,
+ dashboard_id: Optional[int],
+ duration_ms: Optional[int],
+ slice_id: Optional[int],
+ path: Optional[str],
+ path_no_int: Optional[str],
+ ref: Optional[str],
+ referrer: Optional[str],
+ *args: Any,
+ **kwargs: Any,
) -> None:
from superset.models.core import Log
records = kwargs.get("records", list())
- dashboard_id = kwargs.get("dashboard_id")
- slice_id = kwargs.get("slice_id")
- duration_ms = kwargs.get("duration_ms")
- referrer = kwargs.get("referrer")
logs = list()
for record in records:
@@ -188,6 +236,9 @@ class DBEventLogger(AbstractEventLogger):
duration_ms=duration_ms,
referrer=referrer,
user_id=user_id,
+ path=path,
+ path_no_int=path_no_int,
+ ref=ref,
)
logs.append(log)
try:
diff --git a/superset/views/base_api.py b/superset/views/base_api.py
index 1495a79..0f05f07 100644
--- a/superset/views/base_api.py
+++ b/superset/views/base_api.py
@@ -31,7 +31,7 @@ from marshmallow import fields, Schema
from sqlalchemy import and_, distinct, func
from sqlalchemy.orm.query import Query
-from superset.extensions import db, security_manager
+from superset.extensions import db, event_logger, security_manager
from superset.models.core import FavStar
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
@@ -49,6 +49,7 @@ get_related_schema = {
"filter": {"type": "string"},
},
}
+log_context = event_logger.log_context
class RelatedResultResponseSchema(Schema):
@@ -312,49 +313,61 @@ class BaseSupersetModelRestApi(ModelRestApi):
"""
Add statsd metrics to builtin FAB _info endpoint
"""
- duration, response = time_function(super().info_headless, **kwargs)
- self.send_stats_metrics(response, self.info.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.info"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().info_headless, **kwargs)
+ self.send_stats_metrics(response, self.info.__name__, duration)
+ return response
def get_headless(self, pk: int, **kwargs: Any) -> Response:
"""
Add statsd metrics to builtin FAB GET endpoint
"""
- duration, response = time_function(super().get_headless, pk, **kwargs)
- self.send_stats_metrics(response, self.get.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.get"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().get_headless, pk, **kwargs)
+ self.send_stats_metrics(response, self.get.__name__, duration)
+ return response
def get_list_headless(self, **kwargs: Any) -> Response:
"""
Add statsd metrics to builtin FAB GET list endpoint
"""
- duration, response = time_function(super().get_list_headless, **kwargs)
- self.send_stats_metrics(response, self.get_list.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.get_list"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().get_list_headless, **kwargs)
+ self.send_stats_metrics(response, self.get_list.__name__, duration)
+ return response
def post_headless(self) -> Response:
"""
Add statsd metrics to builtin FAB POST endpoint
"""
- duration, response = time_function(super().post_headless)
- self.send_stats_metrics(response, self.post.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.post"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().post_headless)
+ self.send_stats_metrics(response, self.post.__name__, duration)
+ return response
def put_headless(self, pk: int) -> Response:
"""
Add statsd metrics to builtin FAB PUT endpoint
"""
- duration, response = time_function(super().put_headless, pk)
- self.send_stats_metrics(response, self.put.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.put"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().put_headless, pk)
+ self.send_stats_metrics(response, self.put.__name__, duration)
+ return response
def delete_headless(self, pk: int) -> Response:
"""
Add statsd metrics to builtin FAB DELETE endpoint
"""
- duration, response = time_function(super().delete_headless, pk)
- self.send_stats_metrics(response, self.delete.__name__, duration)
- return response
+ ref = f"{self.__class__.__name__}.delete"
+ with log_context(ref, ref, log_to_statsd=False):
+ duration, response = time_function(super().delete_headless, pk)
+ self.send_stats_metrics(response, self.delete.__name__, duration)
+ return response
@expose("/related/<column_name>", methods=["GET"])
@protect()
diff --git a/superset/views/core.py b/superset/views/core.py
index a898faf..72cf12c 100755
--- a/superset/views/core.py
+++ b/superset/views/core.py
@@ -155,6 +155,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
logger = logging.getLogger(__name__)
@has_access_api
+ @event_logger.log_this
@expose("/datasources/")
def datasources(self) -> FlaskResponse:
return self.json_response(
@@ -169,6 +170,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
@has_access_api
+ @event_logger.log_this
@expose("/override_role_permissions/", methods=["POST"])
def override_role_permissions(self) -> FlaskResponse:
"""Updates the role with the give datasource permissions.
@@ -220,8 +222,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
{"granted": granted_perms, "requested": list(db_ds_names)}, status=201
)
- @event_logger.log_this
@has_access
+ @event_logger.log_this
@expose("/request_access/")
def request_access(self) -> FlaskResponse:
datasources = set()
@@ -263,8 +265,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
datasource_names=", ".join([o.name for o in datasources]),
)
- @event_logger.log_this
@has_access
+ @event_logger.log_this
@expose("/approve")
def approve(self) -> FlaskResponse: # pylint: disable=too-many-locals,no-self-use
def clean_fulfilled_requests(session: Session) -> None:
@@ -368,6 +370,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
return redirect("/accessrequestsmodelview/list/")
@has_access
+ @event_logger.log_this
@expose("/slice/<int:slice_id>/")
def slice(self, slice_id: int) -> FlaskResponse: # pylint: disable=no-self-use
_, slc = get_form_data(slice_id, use_slice_data=True)
@@ -450,9 +453,9 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
except SupersetException as ex:
return json_error_response(utils.error_msg_from_exception(ex))
- @event_logger.log_this
@api
@has_access_api
+ @event_logger.log_this
@expose("/annotation_json/<int:layer_id>")
def annotation_json( # pylint: disable=no-self-use
self, layer_id: int
@@ -484,10 +487,10 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
if not is_feature_enabled("ENABLE_EXPLORE_JSON_CSRF_PROTECTION"):
EXPLORE_JSON_METHODS.append("GET")
- @event_logger.log_this
@api
@has_access_api
@handle_api_exception
+ @event_logger.log_this
@expose(
"/explore_json/<datasource_type>/<int:datasource_id>/",
methods=EXPLORE_JSON_METHODS,
@@ -535,8 +538,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
except SupersetException as ex:
return json_error_response(utils.error_msg_from_exception(ex), 400)
- @event_logger.log_this
@has_access
+ @event_logger.log_this
@expose("/import_dashboards", methods=["GET", "POST"])
def import_dashboards(self) -> FlaskResponse:
"""Overrides the dashboards using json instances from the file."""
@@ -578,8 +581,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
"superset/import_dashboards.html", databases=databases
)
- @event_logger.log_this
@has_access
+ @event_logger.log_this
@expose("/explore/<datasource_type>/<int:datasource_id>/", methods=["GET", "POST"])
@expose("/explore/", methods=["GET", "POST"])
def explore( # pylint: disable=too-many-locals,too-many-return-statements
@@ -733,6 +736,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@handle_api_exception
@has_access_api
+ @event_logger.log_this
@expose("/filter/<datasource_type>/<int:datasource_id>/<column>/")
def filter( # pylint: disable=no-self-use
self, datasource_type: str, datasource_id: int, column: str
@@ -881,6 +885,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/schemas/<int:db_id>/")
@expose("/schemas/<int:db_id>/<force_refresh>/")
def schemas( # pylint: disable=no-self-use
@@ -905,6 +910,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/tables/<int:db_id>/<schema>/<substr>/")
@expose("/tables/<int:db_id>/<schema>/<substr>/<force_refresh>/")
def tables( # pylint: disable=too-many-locals,no-self-use
@@ -1014,6 +1020,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/copy_dash/<int:dashboard_id>/", methods=["GET", "POST"])
def copy_dash( # pylint: disable=no-self-use
self, dashboard_id: int
@@ -1063,6 +1070,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/save_dash/<int:dashboard_id>/", methods=["GET", "POST"])
def save_dash( # pylint: disable=no-self-use
self, dashboard_id: int
@@ -1101,6 +1109,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/add_slices/<int:dashboard_id>/", methods=["POST"])
def add_slices( # pylint: disable=no-self-use
self, dashboard_id: int
@@ -1119,6 +1128,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/testconn", methods=["POST", "GET"])
def testconn( # pylint: disable=too-many-return-statements,no-self-use
self,
@@ -1199,6 +1209,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/recent_activity/<int:user_id>/", methods=["GET"])
def recent_activity( # pylint: disable=no-self-use
self, user_id: int
@@ -1297,6 +1308,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/csrf_token/", methods=["GET"])
def csrf_token(self) -> FlaskResponse:
return Response(
@@ -1305,6 +1317,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/available_domains/", methods=["GET"])
def available_domains(self) -> FlaskResponse: # pylint: disable=no-self-use
"""
@@ -1318,6 +1331,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/fave_dashboards_by_username/<username>/", methods=["GET"])
def fave_dashboards_by_username(self, username: str) -> FlaskResponse:
"""This lets us use a user's username to pull favourite dashboards"""
@@ -1326,6 +1340,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/fave_dashboards/<int:user_id>/", methods=["GET"])
def fave_dashboards( # pylint: disable=no-self-use
self, user_id: int
@@ -1360,6 +1375,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/created_dashboards/<int:user_id>/", methods=["GET"])
def created_dashboards( # pylint: disable=no-self-use
self, user_id: int
@@ -1388,6 +1404,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/user_slices", methods=["GET"])
@expose("/user_slices/<int:user_id>/", methods=["GET"])
def user_slices( # pylint: disable=no-self-use
@@ -1439,6 +1456,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/created_slices", methods=["GET"])
@expose("/created_slices/<int:user_id>/", methods=["GET"])
def created_slices( # pylint: disable=no-self-use
@@ -1466,6 +1484,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/fave_slices", methods=["GET"])
@expose("/fave_slices/<int:user_id>/", methods=["GET"])
def fave_slices( # pylint: disable=no-self-use
@@ -1596,6 +1615,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
return json_success(json.dumps(result))
@has_access_api
+ @event_logger.log_this
@expose("/favstar/<class_name>/<int:obj_id>/<action>/")
def favstar( # pylint: disable=no-self-use
self, class_name: str, obj_id: int, action: str
@@ -1629,6 +1649,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/dashboard/<int:dashboard_id>/published/", methods=("GET", "POST"))
def publish( # pylint: disable=no-self-use
self, dashboard_id: int
@@ -1775,7 +1796,6 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
@api
- @event_logger.log_this
@has_access
@expose("/log/", methods=["POST"])
def log(self) -> FlaskResponse: # pylint: disable=no-self-use
@@ -2118,8 +2138,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
return self.json_response("OK")
@has_access_api
- @expose("/validate_sql_json/", methods=["POST", "GET"])
@event_logger.log_this
+ @expose("/validate_sql_json/", methods=["POST", "GET"])
def validate_sql_json( # pylint: disable=too-many-locals,too-many-return-statements,no-self-use
self,
) -> FlaskResponse:
@@ -2304,8 +2324,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@has_access_api
@handle_api_exception
- @expose("/sql_json/", methods=["POST"])
@event_logger.log_this
+ @expose("/sql_json/", methods=["POST"])
def sql_json(self) -> FlaskResponse:
log_params = {
"user_agent": cast(Optional[str], request.headers.get("USER_AGENT"))
@@ -2438,8 +2458,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
@has_access
- @expose("/csv/<client_id>")
@event_logger.log_this
+ @expose("/csv/<client_id>")
def csv(self, client_id: str) -> FlaskResponse: # pylint: disable=no-self-use
"""Download the query results as csv."""
logger.info("Exporting CSV file [%s]", client_id)
@@ -2495,8 +2515,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@handle_api_exception
@has_access
- @expose("/fetch_datasource_metadata")
@event_logger.log_this
+ @expose("/fetch_datasource_metadata")
def fetch_datasource_metadata(self) -> FlaskResponse: # pylint: disable=no-self-use
"""
Fetch the datasource metadata.
@@ -2517,6 +2537,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
return json_success(json.dumps(datasource.data))
@has_access_api
+ @event_logger.log_this
@expose("/queries/<float:last_updated_ms>")
@expose("/queries/<int:last_updated_ms>")
def queries(self, last_updated_ms: Union[float, int]) -> FlaskResponse:
@@ -2550,8 +2571,8 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
return json_success(json.dumps(dict_queries, default=utils.json_int_dttm_ser))
@has_access
- @expose("/search_queries")
@event_logger.log_this
+ @expose("/search_queries")
def search_queries(self) -> FlaskResponse: # pylint: disable=no-self-use
"""
Search for previously run sqllab queries. Used for Sqllab Query Search
@@ -2621,6 +2642,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
500,
)
+ @event_logger.log_this
@expose("/welcome")
def welcome(self) -> FlaskResponse:
"""Personalized welcome page"""
@@ -2651,6 +2673,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
@has_access
+ @event_logger.log_this
@expose("/profile/<username>/")
def profile(self, username: str) -> FlaskResponse:
"""User profile page"""
@@ -2721,6 +2744,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
}
@has_access
+ @event_logger.log_this
@expose("/sqllab", methods=["GET", "POST"])
def sqllab(self) -> FlaskResponse:
"""SQL Editor"""
@@ -2747,7 +2771,9 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
)
@has_access
+ @event_logger.log_this
@expose("/sqllab/history/", methods=["GET"])
+ @event_logger.log_this
def sqllab_search(self) -> FlaskResponse:
if not (
is_feature_enabled("ENABLE_REACT_CRUD_VIEWS")
@@ -2759,6 +2785,7 @@ class Superset(BaseSupersetView): # pylint: disable=too-many-public-methods
@api
@has_access_api
+ @event_logger.log_this
@expose("/schemas_access_for_csv_upload")
def schemas_access_for_csv_upload(self) -> FlaskResponse:
"""