You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@superset.apache.org by be...@apache.org on 2026/02/03 16:46:29 UTC
(superset) branch sl-1-extension-loading created (now 7cb7988e80b)
This is an automated email from the ASF dual-hosted git repository.
beto pushed a change to branch sl-1-extension-loading
in repository https://gitbox.apache.org/repos/asf/superset.git
at 7cb7988e80b feat(extensions): improve extension loading with Pydantic compatibility
This branch includes the following new commits:
new 7cb7988e80b feat(extensions): improve extension loading with Pydantic compatibility
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(superset) 01/01: feat(extensions): improve extension loading with Pydantic compatibility
Posted by be...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
beto pushed a commit to branch sl-1-extension-loading
in repository https://gitbox.apache.org/repos/asf/superset.git
commit 7cb7988e80b9f9b68ee7a45c83151ab221b0d52a
Author: Beto Dealmeida <ro...@dealmeida.net>
AuthorDate: Mon Feb 2 17:23:53 2026 -0500
feat(extensions): improve extension loading with Pydantic compatibility
- Add Pydantic v1/v2 compatibility for manifest validation
- Simplify extension initialization in SupersetAppInitializer
- Fix docker-compose-light.yml for development:
- Add superset-core volume mount
- Fix EXAMPLES_HOST to use db-light service
- Change WEBPACK_DEVSERVER_HOST default to 0.0.0.0
- Add EXTENSIONS_PATH config support
Co-Authored-By: Claude Opus 4.5 <no...@anthropic.com>
---
docker-compose-light.yml | 8 ++-
docker/pythonpath_dev/superset_config.py | 10 +++-
superset/extensions/utils.py | 91 +++++++++++++++++++++++++++++++-
superset/initialization/__init__.py | 27 ++--------
4 files changed, 110 insertions(+), 26 deletions(-)
diff --git a/docker-compose-light.yml b/docker-compose-light.yml
index 09b0c65b0f8..4b805798a4e 100644
--- a/docker-compose-light.yml
+++ b/docker-compose-light.yml
@@ -64,9 +64,11 @@ x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
- ./superset:/app/superset
+ - ./superset-core:/app/superset-core
- ./superset-frontend:/app/superset-frontend
- superset_home_light:/app/superset_home
- ./tests:/app/tests
+ - ./extensions:/app/extensions
x-common-build: &common-build
context: .
target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean`
@@ -138,6 +140,10 @@ services:
DATABASE_DB: superset_light
POSTGRES_DB: superset_light
SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py
+ # Override examples host to use light DB service
+ EXAMPLES_HOST: db-light
+ # Skip example loading for faster startup
+ SUPERSET_LOAD_EXAMPLES: "no"
healthcheck:
disable: true
@@ -160,7 +166,7 @@ services:
# configuring the dev-server to use the host.docker.internal to connect to the backend
superset: "http://superset-light:8088"
# Webpack dev server configuration
- WEBPACK_DEVSERVER_HOST: "${WEBPACK_DEVSERVER_HOST:-127.0.0.1}"
+ WEBPACK_DEVSERVER_HOST: "${WEBPACK_DEVSERVER_HOST:-0.0.0.0}"
WEBPACK_DEVSERVER_PORT: "${WEBPACK_DEVSERVER_PORT:-9000}"
ports:
- "${NODE_PORT:-9001}:9000" # Parameterized port, accessible on all interfaces
diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py
index d88d9899c27..e647b7827ad 100644
--- a/docker/pythonpath_dev/superset_config.py
+++ b/docker/pythonpath_dev/superset_config.py
@@ -105,7 +105,15 @@ class CeleryConfig:
CELERY_CONFIG = CeleryConfig
-FEATURE_FLAGS = {"ALERT_REPORTS": True}
+# Extensions configuration
+# For local development, point to the extensions directory
+# Note: If running in Docker, this path needs to be accessible from inside the container
+EXTENSIONS_PATH = os.getenv("EXTENSIONS_PATH", "/app/extensions")
+
+FEATURE_FLAGS = {
+ "ALERT_REPORTS": True,
+ "ENABLE_EXTENSIONS": True,
+}
ALERT_REPORTS_NOTIFICATION_DRY_RUN = True
WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/" # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/ # noqa: E501
# The base URL for the email report hyperlinks.
diff --git a/superset/extensions/utils.py b/superset/extensions/utils.py
index 883c9114728..e1a9f154292 100644
--- a/superset/extensions/utils.py
+++ b/superset/extensions/utils.py
@@ -25,6 +25,7 @@ from pathlib import Path
from typing import Any, Generator, Iterable, Tuple
from zipfile import ZipFile
+import pydantic
from flask import current_app
from pydantic import ValidationError
from superset_core.extensions.types import Manifest
@@ -35,6 +36,22 @@ from superset.utils.core import check_is_safe_zip
logger = logging.getLogger(__name__)
+# Detect Pydantic version for compatibility
+PYDANTIC_V2 = int(pydantic.VERSION.split(".")[0]) >= 2
+
+
+def _validate_manifest(data: dict[str, Any]) -> Manifest:
+ """
+ Validate manifest data using the appropriate Pydantic method.
+
+ Handles both Pydantic v1 and v2 compatibility.
+ """
+ if PYDANTIC_V2:
+ return Manifest.model_validate(data)
+ else:
+ return Manifest.parse_obj(data)
+
+
FRONTEND_REGEX = re.compile(r"^frontend/dist/([^/]+)$")
BACKEND_REGEX = re.compile(r"^backend/src/(.+)$")
@@ -171,7 +188,7 @@ def get_loaded_extension(
if filename == "manifest.json":
try:
manifest_data = json.loads(content)
- manifest = Manifest.model_validate(manifest_data)
+ manifest = _validate_manifest(manifest_data)
except ValidationError as e:
raise Exception(f"Invalid manifest.json: {e}") from e
except Exception as e:
@@ -223,6 +240,76 @@ def build_extension_data(extension: LoadedExtension) -> dict[str, Any]:
return extension_data
+def load_extension_backend(extension: LoadedExtension) -> None:
+ """
+ Load an extension's backend code and register its entry points.
+
+ This installs the extension's Python modules in-memory and registers
+ any entry points declared in the manifest (e.g., semantic layers).
+ """
+ # Install backend modules in-memory if present
+ if extension.backend:
+ install_in_memory_importer(
+ extension.backend,
+ source_base_path=extension.source_base_path,
+ )
+
+ # Register entry points from manifest
+ manifest = extension.manifest
+ if manifest.backend:
+ for ep_str in manifest.backend.entryPoints:
+ _register_entry_point(ep_str, extension.name)
+
+
+def _register_entry_point(ep_str: str, extension_name: str) -> None:
+ """
+ Parse and register a single entry point string.
+
+ Entry point format: "name = module:ClassName"
+ """
+ if "=" not in ep_str:
+ logger.warning(
+ "Invalid entry point format in extension %s: %s",
+ extension_name,
+ ep_str,
+ )
+ return
+
+ name, _, target = ep_str.partition("=")
+ name = name.strip()
+ target = target.strip()
+
+ if ":" not in target:
+ logger.warning(
+ "Invalid entry point target in extension %s: %s (expected module:Class)",
+ extension_name,
+ target,
+ )
+ return
+
+ module_path, _, class_name = target.partition(":")
+
+ try:
+ module = eager_import(module_path)
+ cls = getattr(module, class_name)
+
+ # Register with semantic layer registry
+ from superset.semantic_layers.registry import register_semantic_layer
+
+ register_semantic_layer(name, cls)
+ logger.info(
+ "Registered entry point '%s' from extension %s",
+ name,
+ extension_name,
+ )
+ except Exception:
+ logger.exception(
+ "Failed to load entry point '%s' from extension %s",
+ name,
+ extension_name,
+ )
+
+
def get_extensions() -> dict[str, LoadedExtension]:
extensions: dict[str, LoadedExtension] = {}
@@ -234,6 +321,7 @@ def get_extensions() -> dict[str, LoadedExtension]:
extension = get_loaded_extension(files, source_base_path=abs_dist_path)
extension_id = extension.manifest.id
extensions[extension_id] = extension
+ load_extension_backend(extension)
logger.info(
"Loading extension %s (ID: %s) from local filesystem",
extension.name,
@@ -248,6 +336,7 @@ def get_extensions() -> dict[str, LoadedExtension]:
extension_id = extension.manifest.id
if extension_id not in extensions: # Don't override LOCAL_EXTENSIONS
extensions[extension_id] = extension
+ load_extension_backend(extension)
logger.info(
"Loading extension %s (ID: %s) from discovery path",
extension.name,
diff --git a/superset/initialization/__init__.py b/superset/initialization/__init__.py
index adc783c1e12..4f2eb9d7648 100644
--- a/superset/initialization/__init__.py
+++ b/superset/initialization/__init__.py
@@ -546,37 +546,18 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods
self.init_extensions()
def init_extensions(self) -> None:
- from superset.extensions.utils import (
- eager_import,
- get_extensions,
- install_in_memory_importer,
- )
+ from superset.extensions.utils import get_extensions
try:
- extensions = get_extensions()
+ # get_extensions() discovers and loads all extensions,
+ # including installing in-memory importers and registering entry points
+ get_extensions()
except Exception: # pylint: disable=broad-except # noqa: S110
# If the db hasn't been initialized yet, an exception will be raised.
# It's fine to ignore this, as in this case there are no extensions
# present yet.
return
- for extension in extensions.values():
- if backend_files := extension.backend:
- install_in_memory_importer(
- backend_files,
- source_base_path=extension.source_base_path,
- )
-
- backend = extension.manifest.backend
-
- if backend and (entrypoints := backend.entryPoints):
- for entrypoint in entrypoints:
- try:
- eager_import(entrypoint)
- except Exception as ex: # pylint: disable=broad-except # noqa: S110
- # Surface exceptions during initialization of extensions
- print(ex)
-
def init_app_in_ctx(self) -> None:
"""
Runs init logic in the context of the app