You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/04/25 18:51:08 UTC
(airflow) branch v3-2-test updated: [v3-2-test] Install shared distros in CI image and tighten shared imports (#65783) (#65832)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new 2b3c66799e0 [v3-2-test] Install shared distros in CI image and tighten shared imports (#65783) (#65832)
2b3c66799e0 is described below
commit 2b3c66799e04ba2211f6dfa80cfdf2824603a00f
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat Apr 25 20:50:59 2026 +0200
[v3-2-test] Install shared distros in CI image and tighten shared imports (#65783) (#65832)
* Use airflow._shared for Stats import in airflow-core dag definition
The new ``check_shared_distributions_structure`` prek hook (#65621)
requires that ``airflow-core`` import shared code via the
``airflow._shared.*`` namespace rather than ``airflow.sdk._shared.*``
(the task-sdk distribution's namespace). One stray ``Stats`` import in
``serialization/definitions/dag.py`` was using the task-sdk path; all
other airflow-core modules already import the same ``Stats`` from
``airflow._shared.observability.metrics.stats``. Switch this one to
match so it shares the singleton already initialized by the executors
and job runners.
* Install shared distributions in CI image so airflow_shared.* resolves
The shared distributions under ``shared/<name>/`` are marked
``Private :: Do Not Upload`` and are not pulled in transitively when
Airflow is installed from a published wheel — those wheels only embed
the modules under ``airflow/_shared/<name>`` and
``airflow/sdk/_shared/<name>``. As a result, ``import airflow_shared.X``
fails in the Lowest Deps and Compat test images even though the source
tree is mounted.
Install the shared distributions from local source inside
``install_development_dependencies.py`` (alongside ``devel-common``)
so the canonical ``airflow_shared.*`` namespace is always available
to test utilities. With that in place, switch ``devel-common`` back
to importing from ``airflow_shared.*`` rather than reaching into
``airflow._shared`` / ``airflow.sdk._shared``:
* ``sphinx_exts/metrics_tables_from_registry.py``
* ``tests_common/pytest_plugin.py`` (``_get_grouped_entry_points``)
* ``tests_common/test_utils/api_fastapi.py`` (``DEFAULT_SENSITIVE_FIELDS``)
* ``tests_common/test_utils/logs.py`` (``DEFAULT_SENSITIVE_FIELDS``)
The two ``DEFAULT_SENSITIVE_FIELDS`` try/except fallbacks collapse
into a single direct import.
* Re-install shared distributions after lowest-direct uv sync
The lowest-direct `uv sync` in `check_force_lowest_dependencies` removes
the `apache-airflow-shared-*` workspace members from the environment
because they aren't transitively required by `airflow-core` or any
provider. With devel-common test utils now importing from
`airflow_shared.*`, test collection in the LowestDeps jobs fails with
`ModuleNotFoundError: No module named 'airflow_shared'`.
Re-install all `shared/<name>/` distributions with `--no-deps` after the
sync so the namespace resolves without disturbing the lowest-direct
resolution that was just applied.
(cherry picked from commit 8eb433b842f2ef97001e636cbd3f2233a8aab36c)
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
---
Dockerfile.ci | 15 +++++++++++++++
airflow-core/src/airflow/serialization/definitions/dag.py | 2 +-
.../src/sphinx_exts/metrics_tables_from_registry.py | 2 +-
devel-common/src/tests_common/pytest_plugin.py | 2 +-
devel-common/src/tests_common/test_utils/api_fastapi.py | 6 +-----
devel-common/src/tests_common/test_utils/logs.py | 6 +-----
scripts/docker/entrypoint_ci.sh | 15 +++++++++++++++
scripts/in_container/install_development_dependencies.py | 10 ++++++++++
8 files changed, 45 insertions(+), 13 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index aa92494945b..7f46cbfff7f 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1483,6 +1483,20 @@ function check_run_tests() {
fi
}
+function reinstall_shared_distributions() {
+ # The shared distributions under shared/<name>/ are workspace members that are not
+ # transitively required by airflow-core or any provider, so the lowest-direct
+ # `uv sync` above wipes them out from the environment. Re-install them with
+ # --no-deps so that the `airflow_shared.*` namespace used by devel-common test
+ # utils resolves at test collection time without disturbing the lowest-direct
+ # resolution that was just applied.
+ echo
+ echo "${COLOR_BLUE}Re-installing shared distributions (airflow_shared.*) after uv sync${COLOR_RESET}"
+ echo
+ # shellcheck disable=SC2046
+ uv pip install --no-deps $(ls -d /opt/airflow/shared/*/)
+}
+
function check_force_lowest_dependencies() {
if [[ ${FORCE_LOWEST_DEPENDENCIES=} != "true" ]]; then
return
@@ -1535,6 +1549,7 @@ function check_force_lowest_dependencies() {
uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
--no-python-downloads --no-managed-python
fi
+ reinstall_shared_distributions
}
function check_airflow_python_client_installation() {
diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py
index a8fa92d12ce..8844a18412c 100644
--- a/airflow-core/src/airflow/serialization/definitions/dag.py
+++ b/airflow-core/src/airflow/serialization/definitions/dag.py
@@ -30,6 +30,7 @@ import attrs
import structlog
from sqlalchemy import func, or_, select, tuple_
+from airflow._shared.observability.metrics.stats import Stats
from airflow._shared.timezones.timezone import coerce_datetime
from airflow.configuration import conf as airflow_conf
from airflow.exceptions import AirflowException, TaskNotFound
@@ -40,7 +41,6 @@ from airflow.models.deadline import Deadline
from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel
from airflow.models.taskinstancekey import TaskInstanceKey
from airflow.models.tasklog import LogTemplate
-from airflow.sdk._shared.observability.metrics.stats import Stats
from airflow.serialization.decoders import decode_deadline_alert
from airflow.serialization.definitions.deadline import DeadlineAlertFields, SerializedReferenceModels
from airflow.serialization.definitions.param import SerializedParamsDict
diff --git a/devel-common/src/sphinx_exts/metrics_tables_from_registry.py b/devel-common/src/sphinx_exts/metrics_tables_from_registry.py
index ed93bf8c845..61aa463e863 100644
--- a/devel-common/src/sphinx_exts/metrics_tables_from_registry.py
+++ b/devel-common/src/sphinx_exts/metrics_tables_from_registry.py
@@ -16,7 +16,7 @@
# under the License.
from __future__ import annotations
-from airflow._shared.observability.metrics.metrics_registry import generate_metrics_rst_from_registry
+from airflow_shared.observability.metrics.metrics_registry import generate_metrics_rst_from_registry
def generate_metrics(app):
diff --git a/devel-common/src/tests_common/pytest_plugin.py b/devel-common/src/tests_common/pytest_plugin.py
index a4b590c5452..17f6ecc4d3c 100644
--- a/devel-common/src/tests_common/pytest_plugin.py
+++ b/devel-common/src/tests_common/pytest_plugin.py
@@ -1848,7 +1848,7 @@ def clear_lru_cache():
return
try:
- from airflow._shared.module_loading import _get_grouped_entry_points
+ from airflow_shared.module_loading import _get_grouped_entry_points
except ImportError:
# compat for airflow < 3.2
from airflow.utils.entry_points import _get_grouped_entry_points
diff --git a/devel-common/src/tests_common/test_utils/api_fastapi.py b/devel-common/src/tests_common/test_utils/api_fastapi.py
index a59f05bf108..0e19c148614 100644
--- a/devel-common/src/tests_common/test_utils/api_fastapi.py
+++ b/devel-common/src/tests_common/test_utils/api_fastapi.py
@@ -22,11 +22,7 @@ from uuid import UUID
from airflow.models import DagRun, Log
from airflow.models.dagrun import DagRunNote
from airflow.models.taskinstance import TaskInstanceNote
-
-try:
- from airflow.sdk._shared.secrets_masker import DEFAULT_SENSITIVE_FIELDS
-except ImportError:
- from airflow.sdk.execution_time.secrets_masker import DEFAULT_SENSITIVE_FIELDS # type:ignore[no-redef]
+from airflow_shared.secrets_masker import DEFAULT_SENSITIVE_FIELDS
sensitive_fields = DEFAULT_SENSITIVE_FIELDS
diff --git a/devel-common/src/tests_common/test_utils/logs.py b/devel-common/src/tests_common/test_utils/logs.py
index 28d7296f380..3f894b88452 100644
--- a/devel-common/src/tests_common/test_utils/logs.py
+++ b/devel-common/src/tests_common/test_utils/logs.py
@@ -28,11 +28,7 @@ from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, NoReturn
from airflow.models import Log
-
-try:
- from airflow.sdk._shared.secrets_masker import DEFAULT_SENSITIVE_FIELDS
-except ImportError:
- from airflow.sdk.execution_time.secrets_masker import DEFAULT_SENSITIVE_FIELDS # type:ignore[no-redef]
+from airflow_shared.secrets_masker import DEFAULT_SENSITIVE_FIELDS
if TYPE_CHECKING:
from structlog.typing import EventDict, WrappedLogger
diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh
index fe0cbf22620..03f761f8584 100755
--- a/scripts/docker/entrypoint_ci.sh
+++ b/scripts/docker/entrypoint_ci.sh
@@ -392,6 +392,20 @@ function check_run_tests() {
fi
}
+function reinstall_shared_distributions() {
+ # The shared distributions under shared/<name>/ are workspace members that are not
+ # transitively required by airflow-core or any provider, so the lowest-direct
+ # `uv sync` above wipes them out from the environment. Re-install them with
+ # --no-deps so that the `airflow_shared.*` namespace used by devel-common test
+ # utils resolves at test collection time without disturbing the lowest-direct
+ # resolution that was just applied.
+ echo
+ echo "${COLOR_BLUE}Re-installing shared distributions (airflow_shared.*) after uv sync${COLOR_RESET}"
+ echo
+ # shellcheck disable=SC2046
+ uv pip install --no-deps $(ls -d /opt/airflow/shared/*/)
+}
+
function check_force_lowest_dependencies() {
if [[ ${FORCE_LOWEST_DEPENDENCIES=} != "true" ]]; then
return
@@ -444,6 +458,7 @@ function check_force_lowest_dependencies() {
uv sync --resolution lowest-direct --no-binary-package lxml --no-binary-package xmlsec --all-extras \
--no-python-downloads --no-managed-python
fi
+ reinstall_shared_distributions
}
function check_airflow_python_client_installation() {
diff --git a/scripts/in_container/install_development_dependencies.py b/scripts/in_container/install_development_dependencies.py
index ee5831b0199..cabd03ed2bc 100755
--- a/scripts/in_container/install_development_dependencies.py
+++ b/scripts/in_container/install_development_dependencies.py
@@ -78,6 +78,16 @@ def install_development_dependencies(constraint: str, github_actions: bool):
)
for provider_id in providers_dependencies:
development_dependencies.extend(providers_dependencies[provider_id]["devel-deps"])
+ # The shared distributions under shared/<name>/ provide the canonical
+ # ``airflow_shared.<name>`` namespace used by devel-common. They are marked
+ # ``Private :: Do Not Upload`` so they don't get pulled from PyPI by Airflow's
+ # transitive deps; install them explicitly from the local source tree so that
+ # ``import airflow_shared.X`` works in Lowest Deps and Compat test images that
+ # otherwise only ship the embedded ``airflow._shared.X`` / ``airflow.sdk._shared.X``.
+ shared_distributions = sorted(
+ path.parent.as_posix() for path in (AIRFLOW_ROOT_PATH / "shared").glob("*/pyproject.toml")
+ )
+ development_dependencies.extend(shared_distributions)
command = ["uv", "pip", "install", *development_dependencies, "--constraints", constraint]
result = run_command(command, check=False, github_actions=github_actions)
if result.returncode != 0: