You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/04/27 12:08:20 UTC
(airflow) branch main updated: Dispose flask_sqlalchemy + airflow ORM engines in fab test fixtures (#65941)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new eee05dd63aa Dispose flask_sqlalchemy + airflow ORM engines in fab test fixtures (#65941)
eee05dd63aa is described below
commit eee05dd63aa94523aa4409191118d545cdae89ef
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Apr 27 14:08:12 2026 +0200
Dispose flask_sqlalchemy + airflow ORM engines in fab test fixtures (#65941)
Flake on a Pendulum2 providers run ended with 8 fab test errors —
``psycopg2.OperationalError: ... FATAL: sorry, too many clients
already`` after 10031 tests had passed. Postgres' default
``max_connections=100`` was being exhausted before fab tests at the
end of the bucket could acquire a connection.
The accumulation came from fab test fixtures, not from per-test
session leaks:
- ``providers/fab/tests/unit/fab/www/views/conftest.py`` called
``settings.configure_orm()`` in an autouse module-scope fixture.
``configure_orm()`` reassigns the global engine **without** disposing
the previous one, orphaning a pool of up to 15 pooled connections
per call. ``settings.reconfigure_orm()`` is the helper that calls
``dispose_orm()`` first.
- Several fab ``app`` fixtures call ``application.create_app()``,
which does ``db = SQLAlchemy(flask_app)`` — flask_sqlalchemy
eagerly creates a per-app engine in ``init_app()``. None of the
fixtures disposed those engines on teardown, so each Flask app's
pool survived the fixture and accumulated for the worker's
lifetime. The fab bucket is one worker that runs many such
fixtures back-to-back.
Fixes:
- ``views/conftest.py``: switch to ``reconfigure_orm()`` (no leak on
re-entry); dispose the flask_sqlalchemy engines on the
module-scope ``app`` fixture teardown.
- ``www/test_auth.py``, ``views/test_views_custom_user_views.py``:
add try/finally around the function-scope ``app`` fixture and
dispose the flask_sqlalchemy engines per test.
- ``auth_manager/test_security.py``: dispose engines on the
module-scope ``app`` fixture teardown.
- ``auth_manager/conftest.py``: convert the session-scope
``minimal_app_for_auth_api`` to ``yield`` and dispose engines on
teardown.
A follow-up branch that bumped ``max_connections`` to 300 has been
dropped — with the leak fixed it's no longer load-bearing.
Failing run: https://github.com/apache/airflow/actions/runs/24976790617/job/73131061767
---
providers/fab/tests/unit/fab/auth_manager/conftest.py | 10 +++++++++-
providers/fab/tests/unit/fab/auth_manager/test_security.py | 11 +++++++++--
providers/fab/tests/unit/fab/www/test_auth.py | 11 ++++++++++-
providers/fab/tests/unit/fab/www/views/conftest.py | 11 ++++++++++-
.../tests/unit/fab/www/views/test_views_custom_user_views.py | 10 +++++++++-
5 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/providers/fab/tests/unit/fab/auth_manager/conftest.py b/providers/fab/tests/unit/fab/auth_manager/conftest.py
index 770999969c0..f8a13ec0588 100644
--- a/providers/fab/tests/unit/fab/auth_manager/conftest.py
+++ b/providers/fab/tests/unit/fab/auth_manager/conftest.py
@@ -51,7 +51,15 @@ def minimal_app_for_auth_api():
_app.config["AUTH_ROLE_PUBLIC"] = None
return _app
- return factory()
+ flask_app = factory()
+ try:
+ yield flask_app
+ finally:
+ # Dispose the flask_sqlalchemy per-app engine so its pooled connections
+ # don't survive the session in long CI runs.
+ with flask_app.app_context():
+ for fab_engine in flask_app.extensions["sqlalchemy"].engines.values():
+ fab_engine.dispose()
@pytest.fixture
diff --git a/providers/fab/tests/unit/fab/auth_manager/test_security.py b/providers/fab/tests/unit/fab/auth_manager/test_security.py
index 6252b08284a..389c054e11b 100644
--- a/providers/fab/tests/unit/fab/auth_manager/test_security.py
+++ b/providers/fab/tests/unit/fab/auth_manager/test_security.py
@@ -235,8 +235,15 @@ def app():
):
_app = application.create_app(enable_plugins=False)
_app.config["WTF_CSRF_ENABLED"] = False
- with _app.app_context():
- yield _app
+ try:
+ with _app.app_context():
+ yield _app
+ finally:
+ # Dispose the flask_sqlalchemy per-app engine so its pooled
+ # connections don't survive the module in long CI runs.
+ with _app.app_context():
+ for fab_engine in _app.extensions["sqlalchemy"].engines.values():
+ fab_engine.dispose()
@pytest.fixture(scope="module")
diff --git a/providers/fab/tests/unit/fab/www/test_auth.py b/providers/fab/tests/unit/fab/www/test_auth.py
index 7400ff2bf14..51e002e0ca9 100644
--- a/providers/fab/tests/unit/fab/www/test_auth.py
+++ b/providers/fab/tests/unit/fab/www/test_auth.py
@@ -41,7 +41,16 @@ def app():
): "airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager",
}
):
- yield application.create_app(enable_plugins=False)
+ flask_app = application.create_app(enable_plugins=False)
+ try:
+ yield flask_app
+ finally:
+ # flask_sqlalchemy creates a per-app engine in init_app(); a
+ # function-scoped app fixture without disposal would leak that
+ # engine's connection pool every test.
+ with flask_app.app_context():
+ for fab_engine in flask_app.extensions["sqlalchemy"].engines.values():
+ fab_engine.dispose()
@pytest.mark.parametrize(
diff --git a/providers/fab/tests/unit/fab/www/views/conftest.py b/providers/fab/tests/unit/fab/www/views/conftest.py
index 94f7bbc7761..3c6e047deb9 100644
--- a/providers/fab/tests/unit/fab/www/views/conftest.py
+++ b/providers/fab/tests/unit/fab/www/views/conftest.py
@@ -34,7 +34,10 @@ from unit.fab.utils import client_with_login
@pytest.fixture(autouse=True, scope="module")
def session():
- settings.configure_orm()
+ # reconfigure_orm() disposes any pre-existing engine before creating a new
+ # one. Plain configure_orm() leaks the previous engine's connection pool,
+ # which exhausts Postgres max_connections in long parallel CI runs.
+ settings.reconfigure_orm()
return settings.Session
@@ -102,6 +105,12 @@ def app(examples_dag_bag):
for user_dict in test_users:
delete_user(app, user_dict["username"])
+ # flask_sqlalchemy creates a per-app engine in init_app(); without disposal
+ # its pooled connections survive the fixture and accumulate across modules
+ # in long CI runs.
+ with app.app_context():
+ for fab_engine in app.extensions["sqlalchemy"].engines.values():
+ fab_engine.dispose()
@pytest.fixture
diff --git a/providers/fab/tests/unit/fab/www/views/test_views_custom_user_views.py b/providers/fab/tests/unit/fab/www/views/test_views_custom_user_views.py
index 524633ea906..7e61cdff785 100644
--- a/providers/fab/tests/unit/fab/www/views/test_views_custom_user_views.py
+++ b/providers/fab/tests/unit/fab/www/views/test_views_custom_user_views.py
@@ -79,7 +79,15 @@ def app():
):
app = application.create_app(enable_plugins=False)
app.config["WTF_CSRF_ENABLED"] = False
- yield app
+ try:
+ yield app
+ finally:
+ # flask_sqlalchemy creates a per-app engine in init_app(); a
+ # function-scoped app fixture without disposal would leak that
+ # engine's connection pool every test.
+ with app.app_context():
+ for fab_engine in app.extensions["sqlalchemy"].engines.values():
+ fab_engine.dispose()
@pytest.fixture