You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2026/04/26 19:47:47 UTC
(airflow) branch v3-2-test updated: [v3-2-test] Enable SQLAlchemy connection pool settings for file-based SQLite (#64888) (#65411)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-2-test by this push:
new 1d99783b173 [v3-2-test] Enable SQLAlchemy connection pool settings for file-based SQLite (#64888) (#65411)
1d99783b173 is described below
commit 1d99783b173e5c3b217e468bfdb49f63da717464
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sun Apr 26 21:45:06 2026 +0200
[v3-2-test] Enable SQLAlchemy connection pool settings for file-based SQLite (#64888) (#65411)
* [v3-2-test] Bump actions/github-script in the github-actions-updates group (#65150) (#65160)
Bumps the github-actions-updates group with 1 update: [actions/github-script](https://github.com/actions/github-script).
Updates `actions/github-script` from 8.0.0 to 9.0.0
- [Release notes](https://github.com/actions/github-script/releases)
- [Commits](https://github.com/actions/github-script/compare/ed597411d8f924073f98dfc5c65a23a2325f34cd...3a2844b7e9c422d3c10d287c895573f7108da1b3)
(cherry picked from commit e5a047ca8d23614cefeaad89e08eb1e3b0482e60)
---
updated-dependencies:
- dependency-name: actions/github-script
dependency-version: 9.0.0
dependency-type: direct:production
update-type: version-update:semver-major
dependency-group: github-actions-updates
...
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
* [v3-2-test] Added breeze generate issue content for airflow-ctl (#65042) (#65241)
* Add breeze generate issue content for airflow-ctl
* add new command to doc
(cherry picked from commit b24538b0bc3a53d8c666b62e52e023a17d102467)
Co-authored-by: Justin Pakzad <11...@users.noreply.github.com>
* [v3-2-test] Run release calendar verification on its own schedule (#65118) (#65242)
* Move release calendar verification to its own scheduled workflow
Run dev/verify_release_calendar.py from a dedicated daily scheduled
workflow instead of as a canary job in the main CI pipeline, and
notify the #release-management Slack channel when the check fails so
the issue is surfaced to release managers directly.
* Include wiki and calendar links in release calendar Slack alert
(cherry picked from commit 048e9a191b4a6625bb4bbad23fad537b256f4062)
* [v3-2-test] Enable SQLAlchemy connection pool settings for file-based SQLite (#64888)
* Enable SQLAlchemy connection pool settings for file-based SQLite
* Fix review comments
* Fix import make_url comment
* Address Wei's final nit
(cherry picked from commit d6ba3c4730bbe6406844a58edf335991ca7149dd)
Co-authored-by: Jason(Zhe-You) Liu <68...@users.noreply.github.com>
---------
Signed-off-by: dependabot[bot] <su...@github.com>
Co-authored-by: github-actions[bot] <41...@users.noreply.github.com>
Co-authored-by: dependabot[bot] <49...@users.noreply.github.com>
Co-authored-by: Jarek Potiuk <ja...@potiuk.com>
Co-authored-by: Justin Pakzad <11...@users.noreply.github.com>
Co-authored-by: Jason(Zhe-You) Liu <68...@users.noreply.github.com>
---
airflow-core/src/airflow/settings.py | 25 ++++++++--
.../tests/unit/core/test_sqlalchemy_config.py | 53 +++++++++++++++++++++-
2 files changed, 74 insertions(+), 4 deletions(-)
diff --git a/airflow-core/src/airflow/settings.py b/airflow-core/src/airflow/settings.py
index bf47f86a2c2..e42aa3ceff6 100644
--- a/airflow-core/src/airflow/settings.py
+++ b/airflow-core/src/airflow/settings.py
@@ -31,6 +31,7 @@ from typing import TYPE_CHECKING, Any, Literal
import pluggy
from packaging.version import Version
from sqlalchemy import create_engine
+from sqlalchemy.engine import make_url
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession as SAAsyncSession,
@@ -481,6 +482,20 @@ def configure_orm(disable_connection_pool=False, pool_class=None):
register_at_fork(after_in_child=clean_in_fork)
+def _is_sqlite_in_memory(url: str) -> bool:
+ """
+ Check if a SQLAlchemy connection URL points to an in-memory SQLite database.
+
+ Handles driver prefixes (e.g. ``sqlite+pysqlite://``), query parameters, and
+ various in-memory forms like ``:memory:`` and ``file::memory:``.
+ """
+ parsed = make_url(url)
+ if parsed.get_backend_name() != "sqlite":
+ return False
+ db = parsed.database
+ return not db or db == ":memory:" or ":memory:" in db
+
+
def prepare_engine_args(disable_connection_pool=False, pool_class=None):
"""Prepare SQLAlchemy engine args."""
DEFAULT_ENGINE_ARGS: dict[str, dict[str, Any]] = {
@@ -510,8 +525,12 @@ def prepare_engine_args(disable_connection_pool=False, pool_class=None):
elif disable_connection_pool or not conf.getboolean("database", "SQL_ALCHEMY_POOL_ENABLED"):
engine_args["poolclass"] = NullPool
log.debug("settings.prepare_engine_args(): Using NullPool")
- elif not SQL_ALCHEMY_CONN.startswith("sqlite"):
- # Pool size engine args not supported by sqlite.
+ elif _is_sqlite_in_memory(SQL_ALCHEMY_CONN):
+ # In-memory SQLite uses SingletonThreadPool which doesn't support pool_size/max_overflow.
+ log.debug("settings.prepare_engine_args(): Skipping pool settings for in-memory SQLite")
+ else:
+ # Pool settings for all file-based databases including SQLite.
+ # SQLAlchemy 2.0+ uses QueuePool by default for file-based SQLite.
# If no config value is defined for the pool size, select a reasonable value.
# 0 means no limit, which could lead to exceeding the Database connection limit.
pool_size = conf.getint("database", "SQL_ALCHEMY_POOL_SIZE", fallback=5)
@@ -538,7 +557,7 @@ def prepare_engine_args(disable_connection_pool=False, pool_class=None):
# Typically, this is a simple statement like "SELECT 1", but may also make use
# of some DBAPI-specific method to test the connection for liveness.
# More information here:
- # https://docs.sqlalchemy.org/en/14/core/pooling.html#disconnect-handling-pessimistic
+ # https://docs.sqlalchemy.org/en/20/core/pooling.html#disconnect-handling-pessimistic
pool_pre_ping = conf.getboolean("database", "SQL_ALCHEMY_POOL_PRE_PING", fallback=True)
log.debug(
diff --git a/airflow-core/tests/unit/core/test_sqlalchemy_config.py b/airflow-core/tests/unit/core/test_sqlalchemy_config.py
index 9c465f05544..db211bab2c5 100644
--- a/airflow-core/tests/unit/core/test_sqlalchemy_config.py
+++ b/airflow-core/tests/unit/core/test_sqlalchemy_config.py
@@ -17,7 +17,7 @@
# under the License.
from __future__ import annotations
-from unittest.mock import patch
+from unittest.mock import call, patch
import pytest
from sqlalchemy.pool import NullPool
@@ -70,6 +70,57 @@ class TestSqlAlchemySettings:
**expected_kwargs,
)
+ @patch("airflow.settings.setup_event_handlers")
+ @patch("airflow.settings.scoped_session")
+ @patch("airflow.settings.sessionmaker")
+ @patch("airflow.settings.create_engine")
+ def test_configure_orm_sqlite_file_based_gets_pool_settings(
+ self,
+ mock_create_engine,
+ mock_sessionmaker,
+ mock_scoped_session,
+ mock_setup_event_handlers,
+ monkeypatch,
+ ):
+ """SQLAlchemy 2.0+ uses QueuePool for file-based SQLite, so pool settings should be applied."""
+ monkeypatch.setattr(settings, "SQL_ALCHEMY_CONN", "sqlite:////tmp/airflow.db")
+ settings.configure_orm()
+ expected_kwargs = dict(
+ connect_args={"check_same_thread": False},
+ max_overflow=10,
+ pool_pre_ping=True,
+ pool_recycle=1800,
+ pool_size=5,
+ future=True,
+ )
+ assert mock_create_engine.mock_calls == [
+ call(
+ "sqlite:////tmp/airflow.db",
+ **expected_kwargs,
+ ),
+ call().url.password.__bool__(),
+ call().url.password.__iter__(),
+ ]
+
+ @pytest.mark.parametrize(
+ "conn_str",
+ [
+ "sqlite://",
+ "sqlite:///:memory:",
+ "sqlite+pysqlite:///:memory:",
+ "sqlite:///:memory:?cache=shared",
+ "sqlite:///file::memory:?cache=shared",
+ ],
+ )
+ def test_prepare_engine_args_sqlite_in_memory_skips_pool_settings(self, conn_str, monkeypatch):
+ """In-memory SQLite uses SingletonThreadPool which doesn't support pool_size/max_overflow."""
+ monkeypatch.setattr(settings, "SQL_ALCHEMY_CONN", conn_str)
+ engine_args = settings.prepare_engine_args()
+ assert "pool_size" not in engine_args
+ assert "max_overflow" not in engine_args
+ assert "pool_recycle" not in engine_args
+ assert "pool_pre_ping" not in engine_args
+
@patch("airflow.settings.setup_event_handlers")
@patch("airflow.settings.scoped_session")
@patch("airflow.settings.sessionmaker")