You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by vi...@apache.org on 2025/04/09 21:45:58 UTC
(airflow) branch main updated: Move FAB related configs to FAB provider (#49017)
This is an automated email from the ASF dual-hosted git repository.
vincbeck pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f40fb4af4e6 Move FAB related configs to FAB provider (#49017)
f40fb4af4e6 is described below
commit f40fb4af4e64601af836e2e47b4a87fc68c95fe4
Author: Vincent <97...@users.noreply.github.com>
AuthorDate: Wed Apr 9 17:45:50 2025 -0400
Move FAB related configs to FAB provider (#49017)
---
airflow-core/newsfragments/49017.significant.rst | 25 +++++++++++++++
airflow-core/src/airflow/__main__.py | 3 --
airflow-core/src/airflow/api_fastapi/app.py | 2 ++
.../src/airflow/cli/commands/config_command.py | 12 +++++++
.../src/airflow/config_templates/config.yml | 37 +---------------------
airflow-core/src/airflow/configuration.py | 16 ----------
airflow-core/src/airflow/models/taskinstance.py | 4 +--
airflow-core/src/airflow/settings.py | 2 +-
airflow-core/src/airflow/utils/db_cleanup.py | 7 +++-
providers/fab/provider.yaml | 28 +++++++++++++++-
.../fab/auth_manager/cli_commands/utils.py | 2 +-
.../fab/auth_manager/security_manager/override.py | 2 +-
.../src/airflow/providers/fab/get_provider_info.py | 16 +++++++++-
providers/fab/src/airflow/providers/fab/www/app.py | 2 +-
.../providers/fab/www/extensions/init_session.py | 4 +--
.../fab/auth_manager/cli_commands/test_utils.py | 2 +-
16 files changed, 97 insertions(+), 67 deletions(-)
diff --git a/airflow-core/newsfragments/49017.significant.rst b/airflow-core/newsfragments/49017.significant.rst
new file mode 100644
index 00000000000..56a9b08831c
--- /dev/null
+++ b/airflow-core/newsfragments/49017.significant.rst
@@ -0,0 +1,25 @@
+Renamed FAB related configuration.
+
+ * Rename configuration ``webserver.config_file`` as ``fab.config_file``
+ * Rename configuration ``webserver.session_backend`` as ``fab.session_backend``
+ * Rename configuration ``webserver.base_url`` as ``api.base_url``
+
+* Types of change
+
+ * [ ] Dag changes
+ * [x] Config changes
+ * [ ] API changes
+ * [ ] CLI changes
+ * [ ] Behaviour changes
+ * [ ] Plugin changes
+ * [ ] Dependency changes
+ * [ ] Code interface changes
+
+* Migration rules needed
+
+
+ * ``airflow config lint``
+
+ * [x] ``webserver.config_file`` → ``fab.config_file``
+ * [x] ``webserver.session_backend`` → ``fab.session_backend``
+ * [x] ``webserver.base_url`` → ``api.base_url``
diff --git a/airflow-core/src/airflow/__main__.py b/airflow-core/src/airflow/__main__.py
index bfebc63946e..c11d3b5afe9 100644
--- a/airflow-core/src/airflow/__main__.py
+++ b/airflow-core/src/airflow/__main__.py
@@ -35,7 +35,6 @@ import argcomplete
# any possible import cycles with settings downstream.
from airflow import configuration
from airflow.cli import cli_parser
-from airflow.configuration import write_webserver_configuration_if_needed
def main():
@@ -53,8 +52,6 @@ def main():
from airflow.configuration import write_default_airflow_configuration_if_needed
conf = write_default_airflow_configuration_if_needed()
- if args.subcommand in ["webserver", "internal-api", "worker"]:
- write_webserver_configuration_if_needed(conf)
args.func(args)
diff --git a/airflow-core/src/airflow/api_fastapi/app.py b/airflow-core/src/airflow/api_fastapi/app.py
index 28c66cabcfc..3f4956660dd 100644
--- a/airflow-core/src/airflow/api_fastapi/app.py
+++ b/airflow-core/src/airflow/api_fastapi/app.py
@@ -36,6 +36,7 @@ from airflow.api_fastapi.core_api.app import (
from airflow.api_fastapi.execution_api.app import create_task_execution_api_app
from airflow.configuration import conf
from airflow.exceptions import AirflowConfigException
+from airflow.utils.providers_configuration_loader import providers_configuration_loaded
if TYPE_CHECKING:
from airflow.api_fastapi.auth.managers.base_auth_manager import BaseAuthManager
@@ -67,6 +68,7 @@ async def lifespan(app: FastAPI):
yield
+@providers_configuration_loaded
def create_app(apps: str = "all") -> FastAPI:
apps_list = apps.split(",") if apps else ["all"]
diff --git a/airflow-core/src/airflow/cli/commands/config_command.py b/airflow-core/src/airflow/cli/commands/config_command.py
index 05d509daf04..1184a50a356 100644
--- a/airflow-core/src/airflow/cli/commands/config_command.py
+++ b/airflow-core/src/airflow/cli/commands/config_command.py
@@ -327,6 +327,14 @@ CONFIGS_CHANGES = [
config=ConfigParameter("webserver", option="auth_rate_limit"),
renamed_to=ConfigParameter("fab", "auth_rate_limit"),
),
+ ConfigChange(
+ config=ConfigParameter("webserver", "config_file"),
+ renamed_to=ConfigParameter("fab", "config_file"),
+ ),
+ ConfigChange(
+ config=ConfigParameter("webserver", "session_backend"),
+ renamed_to=ConfigParameter("fab", "session_backend"),
+ ),
ConfigChange(
config=ConfigParameter("webserver", "session_lifetime_days"),
renamed_to=ConfigParameter("fab", "session_lifetime_minutes"),
@@ -339,6 +347,10 @@ CONFIGS_CHANGES = [
config=ConfigParameter("webserver", "session_lifetime_minutes"),
renamed_to=ConfigParameter("fab", "session_lifetime_minutes"),
),
+ ConfigChange(
+ config=ConfigParameter("webserver", "base_url"),
+ renamed_to=ConfigParameter("api", "base_url"),
+ ),
ConfigChange(
config=ConfigParameter("webserver", "web_server_host"),
renamed_to=ConfigParameter("api", "host"),
diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml
index f0912cc86d6..813316bef09 100644
--- a/airflow-core/src/airflow/config_templates/config.yml
+++ b/airflow-core/src/airflow/config_templates/config.yml
@@ -1708,22 +1708,6 @@ webserver:
type: string
example: ~
default: "Access is Denied"
- config_file:
- description: |
- Path of webserver config file used for configuring the webserver parameters
- version_added: 2.7.0
- type: string
- example: ~
- default: "{AIRFLOW_HOME}/webserver_config.py"
- base_url:
- description: |
- The base url of your website: Airflow cannot guess what domain or CNAME you are using.
- This is used to create links in the Log Url column in the Browse - Task Instances menu,
- as well as in any automated emails sent by Airflow that contain links to your webserver.
- version_added: ~
- type: string
- example: ~
- default: "http://localhost:8080"
default_ui_timezone:
description: |
Default timezone to display all dates in the UI, can be UTC, system, or
@@ -1734,28 +1718,9 @@ webserver:
example: "America/New_York"
# Default is left as UTC for now so the date's don't "suddenly" change on upgrade
default: "UTC"
- session_backend:
- description: |
- The type of backend used to store web session data, can be ``database`` or ``securecookie``. For the
- ``database`` backend, sessions are store in the database and they can be
- managed there (for example when you reset password of the user, all sessions for that user are
- deleted). For the ``securecookie`` backend, sessions are stored in encrypted cookies on the client
- side. The ``securecookie`` mechanism is 'lighter' than database backend, but sessions are not deleted
- when you reset password of the user, which means that other than waiting for expiry time, the only
- way to invalidate all sessions for a user is to change secret_key and restart webserver (which
- also invalidates and logs out all other user's sessions).
-
- When you are using ``database`` backend, make sure to keep your database session table small
- by periodically running ``airflow db clean --table session`` command, especially if you have
- automated API calls that will create a new session for each call rather than reuse the sessions
- stored in browser cookies.
- version_added: 2.2.4
- type: string
- example: securecookie
- default: database
secret_key:
description: |
- Secret key used to run your flask app. It should be as random as possible. However, when running
+ Secret key used to run your api server. It should be as random as possible. However, when running
more than 1 instances of webserver, make sure all of them use the same ``secret_key`` otherwise
one of them will error with "CSRF session token is missing".
The webserver key is also used to authorize requests to Celery workers when logs are retrieved.
diff --git a/airflow-core/src/airflow/configuration.py b/airflow-core/src/airflow/configuration.py
index f3679d66cae..2357f443327 100644
--- a/airflow-core/src/airflow/configuration.py
+++ b/airflow-core/src/airflow/configuration.py
@@ -49,7 +49,6 @@ from airflow.exceptions import AirflowConfigException
from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH
from airflow.utils import yaml
from airflow.utils.module_loading import import_string
-from airflow.utils.providers_configuration_loader import providers_configuration_loaded
from airflow.utils.weight_rule import WeightRule
if TYPE_CHECKING:
@@ -2108,23 +2107,9 @@ def initialize_config() -> AirflowConfigParser:
# file on top of it.
if airflow_config_parser.getboolean("core", "unit_test_mode"):
airflow_config_parser.load_test_config()
- # Set the WEBSERVER_CONFIG variable
- global WEBSERVER_CONFIG
- WEBSERVER_CONFIG = airflow_config_parser.get("webserver", "config_file")
return airflow_config_parser
-@providers_configuration_loaded
-def write_webserver_configuration_if_needed(airflow_config_parser: AirflowConfigParser):
- webserver_config = airflow_config_parser.get("webserver", "config_file")
- if not os.path.isfile(webserver_config):
- import shutil
-
- pathlib.Path(webserver_config).parent.mkdir(parents=True, exist_ok=True)
- log.info("Creating new FAB webserver config file in: %s", webserver_config)
- shutil.copy(_default_config_file_path("default_webserver_config.py"), webserver_config)
-
-
def make_group_other_inaccessible(file_path: str):
try:
permissions = os.stat(file_path)
@@ -2264,7 +2249,6 @@ else:
SECRET_KEY = b64encode(os.urandom(16)).decode("utf-8")
FERNET_KEY = "" # Set only if needed when generating a new file
JWT_SECRET_KEY = ""
-WEBSERVER_CONFIG = "" # Set by initialize_config
conf: AirflowConfigParser = initialize_config()
secrets_backend_list = initialize_secrets_backends()
diff --git a/airflow-core/src/airflow/models/taskinstance.py b/airflow-core/src/airflow/models/taskinstance.py
index 7a5dadcc0ed..e35927df61e 100644
--- a/airflow-core/src/airflow/models/taskinstance.py
+++ b/airflow-core/src/airflow/models/taskinstance.py
@@ -1915,10 +1915,10 @@ class TaskInstance(Base, LoggingMixin):
def log_url(self) -> str:
"""Log URL for TaskInstance."""
run_id = quote(self.run_id)
- base_url = conf.get_mandatory_value("webserver", "BASE_URL")
+ base_url = conf.get_mandatory_value("api", "BASE_URL")
map_index = f"/mapped/{self.map_index}" if self.map_index >= 0 else ""
try_number = f"?try_number={self.try_number}" if self.try_number > 0 else ""
- _log_uri = f"{base_url}/dags/{self.dag_id}/runs/{run_id}/tasks/{self.task_id}{map_index}{try_number}"
+ _log_uri = f"{base_url}dags/{self.dag_id}/runs/{run_id}/tasks/{self.task_id}{map_index}{try_number}"
return _log_uri
diff --git a/airflow-core/src/airflow/settings.py b/airflow-core/src/airflow/settings.py
index d5ba6b2bb7c..49d74770269 100644
--- a/airflow-core/src/airflow/settings.py
+++ b/airflow-core/src/airflow/settings.py
@@ -35,7 +35,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
from airflow import __version__ as airflow_version, policies
-from airflow.configuration import AIRFLOW_HOME, WEBSERVER_CONFIG, conf # noqa: F401
+from airflow.configuration import AIRFLOW_HOME, conf
from airflow.exceptions import AirflowInternalRuntimeError
from airflow.logging_config import configure_logging
from airflow.utils.orm_event_handlers import setup_event_handlers
diff --git a/airflow-core/src/airflow/utils/db_cleanup.py b/airflow-core/src/airflow/utils/db_cleanup.py
index 708c4f0f0f4..48df00ef6ab 100644
--- a/airflow-core/src/airflow/utils/db_cleanup.py
+++ b/airflow-core/src/airflow/utils/db_cleanup.py
@@ -130,7 +130,12 @@ config_list: list[_TableConfig] = [
_TableConfig(table_name="deadline", recency_column_name="deadline"),
]
-if conf.get("webserver", "session_backend") == "database":
+# We need to have `fallback="database"` because this is executed at top level code and provider configuration
+# might not be loaded
+if (
+ "FabAuthManager" in conf.get("core", "auth_manager")
+ and conf.get("fab", "session_backend", fallback="database") == "database"
+):
config_list.append(_TableConfig(table_name="session", recency_column_name="expiry"))
config_dict: dict[str, _TableConfig] = {x.orm_model.name: x for x in sorted(config_list)}
diff --git a/providers/fab/provider.yaml b/providers/fab/provider.yaml
index fbea846d924..abdb05fc94b 100644
--- a/providers/fab/provider.yaml
+++ b/providers/fab/provider.yaml
@@ -79,10 +79,36 @@ config:
auth_backends:
description: |
Comma separated list of auth backends to authenticate users of the API.
- version_added: 2.3.0
+ version_added: 2.0.0
type: string
example: ~
default: "airflow.providers.fab.auth_manager.api.auth.backend.session"
+ config_file:
+ description: |
+ Path of webserver config file used for configuring the webserver parameters
+ version_added: 2.0.0
+ type: string
+ example: ~
+ default: "{AIRFLOW_HOME}/webserver_config.py"
+ session_backend:
+ description: |
+ The type of backend used to store web session data, can be ``database`` or ``securecookie``. For the
+ ``database`` backend, sessions are store in the database and they can be
+ managed there (for example when you reset password of the user, all sessions for that user are
+ deleted). For the ``securecookie`` backend, sessions are stored in encrypted cookies on the client
+ side. The ``securecookie`` mechanism is 'lighter' than database backend, but sessions are not
+ deleted when you reset password of the user, which means that other than waiting for expiry time,
+ the only way to invalidate all sessions for a user is to change secret_key and restart webserver
+ (which also invalidates and logs out all other user's sessions).
+
+ When you are using ``database`` backend, make sure to keep your database session table small
+ by periodically running ``airflow db clean --table session`` command, especially if you have
+ automated API calls that will create a new session for each call rather than reuse the sessions
+ stored in browser cookies.
+ version_added: 2.0.0
+ type: string
+ example: "securecookie"
+ default: "database"
session_lifetime_minutes:
description: |
The UI cookie lifetime in minutes. User will be logged out from UI after
diff --git a/providers/fab/src/airflow/providers/fab/auth_manager/cli_commands/utils.py b/providers/fab/src/airflow/providers/fab/auth_manager/cli_commands/utils.py
index 36ced77e80b..174b3867ad0 100644
--- a/providers/fab/src/airflow/providers/fab/auth_manager/cli_commands/utils.py
+++ b/providers/fab/src/airflow/providers/fab/auth_manager/cli_commands/utils.py
@@ -51,7 +51,7 @@ def _return_appbuilder(app: Flask) -> AirflowAppBuilder:
def get_application_builder() -> Generator[AirflowAppBuilder, None, None]:
static_folder = os.path.join(os.path.dirname(airflow.__file__), "www", "static")
flask_app = Flask(__name__, static_folder=static_folder)
- webserver_config = conf.get_mandatory_value("webserver", "config_file")
+ webserver_config = conf.get_mandatory_value("fab", "config_file")
with flask_app.app_context():
# Enable customizations in webserver_config.py to be applied via Flask.current_app.
flask_app.config.from_pyfile(webserver_config, silent=True)
diff --git a/providers/fab/src/airflow/providers/fab/auth_manager/security_manager/override.py b/providers/fab/src/airflow/providers/fab/auth_manager/security_manager/override.py
index 0e855e57925..1e05a2e1bfe 100644
--- a/providers/fab/src/airflow/providers/fab/auth_manager/security_manager/override.py
+++ b/providers/fab/src/airflow/providers/fab/auth_manager/security_manager/override.py
@@ -2182,7 +2182,7 @@ class FabAirflowSecurityManagerOverride(AirflowSecurityManagerV2):
We need to do this upon successful authentication when using the
database session backend.
"""
- if conf.get("webserver", "SESSION_BACKEND") == "database":
+ if conf.get("fab", "SESSION_BACKEND") == "database":
session.sid = str(uuid.uuid4())
def _get_microsoft_jwks(self) -> list[dict[str, Any]]:
diff --git a/providers/fab/src/airflow/providers/fab/get_provider_info.py b/providers/fab/src/airflow/providers/fab/get_provider_info.py
index df321f84a1e..924dfca466a 100644
--- a/providers/fab/src/airflow/providers/fab/get_provider_info.py
+++ b/providers/fab/src/airflow/providers/fab/get_provider_info.py
@@ -53,11 +53,25 @@ def get_provider_info():
},
"auth_backends": {
"description": "Comma separated list of auth backends to authenticate users of the API.\n",
- "version_added": "2.3.0",
+ "version_added": "2.0.0",
"type": "string",
"example": None,
"default": "airflow.providers.fab.auth_manager.api.auth.backend.session",
},
+ "config_file": {
+ "description": "Path of webserver config file used for configuring the webserver parameters\n",
+ "version_added": "2.0.0",
+ "type": "string",
+ "example": None,
+ "default": "{AIRFLOW_HOME}/webserver_config.py",
+ },
+ "session_backend": {
+ "description": "The type of backend used to store web session data, can be ``database`` or ``securecookie``. For the\n``database`` backend, sessions are store in the database and they can be\nmanaged there (for example when you reset password of the user, all sessions for that user are\ndeleted). For the ``securecookie`` backend, sessions are stored in encrypted cookies on the client\nside. The ``securecookie`` mechanism is 'lighter' than database backend, but ses [...]
+ "version_added": "2.0.0",
+ "type": "string",
+ "example": "securecookie",
+ "default": "database",
+ },
"session_lifetime_minutes": {
"description": "The UI cookie lifetime in minutes. User will be logged out from UI after\n``[fab] session_lifetime_minutes`` of non-activity\n",
"version_added": "2.0.0",
diff --git a/providers/fab/src/airflow/providers/fab/www/app.py b/providers/fab/src/airflow/providers/fab/www/app.py
index e0f7a80ba75..be29f28bd67 100644
--- a/providers/fab/src/airflow/providers/fab/www/app.py
+++ b/providers/fab/src/airflow/providers/fab/www/app.py
@@ -60,7 +60,7 @@ def create_app(enable_plugins: bool):
flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
flask_app.config["PERMANENT_SESSION_LIFETIME"] = timedelta(minutes=get_session_lifetime_config())
- webserver_config = conf.get_mandatory_value("webserver", "config_file")
+ webserver_config = conf.get_mandatory_value("fab", "config_file")
# Enable customizations in webserver_config.py to be applied via Flask.current_app.
with flask_app.app_context():
flask_app.config.from_pyfile(webserver_config, silent=True)
diff --git a/providers/fab/src/airflow/providers/fab/www/extensions/init_session.py b/providers/fab/src/airflow/providers/fab/www/extensions/init_session.py
index e235ba7bcb3..15579bf0180 100644
--- a/providers/fab/src/airflow/providers/fab/www/extensions/init_session.py
+++ b/providers/fab/src/airflow/providers/fab/www/extensions/init_session.py
@@ -29,7 +29,7 @@ from airflow.providers.fab.www.session import (
def init_airflow_session_interface(app):
"""Set airflow session interface."""
config = app.config.copy()
- selected_backend = conf.get("webserver", "SESSION_BACKEND")
+ selected_backend = conf.get("fab", "SESSION_BACKEND")
# A bit of a misnomer - normally cookies expire whenever the browser is closed
# or when they hit their expiry datetime, whichever comes first. "Permanent"
# cookies only expire when they hit their expiry datetime, and can outlive
@@ -59,6 +59,6 @@ def init_airflow_session_interface(app):
else:
raise AirflowConfigException(
"Unrecognized session backend specified in "
- f"web_server_session_backend: '{selected_backend}'. Please set "
+ f"[fab] session_backend: '{selected_backend}'. Please set "
"this to either 'database' or 'securecookie'."
)
diff --git a/providers/fab/tests/unit/fab/auth_manager/cli_commands/test_utils.py b/providers/fab/tests/unit/fab/auth_manager/cli_commands/test_utils.py
index 4b7a60961c8..4cb7485cf1c 100644
--- a/providers/fab/tests/unit/fab/auth_manager/cli_commands/test_utils.py
+++ b/providers/fab/tests/unit/fab/auth_manager/cli_commands/test_utils.py
@@ -39,7 +39,7 @@ pytestmark = pytest.mark.db_test
def flask_app():
"""Fixture to set up the Flask app with the necessary configuration."""
# Get the webserver config file path
- webserver_config = conf.get_mandatory_value("webserver", "config_file")
+ webserver_config = conf.get_mandatory_value("fab", "config_file")
with get_application_builder() as appbuilder:
flask_app = appbuilder.app