You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2025/07/01 15:57:10 UTC
(airflow) branch structlog-in-logging-mixin updated (f74cd395ef5 -> 516be1fae8e)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a change to branch structlog-in-logging-mixin
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard f74cd395ef5 WIP: switch all airflow logging to structlog
add d81798088a6 Updating issue content generation in GH workflows (#52271)
add b070c3a9974 Fix docstring typo in dag_processing/manager.py (#52266)
add fda3fdac390 Clean up remaining DB-dependent tests from OpenSearch provider (#52235)
add 85e559d747f Fix multi line release command in CI (#52281)
add 79d5740cc2e Enhanced the BTEQ operator to ensure platform independence. (#52252)
add 8846bef4f9c Unify selecting constraints option when installing airflow (#52274)
add 7f694ad5101 Enhance Variable set method to use upsert instead of delsert (#48547)
add a041a2a3671 Enable Serde for Pydantic BaseModel and Subclasses (#51059)
add f641ef38e4a Use base AWS classes in Glue Trigger / Sensor and implement custom waiter (#52243)
add 63b2207ec7e Handle exceptions when fetching status in GlueJobHook (#52262)
add 885e40c5c77 Ensure `HttpHook.run()` does not alter `extra_options` passed to it (#51893)
add 8409fb81612 Remove double call to plugin init (#52291)
add 593dc557c2c Remove unused import Sequence from the celery_executor.py (#52290)
add 003566820ce Deprecated import fix for TimeDeltaSensorAsync in example dags (#52285)
add eaa8ca07a42 Grid view optimization (#51805)
add 121a31f0647 Add React Apps to plugin (#52255)
add 9275f7b284b Skip test that needs the .git folder when it is missing (#52305)
add 1c0c01d99ba Python versions in shell params are strings (#52306)
add 4a9067621fc Bump pymssql version to 2.3.5 (#52307)
add 9c7c72c6dd4 Remove pre-commit check-daysago-import-from-utils (#52304)
add feab705264b Use proper show-only value in test_worker.py (#52300)
add c3f9da4846c Revert "Enable Serde for Pydantic BaseModel and Subclasses (#51059)" (#52312)
add c16b88a0009 Fix GlueJobOperator deferred waiting (#52314)
add 8da059aeb60 cleanup stale dependency of methodtools (#52310)
add a3a6fe95cef Enable DatabricksJobRunLink for Databricks plugin, skip provide_session usage in Airflow3 (#52228)
add dd2bcacdc9f Fix mypy errors in GCP `generative_model` (#52321)
add ac9e968fe85 fix: task-sdk AssetEventOperations.get to use alias_name when specified (#52303)
add 3c1568ae773 Fix: Unclosed aiohttp ClientSession and TCPConnector in DatabricksRunNowOperator (deferrable=True) (#52119)
add 266c1ce2e8a Use BaseSensorOperator from task sdk in providers (#52296)
add 40a3c7e21ca feat: Add new query related methods to SnowflakeSqlApiHook (#52157)
add 85a06653155 Attempt2: Fix mypy in gcp generative_model (#52331)
add 57307609c4f Replace occurences of 'get_password' with 'password' to ease migration (#52333)
add 78e4cb4c670 Replace `models.BaseOperator` to Task SDK one for Standard Provider (#52292)
add e7e89a07ff2 Drop support for Python 3.9 (#52072)
add 26dbca59471 Replace usage of 'set_extra' with 'extra' for athena sql hook (#52340)
add c1c8173af69 Replace `models.BaseOperator` to Task SDK one for Alibaba & Airbyte (#52335)
add 8c86c5e9e52 chore: use task_instance as source for all airflow identifiers used in listener (#52339)
add 490947d1b63 Bump google-cloud-bigquery>=3.24.0 (#52337)
add 830c24d82be Cleanup stale Python3.9 dependencies (#52344)
add 0ffa1628bf3 Make airflow-ctl test_login safe for parallel execution by using temp AIRFLOW_HOME (#52345)
add 05497f40309 Improve safety for external views (#52352)
add a5e026f94cb Set snowflake-snowpark-python for Python 3.12 (#52356)
add 14040dab2d2 Bump ibmcloudant>=0.10.0 (#52354)
add a068b8cb422 Fix UnboundLocalError for `edge_job_command_len` (#52328)
add f6b0e4bcad0 Chart: Fix JWT secret name (#52268)
add 4f34f3bd0d2 Fix indexerror in _find_caplog_in_def selective check function (#52369)
add f1d0f2d3433 Bump microsoft kiota packages to 1.9.4 and update tests (#52367)
add 6177bd637fa Check chart annotations with pre-commit (#52365)
add bb041b3e1ac Add new `breeze run` command for non-interactive command execution (#52370)
add 713ddaa6de8 Bump ``uv`` to ``0.7.16`` (#52372)
add 0a95c3327af Replace `models.BaseOperator` to Task SDK one for Google Provider (#52366)
add 3ef47449cf3 Add Python <=> Airflow compat filtering for breeze (#52386)
add 0c2e4163118 docstring update for gcp dataplex operator and hook (#52387)
add 759653961f4 Run release tests always - not only in canary runs (#52389)
add a46a837a480 Add plural per-language forms in check-translations script (#52391)
add d5794656da2 Update click requirement in /dev/breeze (#52361)
add 9140bd659a2 Limit click back to 8.2.0 because it has an ENVVAR bug for flags (#52404)
add ed81cd5700c Replace `models.BaseOperator` to Task SDK one for Asana & Arangodb (#52374)
add 49c28e99dd7 Replace `models.BaseOperator` to Task SDK one for Atlassian (#52376)
add 6c5d879e0a3 Replace `models.BaseOperator` to Task SDK one for Apache Pig (#52375)
add f9c5d085c44 Replace `models.BaseOperator` to Task SDK one for DBT & Databricks (#52377)
add 51b53cccb3b Reduce timeout for task-sdk/airflow-ctl tests job workflow (#52399)
add 8599a9eee93 Provider Migration: Update trino for Airflow 3.0 compatibility (#52383)
add 26d0adf432a Add missing Polish Translations including proper plural forms (#52395)
add ace332d99c6 enhance error message for `breeze --backend none` to suggest setting a valid backend (#52318)
add bbe711e7345 Adding some intelligence to classifying provider commits (#52407)
add b90e5151ca8 Provider Migration: Update github provider for Airflow 3.0 compatibility (#52415)
add 165d5d51178 Bring back providers compatibility checks (#52398)
add e95cd988f9c Change analytics-python to segment-analytics-python (#52401)
add 6d001bc49e6 Provider Migration: Update airbyte provider for Airflow 3.0 compatibility (#52418)
add fb94109212b Sanitize Username (#52419)
add 148a1c3222d Add a script to report outdated versions in constraints (#52406)
add 87fe7ec94ca Skip check-airflow-providers-bug-report-template in non main branch (#52426)
add f685ed2377b Clean some leftovers of Python 3.9 removal - Airflow core pieces (#52424)
add 7c2bd9ad77c Clean some leftovers of Python 3.9 removal - Github pieces (#52423)
add 8fdb7766709 Add inline dependencies for uv run and colors to dependencies script (#52428)
add 85640d38893 Make sure all test version imports come from test_common (#52425)
add aa2342d36aa Provider Migration: Update Oracle for Airflow 3.0 compatibility (#52382)
add 7937a9704a5 Provider Migration: Update Weaviate for Airflow 3.0 compatibility (#52381)
add e6ff18ca629 Add selected packages and explain why to the package scripts (#52433)
add 56c4701a154 Fix failing static check for Oracle provider (#52436)
add ebc06786c9a Replace `models.BaseOperator` to Task SDK one for SFTP (#52435)
add 933a9e2b189 Clean some leftovers of Python 3.9 removal - Task-SDK (#52434)
add 8c3a81b18bf Clean some leftovers of Python 3.9 removal - Airflow CTL pieces (#52430)
add 508fac143c6 Add keycloak to providers removed when running Airflow 2 (#52442)
add c371d9af862 i18n(Ko): Add missing translations in admin.json and common.json (#52417)
add 8ef792dafef Fix editing connection with sensitive extra field (#52403)
add 50e69abdaa6 Replace `models.BaseOperator` to Task SDK one for Apache TinkerPop (#52400)
add d47f87c32ed Force the definition of `execution_api_server_url` based on `api_url` (#52184)
add a05b4de58f1 Improve constraints updated version check script (#52446)
add 689068fb2e3 Update documentation for forcing core execution_api_server_url (#52447)
add d050c6b9c7c Add colors to go tests output in CI (#52454)
add e53e99a85f5 add: version_compat (#52448)
add 3347e035795 Improve terminal handling for breeze commands (#52452)
add ae2d81f8c89 Remove old, unused generate SVG airflowctl pre-commit and fix width (#52457)
add c956455d5aa i18n(Ko): Replace 연결 as 커넥션 (#52440)
add ba6a9710527 Speed-up constraints generation (#52449)
add bcfa26d4ce6 Wire-in dependency check script in CI "finalize" job (#52450)
add ca264ba696b Clean some leftovers of Python 3.9 removal - All the rest (#52432)
add 33afdf97f5b Fix generate-constraints run on different python than base (#52464)
add bdf16bc0be5 Add GITHUB_TOKEN when preparing image for dependency summary (#52472)
add 2d5ba639825 Clean some leftovers of Python 3.9 removal - Files in root (#52463)
add c3f8a81ac0e Rmeove --tty specification for running the dependency script (#52489)
add 0c947b5300f Filter only provided integration paths for breeze integration testing (#52462)
add acacfc41e71 Provider Migration: Update docker for Airflow 3.0 compatibility (#52465)
add 56840411f05 Provider Migration: Replace `models.BaseOperator` to Task SDK for apache/impala (#52455)
add b8af9b5cb9d Provider Migration: Replace `models.BaseOperator` to Task SDK for apache/hive (#52453)
add 5432c6e5c3c Fix StopIteration in snowflake sql tests (#52394)
add 40116e30d7e Cleanup unused args example_pyspark.py (#52492)
add e455329ca35 Make the dependency script executable (#52493)
add 733a562a925 Close German language gap June 28th (#52459)
add 67613f0c38b Replace models.BaseOperator to Task SDK one for Common Providers (#52443)
add d3a33da891f Generally do not force version_compat.py to have pytests (#52496)
add 554ae58a488 Provider Migration: Update Apache Druid for Airflow 3.0 compatibility (#52498)
add a9a7fcc5192 Replace models.BaseOperator to Task SDK for http (#52506)
add 15313b08faf Replace models.BaseOperator to Task SDK for apache/livy (#52499)
add 98fd2792c3e Replace models.BaseOperator to Task SDK for apache/hdfs (#52505)
add 0edc4f48684 Update BaseOperator imports for Airflow 3.0 compatibility (#52503)
add 49593e48c19 Update BaseOperator imports for Airflow 3.0 compatibility (#52504)
add af51d4fe622 Revert "Replace models.BaseOperator to Task SDK for http (#52506)" (#52515)
add e3428dd435e [OpenLineage] Added operator_provider_version to task event (#52468)
add d4540c83060 Add a bunch of no-redef ignores so Mypy is happy (#52507)
add 51ddece3d1a Update Jenkins for Airflow 3.0 `BaseOperator` compatibility (#52510)
add d361df391eb Provider Migration: Update mysql for Airflow 3.0 compatibility (#52500)
add 9a14e66a64c feat: Add explicit support for DatabricksHook to Ol helper (#52253)
add 7494d0f8ac9 Fix various incompatibilities with SQLAlchemy 2.0 (#52518)
add ab347eda82d One more redef needing ignore (#52525)
add 9553a36d1ee Provider Migration: Update Cohere for Airflow 3.0 compatibility (#52379)
add 43d3fb5ac09 feat: Add explicit support for SnowflakeSqlApiHook to Ol helper (#52161)
add 9bdc215e8f9 Provider Migration: Replace `BaseOperator` to Task SDK for `apache/http` (#52528)
add 33164071691 Provider Migration: Update yandex provider for Airflow 3.0 compatibility (#52422)
add 60f780e77f4 Replace models.BaseOperator to Task SDK one for Mongo (#52566)
add 4c7d43c4f30 fix: enable iframe script execution (#52257)
add 57dc55cfa88 Add the `upgrade_sqlalchemy` breeze flag (#52559)
add fa6949202c9 Fix airflow pin for fab provider (#52351)
add f95319ef679 feat: Add real-time clock updates to timezone selector (#52414)
add 97c0b40f094 Allow Providers Iframe script execution (#52569)
add 5228dfb6993 Provider Migration: Replace `BaseOperator` to Task SDK for `ssh` (#52558)
add 6514e3385d1 Provider Migration: Replace `BaseOperator` to Task SDK for `Papermill` (#52565)
add 9428cb7999a Provider Migration: Replace `BaseOperator` to Task SDK for `OpenAI` (#52561)
add 893f8a52b95 Provider Migration: Replace `BaseOperator` to Task SDK for `Pinecone` (#52563)
add eac9a920fb8 Marking test_process_dags_queries_count as flaky (#52535)
add 11c132e75a0 Fix ParseImportError query in get_import_errors endpoint (#52531)
add e1907694334 Migrate segment provider to af3 (#52579)
add 921221b7d72 Set prefix to generate correctly the FAB Auth Manager API ref (#52329)
add 40f6f7ed137 Move compat shim in Standard Provider to `version_compat.py` (#52567)
add 2f9743eebea Provider Migration: Replace `BaseOperator` to Task SDK for `singularity` (#52590)
add 87afb93dc09 Provider Migration: Replace `BaseOperator` to Task SDK for `samba` (#52588)
add f1c1fd43846 Provider Migration: Replace `BaseOperator` to Task SDK for `salesforce` (#52587)
add 2adaf8194a3 Revert "Run release tests always - not only in canary runs (#52389)" (#52594)
add 67df7f3ce17 Fix deferrable mode for SparkKubernetesOperator (#51956)
add 2c2e66f1082 Increase dependency epoch to trigger pip cache invalidation (#52599)
add 800ed1457d7 Add Google Cloud VertexAI and Translate datasets import data verification (#51364)
add 2bdb51550cb Refactor the google cloud DataprocCreateBatchOperator tests (#52573)
add 4b8cd257e9a Upgrade ruff to latest version (0.12.1) (#52562)
add acea31e7a3d Fix SBOM commands to work for Airfow 2 (#52591)
add 12cd06d4136 airflowctl: transition of bulk operations to return BulkResponse (#52458)
add 61aa4d2f791 Provider Migration: Update presto for Airflow 3.0 compatibility (#52608)
add 240508ba809 Provider Migration: Update opensearch for Airflow 3.0 compatibility (#52609)
add c7ce990f0f8 Provider Migration: Update neo4j for Airflow 3.0 compatibility (#52610)
add 7bafaa4afaa Provider Migration: Replace `BaseSensorOperator` to Task SDK for `datadog` (#52583)
add 9138fc19f4c Provider Migration: Replace `BaseOperator` to Task SDK for `dingding` (#52577)
add d1f44205c62 Fix symlink handling for static assets when installed in editable mode with uv (#52612)
add b290af4cf64 Replace models.BaseOperator to Task SDK one for Slack Provider (#52347)
add 3391ba2d41f fix (#52607)
add f9a12244d5e Add regional support for google secret manager hook (#52124)
add 448a1843bad Add more dependency reports (#52606)
add a3621016840 Correctly treat requeues on reschedule sensors as resetting after each reschedule (#51410)
add 760c68b6191 Update `BaseOperator` and `BaseSensorOperator` imports for Airflow 3.0 compatibility in `qdrant` provider (#52600)
add 53b0c6ce0a0 Provider Migration: Replace `models.BaseOperator` to Task SDK for `smtp` (#52596)
add ee67073239b Upgrade uv to 0.7.17 (#52615)
add 5af60f57608 Ensuring XCom return value can be mapped for dynamically-mapped `@task_group`'s (#51556)
add b7b832fe06e Add support for templating the DockerOperator parameter (#52451)
add bf1a6b6f52a Update `grpc` BaseOperator imports for Airflow 3.0 compatibility (#52603)
add 1fd66f1ab5d Provider Migration: Update Apache Kylin for Airflow 3.0 compatibility (#52572)
add f009834171d Update influxdb BaseOperator imports for Airflow 3.0 compatibility (#52602)
add efd0c9afa92 Revert "Fix symlink handling for static assets when installed in editable mode with uv (#52612)" (#52620)
add cdfc44683df Replace `models.BaseOperator` to Task SDK one for OpsGenie (#52564)
add c489678571e Improve dependency report and uppgrading (#52619)
add 97bbf3ba031 Allow more empty loops before stopping log streaming (#52614)
add 0cc667e286d Honor `index_urls` when venv is created with `uv` in `PythonVirtualenvOperator` (#52287)
add d49afebfd17 Provider Migration: Update cassandra for Airflow 3.0 compatibility (#52623)
add 87e34a69638 Bump pyarrow to 16.1.0 minimum version for several providers (#52635)
add dfa81e50850 Disable UP038 ruff rule and revert mandatory `X | Y` in insintance checks
new 516be1fae8e Switch all airflow logging to structlog
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (f74cd395ef5)
\
N -- N -- N refs/heads/structlog-in-logging-mixin (516be1fae8e)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/actions/breeze/action.yml | 4 +-
.github/actions/install-pre-commit/action.yml | 4 +-
.github/actions/prepare_all_ci_images/action.yml | 9 +-
.github/workflows/airflow-distributions-tests.yml | 7 +-
.github/workflows/basic-tests.yml | 5 +-
.github/workflows/ci-amd.yml | 16 +-
.github/workflows/ci-arm.yml | 4 +-
.github/workflows/ci-image-checks.yml | 4 +-
.github/workflows/finalize-tests.yml | 38 +-
.github/workflows/generate-constraints.yml | 82 +--
.github/workflows/prod-image-build.yml | 2 +-
.github/workflows/publish-docs-to-s3.yml | 4 +-
.github/workflows/release_dockerhub_image.yml | 2 +-
.../workflows/release_single_dockerhub_image.yml | 2 +-
.github/workflows/test-providers.yml | 7 +-
.pre-commit-config.yaml | 41 +-
.readthedocs.yml | 2 +-
AGENTS.md | 2 +-
Dockerfile | 4 +-
Dockerfile.ci | 32 +-
INSTALL | 6 +-
README.md | 8 +-
.../docs/administration-and-deployment/plugins.rst | 42 +-
airflow-core/docs/extra-packages-ref.rst | 2 +-
airflow-core/docs/howto/custom-view-plugin.rst | 12 +-
airflow-core/docs/img/airflow_erd.sha256 | 2 +-
airflow-core/docs/img/airflow_erd.svg | 318 ++++-----
.../docs/installation/installing-from-pypi.rst | 24 +-
airflow-core/docs/installation/prerequisites.rst | 2 +-
airflow-core/docs/start.rst | 6 +-
airflow-core/hatch_build.py | 4 +-
airflow-core/pyproject.toml | 9 +-
airflow-core/src/airflow/api_fastapi/app.py | 1 -
.../src/airflow/api_fastapi/auth/tokens.py | 4 +-
.../src/airflow/api_fastapi/common/parameters.py | 19 +-
.../src/airflow/api_fastapi/common/router.py | 3 +-
.../api_fastapi/core_api/datamodels/connections.py | 25 +-
.../api_fastapi/core_api/datamodels/plugins.py | 24 +-
.../api_fastapi/core_api/datamodels/pools.py | 3 +-
.../api_fastapi/core_api/datamodels/ui/common.py | 43 ++
.../api_fastapi/core_api/datamodels/ui/grid.py | 17 +-
.../api_fastapi/core_api/datamodels/variables.py | 2 +-
.../api_fastapi/core_api/openapi/_private_ui.yaml | 438 ++++++++++++-
.../core_api/openapi/v2-rest-api-generated.yaml | 59 +-
.../core_api/routes/public/import_error.py | 8 +-
.../airflow/api_fastapi/core_api/routes/ui/grid.py | 345 +++++++++-
.../src/airflow/api_fastapi/core_api/security.py | 3 +-
.../api_fastapi/core_api/services/ui/grid.py | 60 ++
.../execution_api/datamodels/taskinstance.py | 16 +-
.../src/airflow/api_fastapi/execution_api/deps.py | 8 +-
.../execution_api/versions/v2025_05_20.py | 4 +-
.../src/airflow/callbacks/callback_requests.py | 4 +-
.../src/airflow/callbacks/pipe_callback_sink.py | 3 +-
airflow-core/src/airflow/cli/cli_config.py | 6 +-
.../src/airflow/cli/commands/daemon_utils.py | 2 +-
.../cli/commands/rotate_fernet_key_command.py | 11 +-
airflow-core/src/airflow/cli/simple_table.py | 4 +-
airflow-core/src/airflow/configuration.py | 6 +-
airflow-core/src/airflow/dag_processing/manager.py | 2 +-
.../src/airflow/dag_processing/processor.py | 7 +-
.../example_time_delta_sensor_async.py | 7 +-
.../src/airflow/executors/base_executor.py | 4 +-
.../src/airflow/executors/local_executor.py | 4 +-
airflow-core/src/airflow/executors/workloads.py | 4 +-
airflow-core/src/airflow/io/__init__.py | 3 +-
airflow-core/src/airflow/jobs/job.py | 3 +-
.../src/airflow/jobs/scheduler_job_runner.py | 31 +-
.../src/airflow/jobs/triggerer_job_runner.py | 42 +-
airflow-core/src/airflow/lineage/hook.py | 4 +-
airflow-core/src/airflow/metrics/otel_logger.py | 5 +-
airflow-core/src/airflow/metrics/protocols.py | 4 +-
airflow-core/src/airflow/metrics/statsd_logger.py | 3 +-
airflow-core/src/airflow/metrics/validators.py | 4 +-
.../0036_3_0_0_add_name_field_to_dataset_model.py | 4 +-
.../versions/0038_3_0_0_add_asset_active.py | 2 +-
...9_3_0_0_tweak_assetaliasmodel_to_match_asset.py | 4 +-
airflow-core/src/airflow/models/base.py | 5 +
airflow-core/src/airflow/models/dag.py | 14 +-
airflow-core/src/airflow/models/dag_version.py | 1 +
airflow-core/src/airflow/models/dagrun.py | 31 +-
airflow-core/src/airflow/models/expandinput.py | 4 +-
airflow-core/src/airflow/models/operator.py | 4 +-
airflow-core/src/airflow/models/serialized_dag.py | 4 +-
airflow-core/src/airflow/models/variable.py | 46 +-
airflow-core/src/airflow/plugins_manager.py | 8 +-
airflow-core/src/airflow/providers_manager.py | 4 +-
airflow-core/src/airflow/serialization/serde.py | 6 +-
.../airflow/serialization/serialized_objects.py | 21 +-
.../src/airflow/serialization/serializers/numpy.py | 24 +-
airflow-core/src/airflow/settings.py | 3 +-
airflow-core/src/airflow/stats.py | 3 +-
airflow-core/src/airflow/timetables/interval.py | 4 +-
airflow-core/src/airflow/traces/tracer.py | 3 +-
airflow-core/src/airflow/triggers/base.py | 12 +-
airflow-core/src/airflow/typing_compat.py | 7 +-
.../src/airflow/ui/openapi-gen/queries/common.ts | 35 +
.../ui/openapi-gen/queries/ensureQueryData.ts | 79 +++
.../src/airflow/ui/openapi-gen/queries/prefetch.ts | 79 +++
.../src/airflow/ui/openapi-gen/queries/queries.ts | 79 +++
.../src/airflow/ui/openapi-gen/queries/suspense.ts | 79 +++
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 302 ++++++++-
.../ui/openapi-gen/requests/services.gen.ts | 133 +++-
.../airflow/ui/openapi-gen/requests/types.gen.ts | 200 +++++-
.../airflow/ui/public/i18n/locales/de/admin.json | 5 +-
.../airflow/ui/public/i18n/locales/de/common.json | 9 +
.../ui/public/i18n/locales/de/components.json | 2 +
.../airflow/ui/public/i18n/locales/ko/admin.json | 47 +-
.../airflow/ui/public/i18n/locales/ko/common.json | 11 +-
.../airflow/ui/public/i18n/locales/pl/admin.json | 16 +
.../airflow/ui/public/i18n/locales/pl/common.json | 33 +-
.../ui/public/i18n/locales/pl/components.json | 18 +-
.../src/airflow/ui/public/i18n/locales/pl/dag.json | 10 +
.../ui/public/i18n/locales/pl/dashboard.json | 2 +
.../airflow/ui/src/components/DurationChart.tsx | 17 +-
.../ui/src/layouts/Details/DagRunSelect.tsx | 23 +-
.../airflow/ui/src/layouts/Details/Grid/Bar.tsx | 19 +-
.../airflow/ui/src/layouts/Details/Grid/Grid.tsx | 70 +-
.../layouts/Details/Grid/TaskInstancesColumn.tsx | 5 +-
.../airflow/ui/src/layouts/Details/Grid/utils.ts | 46 +-
.../ui/src/layouts/Details/ToggleGroups.tsx | 15 +-
.../ui/src/layouts/Nav/TimezoneSelector.tsx | 16 +-
.../ui/src/layouts/Nav/UserSettingsButton.tsx | 16 +-
.../ui/src/pages/Connections/ConnectionForm.tsx | 12 +-
.../airflow/ui/src/pages/Dag/Overview/Overview.tsx | 26 +-
airflow-core/src/airflow/ui/src/pages/DagRuns.tsx | 4 +-
airflow-core/src/airflow/ui/src/pages/Iframe.tsx | 4 +-
airflow-core/src/airflow/ui/src/pages/Security.tsx | 13 +-
.../airflow/ui/src/pages/Task/GroupTaskHeader.tsx | 5 +-
.../src/airflow/ui/src/pages/Task/Task.tsx | 18 +-
.../airflow/ui/src/queries/useEditConnection.tsx | 4 +-
airflow-core/src/airflow/ui/src/queries/useGrid.ts | 72 ---
.../useGridRuns.ts} | 31 +-
.../useGridStructure.ts} | 38 +-
.../useGridTISummaries.ts} | 27 +-
.../ui/src/queries/useRefreshOnNewDagRuns.ts | 12 +-
airflow-core/src/airflow/ui/src/router.tsx | 7 +-
airflow-core/src/airflow/ui/src/utils/groupTask.ts | 11 +-
airflow-core/src/airflow/utils/cli.py | 3 +-
.../src/airflow/utils/cli_action_loggers.py | 3 +-
airflow-core/src/airflow/utils/db.py | 3 +-
airflow-core/src/airflow/utils/decorators.py | 3 +-
airflow-core/src/airflow/utils/event_scheduler.py | 2 +-
airflow-core/src/airflow/utils/helpers.py | 4 +-
.../src/airflow/utils/log/file_task_handler.py | 8 +-
airflow-core/src/airflow/utils/log/log_reader.py | 7 +-
.../src/airflow/utils/log/logging_mixin.py | 6 +-
airflow-core/src/airflow/utils/module_loading.py | 3 +-
airflow-core/src/airflow/utils/operator_helpers.py | 4 +-
.../utils/providers_configuration_loader.py | 3 +-
airflow-core/src/airflow/utils/retries.py | 3 +-
airflow-core/src/airflow/utils/session.py | 4 +-
airflow-core/src/airflow/utils/sqlalchemy.py | 7 +
airflow-core/src/airflow/utils/task_group.py | 41 +-
airflow-core/tests/unit/always/test_connection.py | 8 +-
.../tests/unit/always/test_project_structure.py | 24 +-
.../tests/unit/always/test_providers_manager.py | 10 +-
.../unit/api_fastapi/common/test_exceptions.py | 19 +-
.../core_api/routes/public/test_connections.py | 81 +--
.../core_api/routes/public/test_import_error.py | 55 +-
.../api_fastapi/core_api/routes/ui/test_grid.py | 446 +++++--------
.../versions/head/test_task_instances.py | 2 +-
.../unit/cli/commands/test_plugins_command.py | 10 +
.../tests/unit/cluster_policies/__init__.py | 3 +-
airflow-core/tests/unit/core/test_configuration.py | 49 +-
airflow-core/tests/unit/core/test_core.py | 24 +-
.../tests/unit/core/test_sqlalchemy_config.py | 21 +-
.../tests/unit/dag_processing/test_collection.py | 4 +
.../tests/unit/dag_processing/test_processor.py | 3 +-
airflow-core/tests/unit/datasets/test_dataset.py | 10 +-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 128 +++-
.../tests/unit/listeners/test_listeners.py | 12 +-
airflow-core/tests/unit/models/test_cleartasks.py | 9 +-
airflow-core/tests/unit/models/test_dagbag.py | 2 +-
.../tests/unit/models/test_taskinstance.py | 72 ++-
airflow-core/tests/unit/models/test_timestamp.py | 4 +-
airflow-core/tests/unit/models/test_variable.py | 21 +
airflow-core/tests/unit/plugins/test_plugin.py | 10 +
.../tests/unit/utils/log/test_log_reader.py | 8 +-
airflow-ctl/README.md | 2 +-
airflow-ctl/docs/images/command_hashes.txt | 28 +-
airflow-ctl/docs/images/output_assets.svg | 120 ++--
airflow-ctl/docs/images/output_auth.svg | 64 +-
airflow-ctl/docs/images/output_auth_login.svg | 76 +--
airflow-ctl/docs/images/output_backfills.svg | 84 +--
airflow-ctl/docs/images/output_config.svg | 66 +-
airflow-ctl/docs/images/output_connections.svg | 84 +--
airflow-ctl/docs/images/output_dag.svg | 108 ++--
airflow-ctl/docs/images/output_dagrun.svg | 64 +-
airflow-ctl/docs/images/output_jobs.svg | 56 +-
airflow-ctl/docs/images/output_main.svg | 116 ++--
airflow-ctl/docs/images/output_pools.svg | 80 +--
airflow-ctl/docs/images/output_providers.svg | 56 +-
airflow-ctl/docs/images/output_variables.svg | 80 +--
airflow-ctl/docs/images/output_version.svg | 40 +-
.../docs/installation/installing-from-pypi.rst | 2 +-
airflow-ctl/pyproject.toml | 7 +-
airflow-ctl/src/airflowctl/api/client.py | 7 +-
.../src/airflowctl/api/datamodels/generated.py | 92 +--
airflow-ctl/src/airflowctl/api/operations.py | 9 +-
airflow-ctl/src/airflowctl/ctl/cli_config.py | 6 +-
.../src/airflowctl/ctl/commands/pool_command.py | 12 +-
airflow-ctl/src/airflowctl/typing_compat.py | 29 -
airflow-ctl/src/airflowctl/utils/helpers.py | 4 +-
.../tests/airflow_ctl/api/test_operations.py | 24 +-
.../airflow_ctl/ctl/commands/test_auth_command.py | 49 +-
.../airflow_ctl/ctl/commands/test_pool_command.py | 6 +-
chart/Chart.yaml | 2 +-
chart/pyproject.toml | 3 +-
chart/templates/secrets/jwt-secret.yaml | 2 +-
clients/python/pyproject.toml | 5 +-
clients/python/test_python_client.py | 2 +-
constraints/README.md | 6 +-
contributing-docs/03_contributors_quick_start.rst | 24 +-
contributing-docs/05_pull_requests.rst | 2 +-
contributing-docs/07_local_virtualenv.rst | 14 +-
contributing-docs/08_static_code_checks.rst | 11 +-
.../13_airflow_dependencies_and_extras.rst | 10 +-
contributing-docs/testing/docker_compose_tests.rst | 8 +-
contributing-docs/testing/integration_tests.rst | 4 +-
contributing-docs/testing/k8s_tests.rst | 86 +--
contributing-docs/testing/unit_tests.rst | 10 +-
dev/README_RELEASE_AIRFLOW.md | 8 +-
dev/README_RELEASE_PROVIDERS.md | 2 +-
dev/airflow_mypy/plugin/outputs.py | 2 +-
dev/breeze/README.md | 2 +-
dev/breeze/doc/01_installation.rst | 6 +-
dev/breeze/doc/03_developer_tasks.rst | 60 +-
dev/breeze/doc/04_troubleshooting.rst | 8 +-
dev/breeze/doc/05_test_commands.rst | 6 +-
dev/breeze/doc/06_managing_docker_images.rst | 8 +-
dev/breeze/doc/10_advanced_breeze_topics.rst | 2 +-
.../0002-implement-standalone-python-command.md | 2 +-
dev/breeze/doc/ci/02_images.md | 112 ++--
dev/breeze/doc/ci/04_selective_checks.md | 145 +++--
dev/breeze/doc/ci/07_running_ci_locally.md | 4 +-
dev/breeze/doc/images/output-commands.svg | 142 +++--
dev/breeze/doc/images/output_ci-image_build.svg | 6 +-
dev/breeze/doc/images/output_ci-image_build.txt | 2 +-
dev/breeze/doc/images/output_ci-image_load.svg | 4 +-
dev/breeze/doc/images/output_ci-image_load.txt | 2 +-
dev/breeze/doc/images/output_ci-image_pull.svg | 6 +-
dev/breeze/doc/images/output_ci-image_pull.txt | 2 +-
dev/breeze/doc/images/output_ci-image_save.svg | 4 +-
dev/breeze/doc/images/output_ci-image_save.txt | 2 +-
dev/breeze/doc/images/output_ci-image_verify.svg | 6 +-
dev/breeze/doc/images/output_ci-image_verify.txt | 2 +-
.../doc/images/output_k8s_build-k8s-image.svg | 6 +-
.../doc/images/output_k8s_build-k8s-image.txt | 2 +-
.../doc/images/output_k8s_configure-cluster.svg | 6 +-
.../doc/images/output_k8s_configure-cluster.txt | 2 +-
.../doc/images/output_k8s_create-cluster.svg | 62 +-
.../doc/images/output_k8s_create-cluster.txt | 2 +-
.../doc/images/output_k8s_delete-cluster.svg | 4 +-
.../doc/images/output_k8s_delete-cluster.txt | 2 +-
.../doc/images/output_k8s_deploy-airflow.svg | 76 ++-
.../doc/images/output_k8s_deploy-airflow.txt | 2 +-
dev/breeze/doc/images/output_k8s_k9s.svg | 4 +-
dev/breeze/doc/images/output_k8s_k9s.txt | 2 +-
dev/breeze/doc/images/output_k8s_logs.svg | 4 +-
dev/breeze/doc/images/output_k8s_logs.txt | 2 +-
.../doc/images/output_k8s_run-complete-tests.svg | 6 +-
.../doc/images/output_k8s_run-complete-tests.txt | 2 +-
dev/breeze/doc/images/output_k8s_shell.svg | 4 +-
dev/breeze/doc/images/output_k8s_shell.txt | 2 +-
dev/breeze/doc/images/output_k8s_status.svg | 36 +-
dev/breeze/doc/images/output_k8s_status.txt | 2 +-
dev/breeze/doc/images/output_k8s_tests.svg | 6 +-
dev/breeze/doc/images/output_k8s_tests.txt | 2 +-
.../doc/images/output_k8s_upload-k8s-image.svg | 6 +-
.../doc/images/output_k8s_upload-k8s-image.txt | 2 +-
dev/breeze/doc/images/output_prod-image_build.svg | 196 +++---
dev/breeze/doc/images/output_prod-image_build.txt | 2 +-
dev/breeze/doc/images/output_prod-image_load.svg | 4 +-
dev/breeze/doc/images/output_prod-image_load.txt | 2 +-
dev/breeze/doc/images/output_prod-image_pull.svg | 6 +-
dev/breeze/doc/images/output_prod-image_pull.txt | 2 +-
dev/breeze/doc/images/output_prod-image_save.svg | 4 +-
dev/breeze/doc/images/output_prod-image_save.txt | 2 +-
dev/breeze/doc/images/output_prod-image_verify.svg | 6 +-
dev/breeze/doc/images/output_prod-image_verify.txt | 2 +-
...put_release-management_generate-constraints.svg | 48 +-
...put_release-management_generate-constraints.txt | 2 +-
...e-management_install-provider-distributions.svg | 166 +++--
...e-management_install-provider-distributions.txt | 2 +-
...output_release-management_merge-prod-images.svg | 2 +-
...output_release-management_merge-prod-images.txt | 2 +-
...tput_release-management_release-prod-images.svg | 2 +-
...tput_release-management_release-prod-images.txt | 2 +-
...se-management_verify-provider-distributions.svg | 98 ++-
...se-management_verify-provider-distributions.txt | 2 +-
dev/breeze/doc/images/output_run.svg | 256 ++++++++
dev/breeze/doc/images/output_run.txt | 1 +
dev/breeze/doc/images/output_sbom.svg | 8 +-
dev/breeze/doc/images/output_sbom.txt | 2 +-
.../output_sbom_export-dependency-information.svg | 4 +-
.../output_sbom_export-dependency-information.txt | 2 +-
.../images/output_sbom_update-sbom-information.svg | 122 ++--
.../images/output_sbom_update-sbom-information.txt | 2 +-
.../output_setup_check-all-params-in-groups.svg | 2 +-
.../output_setup_check-all-params-in-groups.txt | 2 +-
dev/breeze/doc/images/output_setup_config.svg | 42 +-
dev/breeze/doc/images/output_setup_config.txt | 2 +-
.../output_setup_regenerate-command-images.svg | 2 +-
.../output_setup_regenerate-command-images.txt | 2 +-
dev/breeze/doc/images/output_shell.svg | 158 ++---
dev/breeze/doc/images/output_shell.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 246 ++++---
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
dev/breeze/doc/images/output_static-checks.svg | 18 +-
dev/breeze/doc/images/output_static-checks.txt | 2 +-
.../images/output_testing_airflow-ctl-tests.svg | 4 +-
.../images/output_testing_airflow-ctl-tests.txt | 2 +-
.../output_testing_core-integration-tests.svg | 12 +-
.../output_testing_core-integration-tests.txt | 2 +-
.../doc/images/output_testing_core-tests.svg | 112 ++--
.../doc/images/output_testing_core-tests.txt | 2 +-
.../images/output_testing_docker-compose-tests.svg | 4 +-
.../images/output_testing_docker-compose-tests.txt | 2 +-
.../output_testing_providers-integration-tests.svg | 16 +-
.../output_testing_providers-integration-tests.txt | 2 +-
.../doc/images/output_testing_providers-tests.svg | 130 ++--
.../doc/images/output_testing_providers-tests.txt | 2 +-
.../output_testing_python-api-client-tests.svg | 12 +-
.../output_testing_python-api-client-tests.txt | 2 +-
.../doc/images/output_testing_system-tests.svg | 12 +-
.../doc/images/output_testing_system-tests.txt | 2 +-
.../doc/images/output_testing_task-sdk-tests.svg | 4 +-
.../doc/images/output_testing_task-sdk-tests.txt | 2 +-
dev/breeze/pyproject.toml | 11 +-
.../airflow_breeze/commands/ci_image_commands.py | 13 +-
.../src/airflow_breeze/commands/common_options.py | 39 +-
.../common_package_installation_options.py | 14 -
.../airflow_breeze/commands/developer_commands.py | 177 +++++-
.../commands/developer_commands_config.py | 42 +-
.../commands/minor_release_command.py | 2 +-
.../commands/release_management_commands.py | 42 +-
.../commands/release_management_commands_config.py | 3 +-
.../src/airflow_breeze/commands/sbom_commands.py | 136 +++-
.../commands/sbom_commands_config.py | 9 +-
.../src/airflow_breeze/commands/setup_commands.py | 1 +
.../airflow_breeze/commands/testing_commands.py | 9 +-
.../commands/testing_commands_config.py | 1 +
dev/breeze/src/airflow_breeze/global_constants.py | 25 +-
.../airflow_breeze/params/common_build_params.py | 2 +-
.../src/airflow_breeze/params/shell_params.py | 4 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 3 +-
.../prepare_providers/provider_documentation.py | 87 ++-
dev/breeze/src/airflow_breeze/utils/cdxgen.py | 31 +-
dev/breeze/src/airflow_breeze/utils/console.py | 7 +-
dev/breeze/src/airflow_breeze/utils/github.py | 6 +-
dev/breeze/src/airflow_breeze/utils/image.py | 3 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 2 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 8 +-
dev/breeze/src/airflow_breeze/utils/run_tests.py | 16 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 6 +-
.../src/airflow_breeze/utils/selective_checks.py | 53 +-
dev/breeze/tests/test_cache.py | 6 +-
dev/breeze/tests/test_provider_documentation.py | 52 ++
dev/breeze/tests/test_selective_checks.py | 186 +-----
dev/breeze/tests/test_shell_params.py | 24 +-
dev/breeze/uv.lock | 229 +------
dev/check_files.py | 4 +-
dev/constraints-updated-version-check.py | 567 +++++++++++++++++
dev/i18n/check_translations_completeness.py | 92 ++-
dev/provider_db_inventory.py | 2 +-
dev/pyproject.toml | 2 +-
dev/refresh_images.sh | 4 +-
dev/retag_docker_images.py | 2 +-
devel-common/pyproject.toml | 2 +-
devel-common/src/docs/build_docs.py | 4 +-
.../src/sphinx_exts/providers_extensions.py | 4 +-
.../tests_common/_internals/capture_warnings.py | 4 +-
devel-common/src/tests_common/pytest_plugin.py | 55 +-
.../src/tests_common/test_utils/mock_plugins.py | 1 +
.../test_utils/perf/perf_kit/sqlalchemy.py | 2 +-
.../src/tests_common/test_utils/system_tests.py | 3 +-
.../src/tests_common/test_utils/version_compat.py | 12 +
docker-stack-docs/build-arg-ref.rst | 100 +--
docker-stack-docs/build.rst | 34 +-
.../customizing/add-build-essential-custom.sh | 2 +-
.../docker-examples/customizing/custom-sources.sh | 2 +-
.../customizing/github-different-repository.sh | 2 +-
.../docker-examples/customizing/github-main.sh | 2 +-
.../customizing/github-v2-2-test.sh | 2 +-
.../customizing/pypi-dev-runtime-deps.sh | 2 +-
.../customizing/pypi-extras-and-deps.sh | 2 +-
.../customizing/pypi-selected-version.sh | 2 +-
.../restricted/restricted_environments.sh | 10 +-
docker-stack-docs/entrypoint.rst | 14 +-
docker-stack-docs/pyproject.toml | 3 +-
docker-tests/pyproject.toml | 2 +-
docker-tests/tests/docker_tests/constants.py | 2 +-
docker-tests/tests/docker_tests/docker_utils.py | 4 +-
docs/spelling_wordlist.txt | 1 +
generated/PYPI_README.md | 6 +-
helm-tests/pyproject.toml | 2 +-
.../airflow_aux/test_basic_helm_chart.py | 2 +-
.../airflow_aux/test_container_lifecycle.py | 3 +
.../tests/helm_tests/airflow_core/test_worker.py | 2 +-
kubernetes-tests/pyproject.toml | 2 +-
.../test_kubernetes_pod_operator.py | 2 +-
.../performance_dag/performance_dag_utils.py | 2 +-
providers-summary-docs/installing-from-pypi.rst | 2 +-
providers-summary-docs/pyproject.toml | 3 +-
providers/airbyte/README.rst | 2 +-
providers/airbyte/pyproject.toml | 3 +-
.../airflow/providers/airbyte/operators/airbyte.py | 4 +-
.../airflow/providers/airbyte/sensors/airbyte.py | 4 +-
.../airflow/providers/airbyte}/version_compat.py | 15 +
providers/alibaba/README.rst | 2 +-
providers/alibaba/pyproject.toml | 3 +-
.../providers/alibaba/cloud/hooks/maxcompute.py | 3 +-
.../airflow/providers/alibaba/cloud/hooks/oss.py | 3 +-
.../providers/alibaba/cloud/links/maxcompute.py | 5 +-
.../alibaba/cloud/operators/analyticdb_spark.py | 7 +-
.../alibaba/cloud/operators/maxcompute.py | 11 +-
.../providers/alibaba/cloud/operators/oss.py | 7 +-
.../alibaba/cloud/sensors/analyticdb_spark.py | 7 +-
.../providers/alibaba/cloud/sensors/oss_key.py | 7 +-
.../unit/alibaba/cloud/links/test_maxcompute.py | 4 +-
.../alibaba/cloud/operators/test_maxcompute.py | 1 -
.../tests/unit/alibaba/cloud/utils/test_utils.py | 3 +-
providers/amazon/docs/executors/general.rst | 6 +-
providers/amazon/pyproject.toml | 3 +-
.../providers/amazon/aws/executors/ecs/utils.py | 4 +-
.../executors/utils/exponential_backoff_retry.py | 2 +-
.../src/airflow/providers/amazon/aws/fs/s3.py | 3 +-
.../providers/amazon/aws/hooks/athena_sql.py | 9 +-
.../airflow/providers/amazon/aws/hooks/base_aws.py | 18 +-
.../providers/amazon/aws/hooks/batch_client.py | 3 +-
.../providers/amazon/aws/hooks/batch_waiters.py | 3 +-
.../airflow/providers/amazon/aws/hooks/chime.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/ec2.py | 3 +-
.../src/airflow/providers/amazon/aws/hooks/eks.py | 3 +-
.../src/airflow/providers/amazon/aws/hooks/glue.py | 89 ++-
.../src/airflow/providers/amazon/aws/hooks/rds.py | 3 +-
.../src/airflow/providers/amazon/aws/hooks/s3.py | 4 +-
.../providers/amazon/aws/hooks/sagemaker.py | 4 +-
.../airflow/providers/amazon/aws/operators/glue.py | 30 +-
.../providers/amazon/aws/operators/sagemaker.py | 4 +-
.../providers/amazon/aws/sensors/base_aws.py | 8 +-
.../airflow/providers/amazon/aws/sensors/glue.py | 68 +-
.../src/airflow/providers/amazon/aws/sensors/s3.py | 4 +-
.../amazon/aws/sensors/sagemaker_unified_studio.py | 7 +-
.../amazon/aws/transfers/dynamodb_to_s3.py | 4 +-
.../amazon/aws/transfers/hive_to_dynamodb.py | 4 +-
.../providers/amazon/aws/transfers/s3_to_sql.py | 4 +-
.../providers/amazon/aws/transfers/sql_to_s3.py | 4 +-
.../airflow/providers/amazon/aws/triggers/base.py | 1 -
.../airflow/providers/amazon/aws/triggers/glue.py | 61 +-
.../airflow/providers/amazon/aws/utils/suppress.py | 3 +-
.../airflow/providers/amazon/aws/utils/waiter.py | 2 +-
.../airflow/providers/amazon/aws/waiters/glue.json | 55 ++
.../tests/system/amazon/aws/example_sagemaker.py | 2 +-
.../amazon/aws/tests/test_aws_auth_manager.py | 2 +-
.../tests/system/amazon/aws/utils/__init__.py | 3 +-
.../amazon/tests/system/amazon/aws/utils/k8s.py | 7 +-
.../aws/auth_manager/cli/test_avp_commands.py | 2 +-
.../amazon/aws/auth_manager/cli/test_definition.py | 2 +-
.../amazon/aws/auth_manager/routes/test_login.py | 2 +-
.../aws/auth_manager/test_aws_auth_manager.py | 2 +-
.../amazon/aws/executors/ecs/test_ecs_executor.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_eks.py | 3 +
.../tests/unit/amazon/aws/hooks/test_glue.py | 121 ++++
.../unit/amazon/aws/hooks/test_hooks_signature.py | 1 +
.../tests/unit/amazon/aws/links/test_athena.py | 2 +-
.../tests/unit/amazon/aws/links/test_batch.py | 2 +-
.../tests/unit/amazon/aws/links/test_comprehend.py | 2 +-
.../tests/unit/amazon/aws/links/test_datasync.py | 2 +-
.../amazon/tests/unit/amazon/aws/links/test_ec2.py | 2 +-
.../amazon/tests/unit/amazon/aws/links/test_emr.py | 2 +-
.../tests/unit/amazon/aws/links/test_glue.py | 2 +-
.../tests/unit/amazon/aws/links/test_logs.py | 2 +-
.../tests/unit/amazon/aws/links/test_sagemaker.py | 2 +-
.../aws/links/test_sagemaker_unified_studio.py | 2 +-
.../unit/amazon/aws/links/test_step_function.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_glue.py | 65 +-
.../tests/unit/amazon/aws/triggers/test_glue.py | 56 +-
.../tests/unit/amazon/aws/utils/eks_test_utils.py | 4 +-
.../amazon/aws/utils/test_connection_wrapper.py | 2 +-
.../tests/unit/amazon/aws/waiters/test_glue.py | 45 +-
providers/apache/beam/pyproject.toml | 5 +-
.../airflow/providers/apache/beam/hooks/beam.py | 3 +-
.../providers/apache/beam/operators/beam.py | 9 +-
.../providers/apache/beam}/version_compat.py | 15 +-
.../tests/unit/apache/beam/operators/test_beam.py | 40 +-
providers/apache/cassandra/pyproject.toml | 3 +-
.../providers/apache/cassandra/hooks/cassandra.py | 4 +-
.../providers/apache/cassandra/sensors/record.py | 2 +-
.../providers/apache/cassandra/sensors/table.py | 2 +-
.../providers/apache/cassandra}/version_compat.py | 9 +-
providers/apache/drill/pyproject.toml | 3 +-
providers/apache/druid/pyproject.toml | 3 +-
.../providers/apache/druid/operators/druid.py | 4 +-
.../apache/druid/transfers/hive_to_druid.py | 4 +-
.../providers/apache/druid}/version_compat.py | 13 +
providers/apache/flink/pyproject.toml | 3 +-
.../apache/flink/operators/flink_kubernetes.py | 4 +-
.../apache/flink/sensors/flink_kubernetes.py | 4 +-
.../providers/apache/flink}/version_compat.py | 15 +
providers/apache/hdfs/pyproject.toml | 3 +-
.../airflow/providers/apache/hdfs/hooks/webhdfs.py | 2 -
.../providers/apache/hdfs/sensors/web_hdfs.py | 2 +-
.../providers/apache/hdfs}/version_compat.py | 9 +-
providers/apache/hive/pyproject.toml | 4 +-
.../airflow/providers/apache/hive/hooks/hive.py | 13 +-
.../providers/apache/hive/operators/hive.py | 15 +-
.../providers/apache/hive/operators/hive_stats.py | 6 +-
.../apache/hive/sensors/hive_partition.py | 2 +-
.../apache/hive/sensors/named_hive_partition.py | 2 +-
.../apache/hive/transfers/hive_to_mysql.py | 9 +-
.../apache/hive/transfers/hive_to_samba.py | 8 +-
.../apache/hive/transfers/mssql_to_hive.py | 2 +-
.../apache/hive/transfers/mysql_to_hive.py | 2 +-
.../providers/apache/hive/transfers/s3_to_hive.py | 2 +-
.../apache/hive/transfers/vertica_to_hive.py | 2 +-
.../providers/apache/hive}/version_compat.py | 20 +
providers/apache/iceberg/pyproject.toml | 3 +-
providers/apache/impala/pyproject.toml | 3 +-
.../providers/apache/impala}/version_compat.py | 0
providers/apache/kafka/pyproject.toml | 3 +-
.../providers/apache/kafka/operators/consume.py | 6 +-
.../providers/apache/kafka/operators/produce.py | 6 +-
.../providers/apache/kafka/sensors/kafka.py | 8 +-
.../providers/apache/kafka}/version_compat.py | 12 +-
providers/apache/kylin/pyproject.toml | 3 +-
.../providers/apache/kylin/operators/kylin_cube.py | 2 +-
.../providers/apache/kylin}/version_compat.py | 12 +-
providers/apache/livy/pyproject.toml | 3 +-
.../airflow/providers/apache/livy/hooks/livy.py | 2 -
.../providers/apache/livy/operators/livy.py | 4 +-
.../airflow/providers/apache/livy/sensors/livy.py | 4 +-
.../airflow/providers/apache/livy/triggers/livy.py | 3 -
.../providers/apache/livy}/version_compat.py | 8 +
providers/apache/pig/pyproject.toml | 3 +-
.../airflow/providers/apache/pig/operators/pig.py | 2 +-
.../providers/apache/pig}/version_compat.py | 12 +-
providers/apache/pinot/pyproject.toml | 3 +-
providers/apache/spark/pyproject.toml | 3 +-
.../providers/apache/spark/decorators/pyspark.py | 4 +-
.../providers/apache/spark/operators/spark_sql.py | 2 +-
.../apache/spark/operators/spark_submit.py | 2 +-
.../providers/apache/spark/version_compat.py | 10 +
.../tests/system/apache/spark/example_pyspark.py | 3 +-
.../unit/apache/spark/decorators/test_pyspark.py | 16 +-
providers/apache/tinkerpop/pyproject.toml | 3 +-
.../apache/tinkerpop/operators/gremlin.py | 2 +-
.../providers/apache/tinkerpop}/version_compat.py | 8 +
.../apache/tinkerpop/hooks/test_gremlin.py | 2 +-
providers/apprise/pyproject.toml | 3 +-
providers/arangodb/pyproject.toml | 3 +-
.../providers/arangodb/operators/arangodb.py | 6 +-
.../airflow/providers/arangodb/sensors/arangodb.py | 2 +-
.../airflow/providers/arangodb}/version_compat.py | 12 +
providers/asana/pyproject.toml | 3 +-
.../providers/asana/operators/asana_tasks.py | 2 +-
.../src/airflow/providers/asana}/version_compat.py | 11 +
providers/atlassian/jira/pyproject.toml | 3 +-
.../providers/atlassian/jira/operators/jira.py | 8 +-
.../providers/atlassian/jira/sensors/jira.py | 11 +-
.../providers/atlassian/jira}/version_compat.py | 12 +
.../unit/atlassian/jira/operators/test_jira.py | 20 +-
providers/celery/pyproject.toml | 3 +-
.../providers/celery/executors/celery_executor.py | 1 -
.../celery/executors/celery_executor_utils.py | 8 +-
.../providers/celery/sensors/celery_queue.py | 7 +-
.../unit/celery/executors/test_celery_executor.py | 2 +-
providers/cloudant/provider.yaml | 7 -
providers/cloudant/pyproject.toml | 6 +-
.../airflow/providers/cloudant/hooks/cloudant.py | 6 +-
.../tests/unit/cloudant/hooks/test_cloudant.py | 13 +-
providers/cncf/kubernetes/pyproject.toml | 3 +-
.../airflow/providers/cncf/kubernetes/callbacks.py | 4 +-
.../cncf/kubernetes/decorators/kubernetes.py | 4 +-
.../cncf/kubernetes/decorators/kubernetes_cmd.py | 4 +-
.../executors/kubernetes_executor_types.py | 8 +-
.../providers/cncf/kubernetes/kube_client.py | 13 +-
.../providers/cncf/kubernetes/operators/pod.py | 16 +-
.../cncf/kubernetes/operators/spark_kubernetes.py | 25 +-
.../cncf/kubernetes/sensors/spark_kubernetes.py | 7 +-
.../cncf/kubernetes/utils/k8s_resource_iterator.py | 3 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 3 +-
.../cncf/kubernetes/cli/test_kubernetes_command.py | 2 +-
.../cncf/kubernetes/decorators/test_kubernetes.py | 2 +-
.../kubernetes/decorators/test_kubernetes_cmd.py | 3 +-
.../decorators/test_kubernetes_commons.py | 4 +-
.../executors/test_kubernetes_executor.py | 2 +-
.../kubernetes/operators/test_spark_kubernetes.py | 39 +-
.../cncf/kubernetes/test_template_rendering.py | 2 +-
providers/cohere/pyproject.toml | 3 +-
.../providers/cohere/operators/embedding.py | 8 +-
.../airflow/providers/cohere}/version_compat.py | 14 +
providers/common/compat/pyproject.toml | 3 +-
.../providers/common/compat/standard/operators.py | 3 +-
.../providers/common/compat/version_compat.py | 12 +
providers/common/io/pyproject.toml | 3 +-
.../providers/common/io/operators/file_transfer.py | 4 +-
.../airflow/providers/common/io/version_compat.py | 10 +
.../common/io/example_file_transfer_local_to_s3.py | 3 +-
.../io/tests/unit/common/io/xcom/test_backend.py | 2 +-
providers/common/messaging/pyproject.toml | 3 +-
providers/common/sql/pyproject.toml | 3 +-
.../providers/common/sql/dialects/dialect.py | 4 +-
.../providers/common/sql/dialects/dialect.pyi | 4 +-
.../src/airflow/providers/common/sql/hooks/sql.py | 5 +-
.../src/airflow/providers/common/sql/hooks/sql.pyi | 4 +-
.../common/sql/operators/generic_transfer.py | 4 +-
.../airflow/providers/common/sql/operators/sql.py | 7 +-
.../airflow/providers/common/sql/sensors/sql.py | 11 +-
.../airflow/providers/common/sql/sensors/sql.pyi | 12 +-
.../providers/common/sql}/version_compat.py | 12 +
.../common/sql/operators/test_generic_transfer.py | 59 +-
.../tests/unit/common/sql/operators/test_sql.py | 57 +-
providers/databricks/pyproject.toml | 5 +-
.../providers/databricks/hooks/databricks_base.py | 9 +-
.../providers/databricks/hooks/databricks_sql.py | 27 +-
.../providers/databricks/operators/databricks.py | 27 +-
.../databricks/operators/databricks_repos.py | 2 +-
.../databricks/operators/databricks_sql.py | 2 +-
.../databricks/operators/databricks_workflow.py | 23 +-
.../databricks/plugins/databricks_workflow.py | 175 +++--
.../databricks/sensors/databricks_partition.py | 11 +-
.../providers/databricks/sensors/databricks_sql.py | 11 +-
.../providers/databricks/utils/openlineage.py | 143 +++--
.../airflow/providers/databricks/version_compat.py | 10 +
.../tests/unit/databricks/hooks/test_databricks.py | 56 +-
.../unit/databricks/hooks/test_databricks_sql.py | 20 +-
.../operators/test_databricks_workflow.py | 1 +
.../databricks/plugins/test_databricks_workflow.py | 245 ++++++-
.../unit/databricks/utils/test_openlineage.py | 634 +++++++++++++++++-
providers/datadog/pyproject.toml | 3 +-
.../airflow/providers/datadog/sensors/datadog.py | 11 +-
.../airflow/providers/datadog}/version_compat.py | 13 +
providers/dbt/cloud/pyproject.toml | 3 +-
.../src/airflow/providers/dbt/cloud/hooks/dbt.py | 4 +-
.../airflow/providers/dbt/cloud/operators/dbt.py | 14 +-
.../src/airflow/providers/dbt/cloud/sensors/dbt.py | 2 +-
.../airflow/providers/dbt/cloud/version_compat.py | 16 +
.../tests/unit/dbt/cloud/operators/test_dbt.py | 3 +-
providers/dingding/pyproject.toml | 3 +-
.../providers/dingding/operators/dingding.py | 8 +-
.../airflow/providers/dingding}/version_compat.py | 14 +
providers/discord/pyproject.toml | 3 +-
providers/docker/pyproject.toml | 3 +-
.../airflow/providers/docker/decorators/docker.py | 4 +-
.../airflow/providers/docker/operators/docker.py | 18 +-
.../src/airflow/providers/docker/version_compat.py | 8 +
.../tests/unit/docker/decorators/test_docker.py | 42 +-
.../tests/unit/docker/operators/test_docker.py | 20 +
providers/edge3/docs/deployment.rst | 6 +-
providers/edge3/docs/install_on_windows.rst | 2 +-
providers/edge3/pyproject.toml | 3 +-
.../airflow/providers/edge3/cli/edge_command.py | 2 +-
.../src/airflow/providers/edge3/cli/worker.py | 17 +-
.../providers/edge3/example_dags/win_test.py | 4 +-
.../providers/edge3/executors/edge_executor.py | 5 +-
.../edge3/worker_api/routes/_v2_compat.py | 2 +-
.../edge3/tests/unit/edge3/cli/test_worker.py | 46 ++
providers/elasticsearch/pyproject.toml | 3 +-
.../providers/elasticsearch/log/es_task_handler.py | 7 +-
providers/exasol/pyproject.toml | 3 +-
.../src/airflow/providers/exasol/hooks/exasol.py | 4 +-
providers/fab/pyproject.toml | 3 +-
.../auth_manager/api/auth/backend/basic_auth.py | 3 +-
.../auth_manager/api/auth/backend/kerberos_auth.py | 3 +-
.../fab/auth_manager/api/auth/backend/session.py | 3 +-
.../openapi/v2-fab-auth-manager-generated.yaml | 4 +-
.../fab/auth_manager/security_manager/override.py | 8 +-
.../providers/fab/www/api_connexion/parameters.py | 4 +-
.../providers/fab/www/api_connexion/security.py | 3 +-
.../providers/fab/www/api_connexion/types.py | 12 +-
.../fab/src/airflow/providers/fab/www/auth.py | 4 +-
.../airflow/providers/fab/www/security_manager.py | 2 +-
.../airflow/providers/fab/www/static}/pin_100.png | Bin
.../airflow/providers/fab/www/static}/pin_32.png | Bin
.../api_endpoints/remote_user_api_auth_backend.py | 3 +-
providers/fab/www-hash.txt | 2 +-
providers/facebook/pyproject.toml | 3 +-
providers/ftp/pyproject.toml | 3 +-
.../ftp/src/airflow/providers/ftp/hooks/ftp.py | 3 +-
.../ftp/src/airflow/providers/ftp/sensors/ftp.py | 7 +-
.../src/airflow/providers/ftp}/version_compat.py | 0
providers/git/pyproject.toml | 3 +-
providers/github/pyproject.toml | 3 +-
.../airflow/providers/github/operators/github.py | 11 +-
.../src/airflow/providers/github/sensors/github.py | 11 +-
.../airflow/providers/github}/version_compat.py | 16 +
.../tests/unit/github/operators/test_github.py | 21 +-
providers/google/pyproject.toml | 8 +-
.../airflow/providers/google/ads/operators/ads.py | 2 +-
.../providers/google/ads/transfers/ads_to_gcs.py | 2 +-
.../providers/google/cloud/hooks/bigquery.py | 5 +-
.../providers/google/cloud/hooks/dataflow.py | 4 +-
.../providers/google/cloud/hooks/dataplex.py | 2 +-
.../airflow/providers/google/cloud/hooks/gcs.py | 4 +-
.../providers/google/cloud/hooks/mlengine.py | 3 +-
.../providers/google/cloud/hooks/secret_manager.py | 112 +++-
.../providers/google/cloud/hooks/spanner.py | 4 +-
.../providers/google/cloud/hooks/translate.py | 2 +-
.../cloud/hooks/vertex_ai/generative_model.py | 19 +-
.../airflow/providers/google/cloud/hooks/vision.py | 4 +-
.../airflow/providers/google/cloud/links/base.py | 21 +-
.../providers/google/cloud/links/dataproc.py | 28 +-
.../providers/google/cloud/links/translate.py | 2 +-
.../providers/google/cloud/operators/automl.py | 7 +-
.../google/cloud/operators/bigquery_dts.py | 4 +-
.../providers/google/cloud/operators/cloud_base.py | 2 +-
.../google/cloud/operators/cloud_build.py | 12 +-
.../google/cloud/operators/datacatalog.py | 10 +-
.../providers/google/cloud/operators/dataflow.py | 14 +-
.../providers/google/cloud/operators/dataplex.py | 17 +-
.../providers/google/cloud/operators/dataproc.py | 14 +-
.../google/cloud/operators/dataproc_metastore.py | 2 +-
.../providers/google/cloud/operators/functions.py | 2 +-
.../google/cloud/operators/managed_kafka.py | 9 +-
.../providers/google/cloud/operators/pubsub.py | 4 +-
.../providers/google/cloud/operators/translate.py | 38 +-
.../google/cloud/operators/vertex_ai/auto_ml.py | 16 +-
.../operators/vertex_ai/batch_prediction_job.py | 8 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 28 +-
.../google/cloud/operators/vertex_ai/dataset.py | 48 +-
.../cloud/operators/vertex_ai/endpoint_service.py | 4 +-
.../cloud/operators/vertex_ai/generative_model.py | 20 +-
.../vertex_ai/hyperparameter_tuning_job.py | 2 +-
.../cloud/operators/vertex_ai/model_service.py | 4 +-
.../cloud/operators/vertex_ai/pipeline_job.py | 2 +-
.../google/cloud/operators/vertex_ai/ray.py | 3 +-
.../providers/google/cloud/operators/workflows.py | 2 +-
.../providers/google/cloud/sensors/bigquery.py | 2 +-
.../providers/google/cloud/sensors/bigquery_dts.py | 7 +-
.../providers/google/cloud/sensors/bigtable.py | 7 +-
.../google/cloud/sensors/cloud_composer.py | 7 +-
.../sensors/cloud_storage_transfer_service.py | 7 +-
.../providers/google/cloud/sensors/dataflow.py | 6 +-
.../providers/google/cloud/sensors/dataform.py | 7 +-
.../providers/google/cloud/sensors/datafusion.py | 7 +-
.../providers/google/cloud/sensors/dataplex.py | 7 +-
.../providers/google/cloud/sensors/dataprep.py | 7 +-
.../providers/google/cloud/sensors/dataproc.py | 7 +-
.../google/cloud/sensors/dataproc_metastore.py | 7 +-
.../airflow/providers/google/cloud/sensors/gcs.py | 12 +-
.../providers/google/cloud/sensors/looker.py | 7 +-
.../providers/google/cloud/sensors/pubsub.py | 11 +-
.../providers/google/cloud/sensors/tasks.py | 7 +-
.../cloud/sensors/vertex_ai/feature_store.py | 7 +-
.../providers/google/cloud/sensors/workflows.py | 7 +-
.../google/cloud/transfers/azure_blob_to_gcs.py | 2 +-
.../cloud/transfers/azure_fileshare_to_gcs.py | 2 +-
.../google/cloud/transfers/bigquery_to_bigquery.py | 2 +-
.../google/cloud/transfers/bigquery_to_gcs.py | 2 +-
.../google/cloud/transfers/bigquery_to_sql.py | 2 +-
.../google/cloud/transfers/calendar_to_gcs.py | 2 +-
.../google/cloud/transfers/cassandra_to_gcs.py | 2 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 2 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 2 +-
.../providers/google/cloud/transfers/gcs_to_gcs.py | 2 +-
.../google/cloud/transfers/gcs_to_local.py | 2 +-
.../google/cloud/transfers/gcs_to_sftp.py | 2 +-
.../google/cloud/transfers/gdrive_to_gcs.py | 6 +-
.../google/cloud/transfers/gdrive_to_local.py | 2 +-
.../google/cloud/transfers/http_to_gcs.py | 2 +-
.../google/cloud/transfers/local_to_gcs.py | 2 +-
.../google/cloud/transfers/salesforce_to_gcs.py | 2 +-
.../google/cloud/transfers/sftp_to_gcs.py | 2 +-
.../google/cloud/transfers/sheets_to_gcs.py | 4 +-
.../providers/google/cloud/transfers/sql_to_gcs.py | 2 +-
.../google/cloud/utils/field_validator.py | 3 +-
.../google/common/auth_backend/google_openid.py | 3 +-
.../airflow/providers/google/common/deprecated.py | 3 +-
.../providers/google/common/hooks/base_google.py | 4 +-
.../google/firebase/operators/firestore.py | 2 +-
.../providers/google/leveldb/operators/leveldb.py | 2 +-
.../marketing_platform/links/analytics_admin.py | 9 +-
.../operators/analytics_admin.py | 1 -
.../operators/campaign_manager.py | 8 +-
.../marketing_platform/operators/display_video.py | 12 +-
.../marketing_platform/operators/search_ads.py | 2 +-
.../marketing_platform/sensors/campaign_manager.py | 7 +-
.../marketing_platform/sensors/display_video.py | 7 +-
.../providers/google/suite/operators/sheets.py | 6 +-
.../providers/google/suite/sensors/drive.py | 7 +-
.../google/suite/transfers/gcs_to_gdrive.py | 2 +-
.../google/suite/transfers/gcs_to_sheets.py | 2 +-
.../google/suite/transfers/local_to_drive.py | 2 +-
.../src/airflow/providers/google/version_compat.py | 28 +
.../example_dataflow_native_python_async.py | 2 +-
.../example_dataflow_sensors_deferrable.py | 2 +-
.../kubernetes_engine/example_kubernetes_engine.py | 2 +-
.../example_kubernetes_engine_async.py | 2 +-
.../google/cloud/translate/example_translate.py | 3 +-
.../tests/unit/google/cloud/hooks/test_bigquery.py | 34 +-
.../unit/google/cloud/hooks/test_secret_manager.py | 71 ++-
.../unit/google/cloud/links/test_base_link.py | 3 +-
.../unit/google/cloud/links/test_cloud_run.py | 3 +-
.../tests/unit/google/cloud/links/test_dataplex.py | 3 +-
.../unit/google/cloud/links/test_translate.py | 2 +-
.../unit/google/cloud/operators/test_automl.py | 8 +-
.../unit/google/cloud/operators/test_bigquery.py | 216 +++----
.../google/cloud/operators/test_bigquery_dts.py | 12 +-
.../google/cloud/operators/test_cloud_build.py | 2 +-
.../google/cloud/operators/test_datacatalog.py | 85 +--
.../unit/google/cloud/operators/test_dataflow.py | 31 +-
.../unit/google/cloud/operators/test_dataproc.py | 204 +++---
.../unit/google/cloud/operators/test_functions.py | 15 +-
.../google/cloud/operators/test_life_sciences.py | 8 +-
.../unit/google/cloud/operators/test_translate.py | 53 +-
.../unit/google/cloud/operators/test_vertex_ai.py | 111 ++--
.../google/cloud/transfers/test_sheets_to_gcs.py | 18 +-
.../tests/unit/google/cloud/utils/airflow_util.py | 2 +-
.../common/auth_backend/test_google_openid.py | 2 +-
.../links/test_analytics_admin.py | 4 +-
.../operators/test_campaign_manager.py | 85 ++-
.../operators/test_display_video.py | 103 ++-
.../unit/google/suite/operators/test_sheets.py | 13 +-
providers/grpc/pyproject.toml | 3 +-
.../grpc/src/airflow/providers/grpc/hooks/grpc.py | 4 +-
.../src/airflow/providers/grpc/operators/grpc.py | 6 +-
.../src/airflow/providers/grpc}/version_compat.py | 8 +
providers/hashicorp/pyproject.toml | 3 +-
providers/http/pyproject.toml | 3 +-
.../http/src/airflow/providers/http/hooks/http.py | 138 ++--
.../src/airflow/providers/http/operators/http.py | 6 +-
.../src/airflow/providers/http/sensors/http.py | 12 +-
.../src/airflow/providers/http/triggers/http.py | 2 +-
.../src/airflow/providers/http}/version_compat.py | 8 +
providers/http/tests/unit/http/hooks/test_http.py | 9 +-
.../http/tests/unit/http/sensors/test_http.py | 15 +-
providers/imap/pyproject.toml | 3 +-
.../providers/imap/sensors/imap_attachment.py | 7 +-
.../src/airflow/providers/imap}/version_compat.py | 0
providers/influxdb/pyproject.toml | 3 +-
.../providers/influxdb/operators/influxdb.py | 2 +-
.../airflow/providers/influxdb}/version_compat.py | 8 +
providers/jdbc/pyproject.toml | 3 +-
providers/jenkins/pyproject.toml | 3 +-
.../jenkins/operators/jenkins_job_trigger.py | 9 +-
.../airflow/providers/jenkins/sensors/jenkins.py | 8 +-
.../airflow/providers/jenkins}/version_compat.py | 15 +
providers/keycloak/pyproject.toml | 3 +-
providers/microsoft/azure/pyproject.toml | 13 +-
.../airflow/providers/microsoft/azure/hooks/asb.py | 3 +-
.../providers/microsoft/azure/hooks/cosmos.py | 4 +-
.../microsoft/azure/hooks/data_factory.py | 7 +-
.../providers/microsoft/azure/hooks/data_lake.py | 4 +-
.../providers/microsoft/azure/hooks/msgraph.py | 6 +-
.../providers/microsoft/azure/hooks/synapse.py | 4 +-
.../providers/microsoft/azure/hooks/wasb.py | 4 +-
.../providers/microsoft/azure/operators/asb.py | 4 +-
.../providers/microsoft/azure/operators/msgraph.py | 3 +-
.../providers/microsoft/azure/sensors/cosmos.py | 7 +-
.../providers/microsoft/azure/sensors/msgraph.py | 11 +-
.../providers/microsoft/azure/sensors/wasb.py | 7 +-
.../unit/microsoft/azure/hooks/test_msgraph.py | 4 +-
.../unit/microsoft/azure/operators/test_adx.py | 3 +-
.../unit/microsoft/azure/operators/test_msgraph.py | 54 +-
.../unit/microsoft/azure/operators/test_synapse.py | 3 +-
.../unit/microsoft/azure/sensors/test_msgraph.py | 28 +-
providers/microsoft/mssql/pyproject.toml | 14 +-
providers/microsoft/psrp/pyproject.toml | 3 +-
.../airflow/providers/microsoft/psrp/hooks/psrp.py | 4 +-
providers/microsoft/winrm/pyproject.toml | 3 +-
providers/mongo/pyproject.toml | 3 +-
.../src/airflow/providers/mongo/hooks/mongo.py | 2 +-
.../src/airflow/providers/mongo/sensors/mongo.py | 2 +-
.../src/airflow/providers/mongo}/version_compat.py | 9 +-
providers/mysql/pyproject.toml | 3 +-
.../providers/mysql/transfers/presto_to_mysql.py | 2 +-
.../providers/mysql/transfers/s3_to_mysql.py | 2 +-
.../providers/mysql/transfers/trino_to_mysql.py | 2 +-
.../providers/mysql/transfers/vertica_to_mysql.py | 2 +-
.../src/airflow/providers/mysql}/version_compat.py | 11 +
.../unit/mysql/transfers/test_trino_to_mysql.py | 31 +-
providers/neo4j/pyproject.toml | 3 +-
.../src/airflow/providers/neo4j/operators/neo4j.py | 2 +-
.../src/airflow/providers/neo4j}/version_compat.py | 12 +-
providers/odbc/pyproject.toml | 3 +-
providers/openai/pyproject.toml | 3 +-
.../airflow/providers/openai/operators/openai.py | 4 +-
.../airflow/providers/openai}/version_compat.py | 9 +-
providers/openfaas/pyproject.toml | 3 +-
providers/openlineage/pyproject.toml | 3 +-
.../providers/openlineage/extractors/base.py | 4 +-
.../providers/openlineage/extractors/python.py | 2 +-
.../providers/openlineage/plugins/listener.py | 38 +-
.../src/airflow/providers/openlineage/sqlparser.py | 3 +-
.../src/airflow/providers/openlineage/utils/sql.py | 4 +-
.../airflow/providers/openlineage/utils/utils.py | 39 +-
.../example_openlineage_base_complex_dag.py | 5 +-
.../tests/unit/openlineage/extractors/test_base.py | 8 +-
.../unit/openlineage/plugins/test_listener.py | 148 ++---
.../openlineage/utils/test_selective_enable.py | 2 +-
.../tests/unit/openlineage/utils/test_utils.py | 100 ++-
providers/opensearch/pyproject.toml | 3 +-
.../providers/opensearch/log/os_task_handler.py | 7 +-
.../providers/opensearch/operators/opensearch.py | 2 +-
.../airflow/providers/opensearch/version_compat.py | 10 +
.../unit/opensearch/log/test_os_json_formatter.py | 1 -
.../tests/unit/opensearch/log/test_os_response.py | 1 -
.../unit/opensearch/log/test_os_task_handler.py | 17 +-
.../unit/opensearch/operators/test_opensearch.py | 1 -
providers/opsgenie/pyproject.toml | 3 +-
.../providers/opsgenie/operators/opsgenie.py | 2 +-
.../airflow/providers/opsgenie}/version_compat.py | 9 +-
providers/oracle/pyproject.toml | 3 +-
.../airflow/providers/oracle/operators/oracle.py | 8 +-
.../providers/oracle/transfers/oracle_to_oracle.py | 8 +-
.../airflow/providers/oracle}/version_compat.py | 14 +
providers/pagerduty/pyproject.toml | 3 +-
.../airflow/providers/pagerduty/hooks/pagerduty.py | 2 +-
.../providers/pagerduty/hooks/pagerduty_events.py | 2 +-
providers/papermill/pyproject.toml | 3 +-
.../providers/papermill/operators/papermill.py | 4 +-
.../airflow/providers/papermill}/version_compat.py | 11 +
providers/pgvector/pyproject.toml | 3 +-
providers/pinecone/pyproject.toml | 3 +-
.../providers/pinecone/operators/pinecone.py | 5 +-
.../airflow/providers/pinecone}/version_compat.py | 11 +
providers/postgres/pyproject.toml | 3 +-
.../airflow/providers/postgres/hooks/postgres.py | 4 +-
providers/presto/pyproject.toml | 3 +-
.../providers/presto/transfers/gcs_to_presto.py | 2 +-
.../src/airflow/providers/presto/version_compat.py | 10 +
providers/qdrant/pyproject.toml | 3 +-
.../airflow/providers/qdrant/operators/qdrant.py | 2 +-
.../airflow/providers/qdrant}/version_compat.py | 8 +
providers/redis/pyproject.toml | 3 +-
.../airflow/providers/redis/sensors/redis_key.py | 7 +-
.../providers/redis/sensors/redis_pub_sub.py | 7 +-
providers/salesforce/pyproject.toml | 3 +-
.../airflow/providers/salesforce/operators/bulk.py | 11 +-
.../salesforce/operators/salesforce_apex_rest.py | 8 +-
.../providers/salesforce}/version_compat.py | 11 +
providers/samba/pyproject.toml | 3 +-
.../providers/samba/transfers/gcs_to_samba.py | 8 +-
.../src/airflow/providers/samba}/version_compat.py | 10 +
providers/segment/pyproject.toml | 7 +-
.../src/airflow/providers/segment/hooks/segment.py | 2 +-
.../segment/operators/segment_track_event.py | 8 +-
.../airflow/providers/segment}/version_compat.py | 15 +
providers/sendgrid/pyproject.toml | 3 +-
.../airflow/providers/sendgrid/utils/emailer.py | 3 +-
providers/sftp/pyproject.toml | 3 +-
.../providers/sftp/decorators/sensors/sftp.py | 3 +-
.../sftp/src/airflow/providers/sftp/hooks/sftp.py | 4 +-
.../src/airflow/providers/sftp/operators/sftp.py | 2 +-
.../src/airflow/providers/sftp/sensors/sftp.py | 6 +-
.../src/airflow/providers/sftp/version_compat.py | 8 +
providers/singularity/pyproject.toml | 3 +-
.../providers/singularity/operators/singularity.py | 4 +-
.../providers/singularity}/version_compat.py | 10 +
providers/slack/pyproject.toml | 3 +-
.../airflow/providers/slack/hooks/slack_webhook.py | 3 +-
.../src/airflow/providers/slack/operators/slack.py | 6 +-
.../providers/slack/operators/slack_webhook.py | 2 +-
.../providers/slack/transfers/base_sql_to_slack.py | 2 +-
.../providers/slack/transfers/sql_to_slack.py | 4 +-
.../src/airflow/providers/slack}/version_compat.py | 12 +-
.../slack/tests/unit/slack/hooks/test_slack.py | 6 +-
.../tests/unit/slack/hooks/test_slack_webhook.py | 6 +-
.../slack/transfers/test_sql_to_slack_webhook.py | 81 +--
providers/smtp/pyproject.toml | 3 +-
.../src/airflow/providers/smtp/operators/smtp.py | 8 +-
.../src/airflow/providers/smtp}/version_compat.py | 13 +
providers/snowflake/pyproject.toml | 6 +-
.../providers/snowflake/decorators/snowpark.py | 4 +-
.../airflow/providers/snowflake/hooks/snowflake.py | 36 +-
.../providers/snowflake/hooks/snowflake_sql_api.py | 81 ++-
.../providers/snowflake/operators/snowpark.py | 4 +-
.../providers/snowflake/utils/openlineage.py | 99 ++-
.../airflow/providers/snowflake/utils/snowpark.py | 3 +-
.../unit/snowflake/decorators/test_snowpark.py | 25 +-
.../tests/unit/snowflake/hooks/test_snowflake.py | 39 +-
.../unit/snowflake/hooks/test_snowflake_sql_api.py | 333 ++++++++--
.../unit/snowflake/operators/test_snowflake.py | 17 +-
.../unit/snowflake/operators/test_snowpark.py | 18 +-
.../transfers/test_copy_into_snowflake.py | 2 +-
.../tests/unit/snowflake/utils/test_openlineage.py | 708 ++++++++++++++++++++-
providers/sqlite/pyproject.toml | 3 +-
providers/ssh/pyproject.toml | 3 +-
.../ssh/src/airflow/providers/ssh/operators/ssh.py | 2 +-
.../src/airflow/providers/ssh}/version_compat.py | 9 +-
providers/ssh/tests/unit/ssh/operators/test_ssh.py | 7 +-
providers/standard/docs/operators/python.rst | 4 +
providers/standard/pyproject.toml | 3 +-
.../airflow/providers/standard/decorators/bash.py | 4 +-
.../standard/decorators/branch_external_python.py | 3 +-
.../providers/standard/decorators/branch_python.py | 3 +-
.../standard/decorators/branch_virtualenv.py | 3 +-
.../standard/decorators/external_python.py | 3 +-
.../providers/standard/decorators/python.py | 4 +-
.../standard/decorators/python_virtualenv.py | 3 +-
.../providers/standard/decorators/sensor.py | 4 +-
.../providers/standard/decorators/short_circuit.py | 3 +-
.../airflow/providers/standard/operators/bash.py | 7 +-
.../airflow/providers/standard/operators/branch.py | 5 +-
.../airflow/providers/standard/operators/empty.py | 2 +-
.../airflow/providers/standard/operators/python.py | 9 +-
.../airflow/providers/standard/operators/smooth.py | 2 +-
.../providers/standard/operators/trigger_dagrun.py | 5 +-
.../src/airflow/providers/standard/sensors/bash.py | 7 +-
.../providers/standard/sensors/date_time.py | 7 +-
.../providers/standard/sensors/external_task.py | 31 +-
.../providers/standard/sensors/filesystem.py | 7 +-
.../airflow/providers/standard/sensors/python.py | 11 +-
.../src/airflow/providers/standard/sensors/time.py | 7 +-
.../providers/standard/sensors/time_delta.py | 7 +-
.../airflow/providers/standard/sensors/weekday.py | 7 +-
.../providers/standard/utils/python_virtualenv.py | 11 +-
.../airflow/providers/standard/utils/skipmixin.py | 15 +-
.../airflow/providers/standard/version_compat.py | 27 +-
.../tests/unit/standard/decorators/test_bash.py | 4 +-
.../decorators/test_branch_external_python.py | 14 +-
.../unit/standard/decorators/test_branch_python.py | 14 +-
.../standard/decorators/test_branch_virtualenv.py | 14 +-
.../standard/decorators/test_external_python.py | 72 ++-
.../tests/unit/standard/decorators/test_python.py | 140 ++--
.../standard/decorators/test_python_virtualenv.py | 100 +--
.../unit/standard/decorators/test_short_circuit.py | 12 +-
.../tests/unit/standard/operators/test_bash.py | 12 +-
.../standard/operators/test_branch_operator.py | 41 +-
.../tests/unit/standard/operators/test_datetime.py | 29 +-
.../operators/test_latest_only_operator.py | 25 +-
.../tests/unit/standard/operators/test_python.py | 180 +++---
.../unit/standard/operators/test_trigger_dagrun.py | 4 +-
.../tests/unit/standard/operators/test_weekday.py | 14 +-
.../standard/sensors/test_external_task_sensor.py | 8 +-
.../tests/unit/standard/sensors/test_time_delta.py | 78 ++-
.../tests/unit/standard/sensors/test_weekday.py | 2 +-
.../tests/unit/standard/triggers/test_file.py | 3 +-
.../unit/standard/utils/test_python_virtualenv.py | 21 +-
providers/tableau/pyproject.toml | 3 +-
.../airflow/providers/tableau/sensors/tableau.py | 7 +-
.../airflow/providers/tableau}/version_compat.py | 0
providers/telegram/pyproject.toml | 3 +-
providers/teradata/pyproject.toml | 3 +-
.../src/airflow/providers/teradata/hooks/bteq.py | 18 +-
.../airflow/providers/teradata/operators/bteq.py | 2 -
.../airflow/providers/teradata/utils/bteq_util.py | 37 +-
.../tests/unit/teradata/hooks/test_bteq.py | 32 +-
.../tests/unit/teradata/operators/test_bteq.py | 4 +-
.../tests/unit/teradata/utils/test_bteq_util.py | 201 +++++-
providers/trino/pyproject.toml | 3 +-
.../providers/trino/transfers/gcs_to_trino.py | 2 +-
.../src/airflow/providers/trino/version_compat.py | 10 +
providers/vertica/pyproject.toml | 3 +-
.../src/airflow/providers/vertica/hooks/vertica.py | 4 +-
providers/weaviate/pyproject.toml | 3 +-
.../airflow/providers/weaviate/hooks/weaviate.py | 3 +-
.../providers/weaviate/operators/weaviate.py | 8 +-
.../airflow/providers/weaviate}/version_compat.py | 14 +
providers/yandex/pyproject.toml | 4 +-
.../src/airflow/providers/yandex/links/yq.py | 23 +-
.../airflow/providers/yandex/operators/dataproc.py | 8 +-
.../src/airflow/providers/yandex/operators/yq.py | 10 +-
.../airflow/providers/yandex}/version_compat.py | 25 +-
.../yandex/tests/unit/yandex/links/test_yq.py | 15 +-
.../yandex/tests/unit/yandex/operators/test_yq.py | 32 +-
providers/ydb/pyproject.toml | 3 +-
providers/zendesk/pyproject.toml | 3 +-
pyproject.toml | 10 +-
scripts/ci/airflow_version_check.py | 2 +-
scripts/ci/docker-compose/devcontainer.env | 7 +-
scripts/ci/docker-compose/devcontainer.yml | 2 +-
...ation-gremlin.yml => integration-tinkerpop.yml} | 0
scripts/ci/install_breeze.sh | 2 +-
.../pre_commit/check_airflow_v_imports_in_tests.py | 81 +++
scripts/ci/pre_commit/check_min_python_version.py | 2 +-
scripts/ci/pre_commit/common_precommit_utils.py | 6 +-
scripts/ci/pre_commit/mypy.py | 2 +-
scripts/ci/pre_commit/mypy_folder.py | 2 +-
scripts/ci/pre_commit/ts_compile_lint_ui.py | 2 +
.../ci/pre_commit/update_providers_dependencies.py | 2 +-
.../ci/pre_commit/validate_chart_annotations.py | 75 +++
scripts/ci/testing/summarize_captured_warnings.py | 4 +-
scripts/docker/entrypoint_ci.sh | 27 +-
scripts/in_container/check_environment.sh | 12 +-
.../in_container/install_airflow_and_providers.py | 351 +++++-----
.../install_development_dependencies.py | 2 +-
.../in_container/run_capture_airflowctl_help.py | 14 +-
scripts/in_container/run_generate_constraints.py | 4 +-
.../in_container/run_generate_openapi_spec_fab.py | 4 +-
.../in_container/run_provider_yaml_files_check.py | 4 +-
scripts/tools/setup_breeze | 2 +-
setup_idea.py | 4 +-
task-sdk/pyproject.toml | 11 +-
task-sdk/src/airflow/sdk/api/client.py | 2 +-
task-sdk/src/airflow/sdk/bases/decorator.py | 9 +-
task-sdk/src/airflow/sdk/bases/sensor.py | 4 +-
.../src/airflow/sdk/definitions/asset/__init__.py | 5 +-
task-sdk/src/airflow/sdk/definitions/dag.py | 15 +-
task-sdk/src/airflow/sdk/definitions/deadline.py | 3 +-
.../airflow/sdk/definitions/decorators/__init__.py | 2 +-
.../sdk/definitions/decorators/__init__.pyi | 4 +-
.../sdk/definitions/decorators/condition.py | 5 +-
.../sdk/definitions/decorators/setup_teardown.py | 3 +-
.../sdk/definitions/decorators/task_group.py | 16 +-
.../src/airflow/sdk/definitions/mappedoperator.py | 14 +-
task-sdk/src/airflow/sdk/definitions/xcom_arg.py | 4 +-
.../airflow/sdk/execution_time/callback_runner.py | 3 +-
task-sdk/src/airflow/sdk/execution_time/comms.py | 104 ++-
.../airflow/sdk/execution_time/secrets_masker.py | 22 +-
.../src/airflow/sdk/execution_time/supervisor.py | 13 +-
task-sdk/src/airflow/sdk/log.py | 3 +-
task-sdk/src/airflow/sdk/types.py | 4 +-
task-sdk/tests/task_sdk/api/test_client.py | 47 ++
task-sdk/tests/task_sdk/bases/test_operator.py | 16 +-
.../definitions/decorators/test_task_group.py | 52 +-
task-sdk/tests/task_sdk/definitions/test_asset.py | 6 +-
.../task_sdk/definitions/test_mappedoperator.py | 3 +-
.../tests/task_sdk/definitions/test_xcom_arg.py | 2 +-
1110 files changed, 14727 insertions(+), 7338 deletions(-)
delete mode 100644 airflow-core/src/airflow/ui/src/queries/useGrid.ts
copy airflow-core/src/airflow/ui/src/{context/timezone/TimezoneProvider.tsx => queries/useGridRuns.ts} (54%)
copy airflow-core/src/airflow/ui/src/{context/timezone/TimezoneProvider.tsx => queries/useGridStructure.ts} (52%)
copy airflow-core/src/airflow/ui/src/{pages/Run/AssetEvents.tsx => queries/useGridTISummaries.ts} (55%)
delete mode 100644 airflow-ctl/src/airflowctl/typing_compat.py
create mode 100644 dev/breeze/doc/images/output_run.svg
create mode 100644 dev/breeze/doc/images/output_run.txt
create mode 100755 dev/constraints-updated-version-check.py
copy providers/{apache/spark/src/airflow/providers/apache/spark => airbyte/src/airflow/providers/airbyte}/version_compat.py (78%)
copy providers/{celery/src/airflow/providers/celery => apache/beam/src/airflow/providers/apache/beam}/version_compat.py (78%)
copy {devel-common/src/tests_common/test_utils => providers/apache/cassandra/src/airflow/providers/apache/cassandra}/version_compat.py (87%)
copy providers/apache/{spark/src/airflow/providers/apache/spark => druid/src/airflow/providers/apache/druid}/version_compat.py (83%)
copy providers/apache/{spark/src/airflow/providers/apache/spark => flink/src/airflow/providers/apache/flink}/version_compat.py (78%)
copy {devel-common/src/tests_common/test_utils => providers/apache/hdfs/src/airflow/providers/apache/hdfs}/version_compat.py (87%)
copy providers/{alibaba/src/airflow/providers/alibaba => apache/hive/src/airflow/providers/apache/hive}/version_compat.py (70%)
copy providers/{alibaba/src/airflow/providers/alibaba => apache/impala/src/airflow/providers/apache/impala}/version_compat.py (100%)
copy {devel-common/src/tests_common/test_utils => providers/apache/kafka/src/airflow/providers/apache/kafka}/version_compat.py (89%)
copy {devel-common/src/tests_common/test_utils => providers/apache/kylin/src/airflow/providers/apache/kylin}/version_compat.py (88%)
copy providers/{amazon/src/airflow/providers/amazon => apache/livy/src/airflow/providers/apache/livy}/version_compat.py (84%)
copy {devel-common/src/tests_common/test_utils => providers/apache/pig/src/airflow/providers/apache/pig}/version_compat.py (89%)
copy providers/{celery/src/airflow/providers/celery => apache/tinkerpop/src/airflow/providers/apache/tinkerpop}/version_compat.py (87%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => arangodb/src/airflow/providers/arangodb}/version_compat.py (83%)
copy providers/{celery/src/airflow/providers/celery => asana/src/airflow/providers/asana}/version_compat.py (86%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => atlassian/jira/src/airflow/providers/atlassian/jira}/version_compat.py (83%)
copy providers/{celery/src/airflow/providers/celery => cohere/src/airflow/providers/cohere}/version_compat.py (79%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => common/sql/src/airflow/providers/common/sql}/version_compat.py (83%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => datadog/src/airflow/providers/datadog}/version_compat.py (80%)
copy providers/{celery/src/airflow/providers/celery => dingding/src/airflow/providers/dingding}/version_compat.py (76%)
copy {airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/public => providers/fab/src/airflow/providers/fab/www/static}/pin_100.png (100%)
copy {airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/public => providers/fab/src/airflow/providers/fab/www/static}/pin_32.png (100%)
copy providers/{alibaba/src/airflow/providers/alibaba => ftp/src/airflow/providers/ftp}/version_compat.py (100%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => github/src/airflow/providers/github}/version_compat.py (78%)
copy providers/{celery/src/airflow/providers/celery => grpc/src/airflow/providers/grpc}/version_compat.py (85%)
copy providers/{amazon/src/airflow/providers/amazon => http/src/airflow/providers/http}/version_compat.py (82%)
copy providers/{alibaba/src/airflow/providers/alibaba => imap/src/airflow/providers/imap}/version_compat.py (100%)
copy providers/{celery/src/airflow/providers/celery => influxdb/src/airflow/providers/influxdb}/version_compat.py (85%)
copy providers/{alibaba/src/airflow/providers/alibaba => jenkins/src/airflow/providers/jenkins}/version_compat.py (76%)
copy {devel-common/src/tests_common/test_utils => providers/mongo/src/airflow/providers/mongo}/version_compat.py (87%)
copy providers/{celery/src/airflow/providers/celery => mysql/src/airflow/providers/mysql}/version_compat.py (86%)
copy {devel-common/src/tests_common/test_utils => providers/neo4j/src/airflow/providers/neo4j}/version_compat.py (89%)
copy {devel-common/src/tests_common/test_utils => providers/openai/src/airflow/providers/openai}/version_compat.py (88%)
copy {devel-common/src/tests_common/test_utils => providers/opsgenie/src/airflow/providers/opsgenie}/version_compat.py (88%)
copy providers/{celery/src/airflow/providers/celery => oracle/src/airflow/providers/oracle}/version_compat.py (79%)
copy providers/{celery/src/airflow/providers/celery => papermill/src/airflow/providers/papermill}/version_compat.py (85%)
copy providers/{celery/src/airflow/providers/celery => pinecone/src/airflow/providers/pinecone}/version_compat.py (85%)
copy providers/{celery/src/airflow/providers/celery => qdrant/src/airflow/providers/qdrant}/version_compat.py (87%)
copy providers/{celery/src/airflow/providers/celery => salesforce/src/airflow/providers/salesforce}/version_compat.py (77%)
copy providers/{celery/src/airflow/providers/celery => samba/src/airflow/providers/samba}/version_compat.py (77%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => segment/src/airflow/providers/segment}/version_compat.py (78%)
copy providers/{celery/src/airflow/providers/celery => singularity/src/airflow/providers/singularity}/version_compat.py (77%)
copy {devel-common/src/tests_common/test_utils => providers/slack/src/airflow/providers/slack}/version_compat.py (89%)
copy providers/{apache/spark/src/airflow/providers/apache/spark => smtp/src/airflow/providers/smtp}/version_compat.py (81%)
copy {devel-common/src/tests_common/test_utils => providers/ssh/src/airflow/providers/ssh}/version_compat.py (88%)
copy providers/{alibaba/src/airflow/providers/alibaba => tableau/src/airflow/providers/tableau}/version_compat.py (100%)
copy providers/{celery/src/airflow/providers/celery => weaviate/src/airflow/providers/weaviate}/version_compat.py (79%)
copy providers/{alibaba/src/airflow/providers/alibaba => yandex/src/airflow/providers/yandex}/version_compat.py (69%)
rename scripts/ci/docker-compose/{integration-gremlin.yml => integration-tinkerpop.yml} (100%)
create mode 100755 scripts/ci/pre_commit/check_airflow_v_imports_in_tests.py
create mode 100755 scripts/ci/pre_commit/validate_chart_annotations.py
(airflow) 01/01: Switch all airflow logging to structlog
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch structlog-in-logging-mixin
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 516be1fae8ebb3295df6dbc49179ae6abe12edc8
Author: Ash Berlin-Taylor <as...@apache.org>
AuthorDate: Thu Mar 6 15:06:53 2025 +0000
Switch all airflow logging to structlog
This change continues the work that was started in Airflow 3.0 and AIP-72 to
use Structlog in place of stdlib logging in Airflow.
The primary change here is to make LoggingMixin return a customized
structlogger; customized to maintain surface compatability with logging.Logger
(surface meaning things like handlers and filters aren't preserved) and to
then capture/redirect all logging via stdlib via Structlog processors.
This is the first step in allowing all Airflow components to be able to
produce JSON logs natively.
---
.github/boring-cyborg.yml | 2 +-
airflow-core/pyproject.toml | 2 +
airflow-core/src/airflow/_logging/__init__.py | 16 +
airflow-core/src/airflow/_logging/structlog.py | 425 +++++++++++++++++++++
.../config_templates/airflow_local_settings.py | 3 +-
airflow-core/src/airflow/logging_config.py | 9 +-
airflow-core/src/airflow/utils/cli.py | 11 +
.../airflow/utils/log/file_processor_handler.py | 4 +-
.../src/airflow/utils/log/logging_mixin.py | 19 +-
airflow-core/src/airflow/utils/retries.py | 2 +-
.../tests/unit/always/test_providers_manager.py | 9 +-
.../tests/unit/cli/commands/test_task_command.py | 3 -
.../tests/unit/core/test_logging_config.py | 367 ------------------
airflow-core/tests/unit/core/test_stats.py | 8 +-
.../tests/unit/dag_processing/test_manager.py | 3 -
.../tests/unit/models/test_baseoperator.py | 10 -
.../tests/unit/utils/log/test_log_reader.py | 28 +-
airflow-core/tests/unit/utils/test_log_handlers.py | 6 +-
.../tests/unit/utils/test_logging_mixin.py | 12 -
.../tests/unit/utils/test_process_utils.py | 11 +-
.../test_task_handler_with_custom_formatter.py | 115 ------
devel-common/src/tests_common/pytest_plugin.py | 76 ++--
devel-common/src/tests_common/test_utils/logs.py | 217 +++++++++++
.../src/airflow/sdk/definitions/_internal/node.py | 10 +-
task-sdk/src/airflow/sdk/definitions/dag.py | 24 --
task-sdk/tests/task_sdk/bases/test_operator.py | 9 +
26 files changed, 776 insertions(+), 625 deletions(-)
diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml
index 9c5a437d3e0..e2fcfb75b43 100644
--- a/.github/boring-cyborg.yml
+++ b/.github/boring-cyborg.yml
@@ -408,7 +408,7 @@ labelPRBasedOnFilePath:
area:Logging:
- airflow-core/src/airflow/config_templates/airflow_local_settings.py
- - airflow-core/tests/unit/core/test_logging_config.py
+ - airflow-core/src/airflow/_logging/**/*
- airflow-core/src/airflow/utils/log/**/*
- airflow-core/docs/administration-and-deployment/logging-monitoring/logging-*.rst
- airflow-core/tests/unit/utils/log/**/*
diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml
index d5d03e4021c..4954f5ba054 100644
--- a/airflow-core/pyproject.toml
+++ b/airflow-core/pyproject.toml
@@ -118,6 +118,7 @@ dependencies = [
# See https://github.com/pygments/pygments/issues/2834
"pygments>=2.0.1,!=2.19.0",
"pyjwt>=2.10.0",
+ "pygtrie>=2.5.0",
"python-daemon>=3.0.0",
"python-dateutil>=2.7.0",
"python-slugify>=5.0",
@@ -133,6 +134,7 @@ dependencies = [
"sqlalchemy[asyncio]>=1.4.49,<2.0",
"sqlalchemy-jsonfield>=1.0",
"sqlalchemy-utils>=0.41.2",
+ "structlog>=25.4.0",
"svcs>=25.1.0",
"tabulate>=0.9.0",
"tenacity>=8.3.0",
diff --git a/airflow-core/src/airflow/_logging/__init__.py b/airflow-core/src/airflow/_logging/__init__.py
new file mode 100644
index 00000000000..13a83393a91
--- /dev/null
+++ b/airflow-core/src/airflow/_logging/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow-core/src/airflow/_logging/structlog.py b/airflow-core/src/airflow/_logging/structlog.py
new file mode 100644
index 00000000000..9ff0583db52
--- /dev/null
+++ b/airflow-core/src/airflow/_logging/structlog.py
@@ -0,0 +1,425 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import logging
+import os
+import warnings
+from functools import cache
+from typing import Any, BinaryIO, Callable, Generic, Protocol, TextIO, TypeVar, Union, cast
+
+import pygtrie
+import structlog
+from structlog.processors import NAME_TO_LEVEL
+from structlog.typing import (
+ BindableLogger,
+ EventDict,
+ FilteringBoundLogger,
+ WrappedLogger,
+)
+
+log = logging.getLogger(__name__)
+
+
+class AirflowFilteringBoundLogger(FilteringBoundLogger, Protocol):
+ def isEnabledFor(self, level: int): ...
+ def getEffectiveLevel(self) -> int: ...
+
+ name: str
+
+
+LEVEL_TO_FILTERING_LOGGER: dict[int, type[AirflowFilteringBoundLogger]] = {}
+
+
+def _make_airflow_structlogger(min_level):
+ # This uses https://github.com/hynek/structlog/blob/2f0cc42d/src/structlog/_native.py#L126
+ # as inspiration
+
+ LEVEL_TO_NAME = {v: k for k, v in NAME_TO_LEVEL.items()}
+
+ def isEnabledFor(self: Any, level):
+ return self.is_enabled_for(level)
+
+ def getEffectiveLevel(self: Any):
+ return self.get_effective_level()
+
+ @property
+ def name(self):
+ return self._logger.name
+
+ base = structlog.make_filtering_bound_logger(min_level)
+
+ cls = type(
+ f"AirflowBoundLoggerFilteringAt{LEVEL_TO_NAME.get(min_level, 'Notset').capitalize()}",
+ (base,),
+ {
+ "isEnabledFor": isEnabledFor,
+ "getEffectiveLevel": getEffectiveLevel,
+ "name": name,
+ },
+ )
+ LEVEL_TO_FILTERING_LOGGER[min_level] = cls
+ return cls
+
+
+AirflowBoundLoggerFilteringAtNotset = _make_airflow_structlogger(NAME_TO_LEVEL["notset"])
+AirflowBoundLoggerFilteringAtDebug = _make_airflow_structlogger(NAME_TO_LEVEL["debug"])
+AirflowBoundLoggerFilteringAtInfo = _make_airflow_structlogger(NAME_TO_LEVEL["info"])
+AirflowBoundLoggerFilteringAtWarning = _make_airflow_structlogger(NAME_TO_LEVEL["warning"])
+AirflowBoundLoggerFilteringAtError = _make_airflow_structlogger(NAME_TO_LEVEL["error"])
+AirflowBoundLoggerFilteringAtCritical = _make_airflow_structlogger(NAME_TO_LEVEL["critical"])
+
+# We use a trie structure (sometimes also called a "prefix tree") so that we can easily and quickly find the
+# most suitable log level to apply. This mirrors the logging level cascade behavior from stdlib logging,
+# without the complexity of multiple handlers etc
+PER_LOGGER_LEVELS = pygtrie.StringTrie(separator=".")
+PER_LOGGER_LEVELS.update(
+ {
+ # Top level logging default - changed to respect config in `configure_structlog`
+ "": NAME_TO_LEVEL["info"],
+ }
+)
+
+
+def make_filtering_logger() -> Callable[..., BindableLogger]:
+ def maker(logger: WrappedLogger, *args, **kwargs):
+ # If the logger is a NamedBytesLogger/NamedWriteLogger (an Airflow specific subclass) then
+ # look up the global per-logger config and redirect to a new class.
+
+ logger_name = kwargs.get("context", {}).get("logger_name")
+ if not logger_name and isinstance(logger, (NamedWriteLogger, NamedBytesLogger)):
+ logger_name = logger.name
+
+ if logger_name:
+ level = PER_LOGGER_LEVELS.longest_prefix(logger_name).get(PER_LOGGER_LEVELS[""])
+ else:
+ level = PER_LOGGER_LEVELS[""]
+ return LEVEL_TO_FILTERING_LOGGER[level](logger, *args, **kwargs) # type: ignore[call-arg]
+
+ return maker
+
+
+class NamedBytesLogger(structlog.BytesLogger):
+ __slots__ = ("name",)
+
+ def __init__(self, name: str | None = None, file: BinaryIO | None = None):
+ self.name = name
+ super().__init__(file)
+
+
+class NamedWriteLogger(structlog.WriteLogger):
+ __slots__ = ("name",)
+
+ def __init__(self, name: str | None = None, file: TextIO | None = None):
+ self.name = name
+ super().__init__(file)
+
+
+LogOutputType = TypeVar("LogOutputType", bound=Union[TextIO, BinaryIO])
+
+
+class LoggerFactory(Generic[LogOutputType]):
+ def __init__(
+ self,
+ cls: Callable[[str | None, LogOutputType | None], WrappedLogger],
+ io: LogOutputType | None = None,
+ ):
+ self.cls = cls
+ self.io = io
+
+ def __call__(self, logger_name: str | None = None, *args: Any) -> WrappedLogger:
+ return self.cls(logger_name, self.io)
+
+
+def logger_name(logger: Any, method_name: Any, event_dict: EventDict) -> EventDict:
+ if logger_name := (event_dict.pop("logger_name", None) or getattr(logger, "name", None)):
+ event_dict.setdefault("logger", logger_name)
+ return event_dict
+
+
+# `eyJ` is `{"` in base64 encoding -- and any value that starts like that is in high likely hood a JWT
+# token. Better safe than sorry
+def redact_jwt(logger: Any, method_name: str, event_dict: EventDict) -> EventDict:
+ for k, v in event_dict.items():
+ if isinstance(v, str) and v.startswith("eyJ"):
+ event_dict[k] = "eyJ***"
+ return event_dict
+
+
+def drop_positional_args(logger: Any, method_name: Any, event_dict: EventDict) -> EventDict:
+ event_dict.pop("positional_args", None)
+ return event_dict
+
+
+# This is a placeholder fn, that is "edited" in place via the `suppress_logs_and_warning` decorator
+# The reason we need to do it this way is that structlog caches loggers on first use, and those include the
+# configured processors, so we can't get away with changing the config as it won't have any effect once the
+# logger obj is created and has been used once
+def respect_stdlib_disable(logger: Any, method_name: Any, event_dict: EventDict) -> EventDict:
+ return event_dict
+
+
+@cache
+def structlog_processors(
+ json_output: bool,
+):
+ if json_output:
+ timestamper = structlog.processors.MaybeTimeStamper(fmt="iso")
+ else:
+ timestamper = structlog.processors.MaybeTimeStamper(fmt="%Y-%m-%d %H:%M:%S.%f")
+
+ processors: list[structlog.typing.Processor] = [
+ respect_stdlib_disable,
+ timestamper,
+ structlog.contextvars.merge_contextvars,
+ structlog.processors.add_log_level,
+ structlog.stdlib.PositionalArgumentsFormatter(),
+ logger_name,
+ redact_jwt,
+ structlog.processors.StackInfoRenderer(),
+ ]
+
+ # Imports to suppress showing code from these modules. We need the import to get the filepath for
+ # structlog to ignore.
+ import contextlib
+
+ import click
+ import httpcore
+ import httpx
+
+ suppress = (
+ click,
+ contextlib,
+ httpx,
+ httpcore,
+ httpx,
+ )
+
+ if json_output:
+ dict_exc_formatter = structlog.tracebacks.ExceptionDictTransformer(
+ use_rich=False, show_locals=False, suppress=suppress
+ )
+
+ dict_tracebacks = structlog.processors.ExceptionRenderer(dict_exc_formatter)
+
+ import msgspec
+
+ def json_dumps(msg, default):
+ # Note: this is likely an "expensive" step, but lets massage the dict order for nice
+ # viewing of the raw JSON logs.
+ # Maybe we don't need this once the UI renders the JSON instead of displaying the raw text
+ msg = {
+ "timestamp": msg.pop("timestamp"),
+ "level": msg.pop("level"),
+ "event": msg.pop("event"),
+ **msg,
+ }
+ return msgspec.json.encode(msg, enc_hook=default)
+
+ def json_processor(logger: Any, method_name: Any, event_dict: EventDict) -> str:
+ # Stdlib logging doesn't need the re-ordering, it's fine as it is
+ return msgspec.json.encode(event_dict).decode("utf-8")
+
+ json = structlog.processors.JSONRenderer(serializer=json_dumps)
+
+ processors.extend(
+ (
+ dict_tracebacks,
+ structlog.processors.UnicodeDecoder(),
+ json,
+ ),
+ )
+
+ return processors, {
+ "timestamper": timestamper,
+ "dict_tracebacks": dict_tracebacks,
+ "json": json_processor,
+ }
+ rich_exc_formatter = structlog.dev.RichTracebackFormatter(
+ # These values are picked somewhat arbitrarily to produce useful-but-compact tracebacks. If
+ # we ever need to change these then they should be configurable.
+ extra_lines=0,
+ max_frames=30,
+ indent_guides=False,
+ suppress=suppress,
+ )
+ my_styles = structlog.dev.ConsoleRenderer.get_default_level_styles()
+ my_styles["debug"] = structlog.dev.CYAN
+
+ console = structlog.dev.ConsoleRenderer(exception_formatter=rich_exc_formatter, level_styles=my_styles)
+ processors.append(console)
+ return processors, {
+ "timestamper": timestamper,
+ "console": console,
+ }
+
+
+def configure_structlog(
+ json_output: bool = False,
+ log_level: str = "DEBUG",
+ cache_logger_on_first_use: bool = True,
+ stdlib_config: dict | None = None,
+):
+ if "fatal" not in NAME_TO_LEVEL:
+ NAME_TO_LEVEL["fatal"] = NAME_TO_LEVEL["critical"]
+
+ stdlib_config = stdlib_config or {}
+
+ """Set up struct logging and stdlib logging config."""
+ PER_LOGGER_LEVELS[""] = NAME_TO_LEVEL[log_level.lower()]
+
+ if json_output:
+ formatter = "plain"
+ else:
+ formatter = "colored"
+ processors, named = structlog_processors(json_output)
+ timestamper = named["timestamper"]
+
+ pre_chain: list[structlog.typing.Processor] = [
+ # Add the log level and a timestamp to the event_dict if the log entry
+ # is not from structlog.
+ structlog.stdlib.add_log_level,
+ structlog.stdlib.add_logger_name,
+ timestamper,
+ structlog.contextvars.merge_contextvars,
+ redact_jwt,
+ ]
+
+ # Don't cache the loggers during tests, it make it hard to capture them
+ if "PYTEST_VERSION" in os.environ:
+ cache_logger_on_first_use = False
+
+ color_formatter: list[structlog.typing.Processor] = [
+ structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+ drop_positional_args,
+ ]
+ std_lib_formatter: list[structlog.typing.Processor] = [
+ structlog.stdlib.ProcessorFormatter.remove_processors_meta,
+ drop_positional_args,
+ ]
+
+ wrapper_class = cast("type[BindableLogger]", make_filtering_logger())
+ if json_output:
+ structlog.configure(
+ processors=processors,
+ cache_logger_on_first_use=cache_logger_on_first_use,
+ wrapper_class=wrapper_class,
+ logger_factory=LoggerFactory(NamedBytesLogger), # type: ignore[type-var]
+ )
+
+ pre_chain.append(named["dict_tracebacks"])
+ color_formatter.append(named["json"])
+ std_lib_formatter.append(named["json"])
+ else:
+ structlog.configure(
+ processors=processors,
+ cache_logger_on_first_use=cache_logger_on_first_use,
+ wrapper_class=wrapper_class,
+ logger_factory=LoggerFactory(NamedWriteLogger), # type: ignore[type-var]
+ )
+ color_formatter.append(named["console"])
+
+ global _warnings_showwarning
+
+ if _warnings_showwarning is None:
+ _warnings_showwarning = warnings.showwarning
+ # Capture warnings and show them via structlog
+ warnings.showwarning = _showwarning
+
+ import logging.config
+
+ config = {**stdlib_config}
+ config["formatters"] = {**config["formatters"]}
+ config["handlers"] = {**config["handlers"]}
+ config["loggers"] = {**config["loggers"]}
+ config["formatters"].update(
+ {
+ "plain": {
+ "()": structlog.stdlib.ProcessorFormatter,
+ "processors": std_lib_formatter,
+ "foreign_pre_chain": pre_chain,
+ "pass_foreign_args": True,
+ },
+ "colored": {
+ "()": structlog.stdlib.ProcessorFormatter,
+ "processors": color_formatter,
+ "foreign_pre_chain": pre_chain,
+ "pass_foreign_args": True,
+ },
+ }
+ )
+ for section in (config["loggers"], config["handlers"]):
+ for log_config in section.values():
+ # We want everything to go via structlog, remove whatever the user might have configured
+ log_config.pop("stream", None)
+ log_config.pop("formatter", None)
+ # log_config.pop("handlers", None)
+
+ config["handlers"].update(
+ {
+ "default": {
+ "level": log_level.upper(),
+ "class": "logging.StreamHandler",
+ "formatter": formatter,
+ },
+ }
+ )
+ config["loggers"].update(
+ {
+ # Set Airflow logging to the level requested, but most everything else at "INFO"
+ "airflow": {"level": log_level.upper()},
+ # These ones are too chatty even at info
+ "httpx": {"level": "WARN"},
+ "sqlalchemy.engine": {"level": "WARN"},
+ }
+ )
+ config["root"] = {
+ "handlers": ["default"],
+ "level": "INFO",
+ "propagate": True,
+ }
+
+ logging.config.dictConfig(config)
+
+
+_warnings_showwarning: Any = None
+
+
+def _showwarning(
+ message: Warning | str,
+ category: type[Warning],
+ filename: str,
+ lineno: int,
+ file: TextIO | None = None,
+ line: str | None = None,
+) -> Any:
+ """
+ Redirects warnings to structlog so they appear in task logs etc.
+
+ Implementation of showwarnings which redirects to logging, which will first
+ check to see if the file parameter is None. If a file is specified, it will
+ delegate to the original warnings implementation of showwarning. Otherwise,
+ it will call warnings.formatwarning and will log the resulting string to a
+ warnings logger named "py.warnings" with level logging.WARNING.
+ """
+ if file is not None:
+ if _warnings_showwarning is not None:
+ _warnings_showwarning(message, category, filename, lineno, file, line)
+ else:
+ log = structlog.get_logger("py.warnings")
+ log.warning(str(message), category=category.__name__, filename=filename, lineno=lineno)
diff --git a/airflow-core/src/airflow/config_templates/airflow_local_settings.py b/airflow-core/src/airflow/config_templates/airflow_local_settings.py
index cb1cc364e2c..4bbd849e083 100644
--- a/airflow-core/src/airflow/config_templates/airflow_local_settings.py
+++ b/airflow-core/src/airflow/config_templates/airflow_local_settings.py
@@ -77,7 +77,8 @@ DEFAULT_LOGGING_CONFIG: dict[str, Any] = {
},
"handlers": {
"console": {
- "class": "airflow.utils.log.logging_mixin.RedirectStdHandler",
+ "class": "logging.StreamHandler",
+ # "class": "airflow.utils.log.logging_mixin.RedirectStdHandler",
"formatter": "airflow_coloured",
"stream": "sys.stdout",
"filters": ["mask_secrets"],
diff --git a/airflow-core/src/airflow/logging_config.py b/airflow-core/src/airflow/logging_config.py
index b0c0b355155..d0f3a4aa066 100644
--- a/airflow-core/src/airflow/logging_config.py
+++ b/airflow-core/src/airflow/logging_config.py
@@ -19,9 +19,9 @@ from __future__ import annotations
import logging
import warnings
-from logging.config import dictConfig
from typing import TYPE_CHECKING, Any
+from airflow._logging.structlog import configure_structlog
from airflow.configuration import conf
from airflow.exceptions import AirflowConfigException
from airflow.utils.module_loading import import_string
@@ -94,20 +94,21 @@ def configure_logging():
if "mask_secrets" not in task_handler_config["filters"]:
task_handler_config["filters"].append("mask_secrets")
+ level: str = conf.get_mandatory_value("logging", "LOGGING_LEVEL").upper()
# Try to init logging
- dictConfig(logging_config)
+ configure_structlog(log_level=level, stdlib_config=logging_config)
except (ValueError, KeyError) as e:
log.error("Unable to load the config, contains a configuration error.")
# When there is an error in the config, escalate the exception
# otherwise Airflow would silently fall back on the default config
raise e
- validate_logging_config(logging_config)
+ validate_logging_config()
return logging_class_path
-def validate_logging_config(logging_config):
+def validate_logging_config():
"""Validate the provided Logging Config."""
# Now lets validate the other logging-related settings
task_log_reader = conf.get("logging", "task_log_reader")
diff --git a/airflow-core/src/airflow/utils/cli.py b/airflow-core/src/airflow/utils/cli.py
index ac7723d2933..390a3d90474 100644
--- a/airflow-core/src/airflow/utils/cli.py
+++ b/airflow-core/src/airflow/utils/cli.py
@@ -426,15 +426,26 @@ def suppress_logs_and_warning(f: T) -> T:
if args[0].verbose:
f(*args, **kwargs)
else:
+ from airflow._logging.structlog import respect_stdlib_disable
+
with warnings.catch_warnings():
warnings.simplefilter("ignore")
logging.disable(logging.CRITICAL)
+
+ def drop(*_, **__):
+ from structlog import DropEvent
+
+ raise DropEvent()
+
+ old_fn = respect_stdlib_disable.__code__
+ respect_stdlib_disable.__code__ = drop.__code__
try:
f(*args, **kwargs)
finally:
# logging output again depends on the effective
# levels of individual loggers
logging.disable(logging.NOTSET)
+ respect_stdlib_disable.__code__ = old_fn
return cast("T", _wrapper)
diff --git a/airflow-core/src/airflow/utils/log/file_processor_handler.py b/airflow-core/src/airflow/utils/log/file_processor_handler.py
index 5dbd034cea8..cf0cec6ed83 100644
--- a/airflow-core/src/airflow/utils/log/file_processor_handler.py
+++ b/airflow-core/src/airflow/utils/log/file_processor_handler.py
@@ -25,7 +25,7 @@ from pathlib import Path
from airflow import settings
from airflow.utils import timezone
from airflow.utils.helpers import parse_template_string
-from airflow.utils.log.logging_mixin import DISABLE_PROPOGATE
+from airflow.utils.log.logging_mixin import SetContextPropagate
from airflow.utils.log.non_caching_file_handler import NonCachingFileHandler
logger = logging.getLogger(__name__)
@@ -69,7 +69,7 @@ class FileProcessorHandler(logging.Handler):
self._symlink_latest_log_directory()
self._cur_date = datetime.today()
- return DISABLE_PROPOGATE
+ return SetContextPropagate.DISABLE_PROPAGATE
def emit(self, record):
if self.handler is not None:
diff --git a/airflow-core/src/airflow/utils/log/logging_mixin.py b/airflow-core/src/airflow/utils/log/logging_mixin.py
index 1b3e24442df..c624d7a8992 100644
--- a/airflow-core/src/airflow/utils/log/logging_mixin.py
+++ b/airflow-core/src/airflow/utils/log/logging_mixin.py
@@ -19,15 +19,16 @@ from __future__ import annotations
import abc
import enum
-import logging
import re
import sys
from io import TextIOBase, UnsupportedOperation
from logging import Handler, StreamHandler
from typing import IO, TYPE_CHECKING, Any, TypeVar, cast
+import structlog
+
if TYPE_CHECKING:
- from logging import Logger
+ from airflow._logging.structlog import AirflowFilteringBoundLogger
# 7-bit C1 ANSI escape sequences
ANSI_ESCAPE = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]")
@@ -67,7 +68,7 @@ _T = TypeVar("_T")
class LoggingMixin:
"""Convenience super-class to have a logger configured with the class name."""
- _log: logging.Logger | None = None
+ _log: AirflowFilteringBoundLogger | None = None
# Parent logger used by this class. It should match one of the loggers defined in the
# `logging_config_class`. By default, this attribute is used to create the final name of the logger, and
@@ -104,29 +105,27 @@ class LoggingMixin:
return logger_name
@classmethod
- def _get_log(cls, obj: Any, clazz: type[_T]) -> Logger:
+ def _get_log(cls, obj: Any, clazz: type[_T]) -> AirflowFilteringBoundLogger:
if obj._log is None:
logger_name: str = cls._create_logger_name(
logged_class=clazz,
log_config_logger_name=obj._log_config_logger_name,
class_logger_name=obj._logger_name,
)
- obj._log = logging.getLogger(logger_name)
+ obj._log = structlog.get_logger(logger_name)
return obj._log
@classmethod
- def logger(cls) -> Logger:
+ def logger(cls) -> AirflowFilteringBoundLogger:
"""Return a logger."""
return LoggingMixin._get_log(cls, cls)
@property
- def log(self) -> Logger:
+ def log(self) -> AirflowFilteringBoundLogger:
"""Return a logger."""
return LoggingMixin._get_log(self, self.__class__)
- def _set_context(self, context):
- if context is not None:
- set_context(self.log, context)
+ def _set_context(self, context): ...
class ExternalLoggingMixin:
diff --git a/airflow-core/src/airflow/utils/retries.py b/airflow-core/src/airflow/utils/retries.py
index 4428db2a493..f84a16f6bde 100644
--- a/airflow-core/src/airflow/utils/retries.py
+++ b/airflow-core/src/airflow/utils/retries.py
@@ -43,7 +43,7 @@ def run_with_db_retries(max_retries: int = MAX_DB_RETRIES, logger: logging.Logge
reraise=True,
**kwargs,
)
- if logger and isinstance(logger, logging.Logger):
+ if logger is not None:
retry_kwargs["before_sleep"] = tenacity.before_sleep_log(logger, logging.DEBUG, True)
return tenacity.Retrying(**retry_kwargs)
diff --git a/airflow-core/tests/unit/always/test_providers_manager.py b/airflow-core/tests/unit/always/test_providers_manager.py
index d0a2037f9bd..8f382f5b3ad 100644
--- a/airflow-core/tests/unit/always/test_providers_manager.py
+++ b/airflow-core/tests/unit/always/test_providers_manager.py
@@ -117,8 +117,8 @@ class TestProviderManager:
)
providers_manager._discover_hooks()
_ = providers_manager._hooks_lazy_dict["wrong-connection-type"]
- assert len(self._caplog.records) == 1
- assert "Inconsistency!" in self._caplog.records[0].message
+ assert len(self._caplog.entries) == 1
+ assert "Inconsistency!" in self._caplog[0]["event"]
assert "sftp" not in providers_manager.hooks
def test_warning_logs_not_generated(self):
@@ -162,11 +162,12 @@ class TestProviderManager:
providers_manager._discover_hooks()
_ = providers_manager._hooks_lazy_dict["dummy"]
assert len(self._caplog.records) == 1
- assert "The connection type 'dummy' is already registered" in self._caplog.records[0].message
+ msg = self._caplog.messages[0]
+ assert msg.startswith("The connection type 'dummy' is already registered")
assert (
"different class names: 'airflow.providers.dummy.hooks.dummy.DummyHook'"
" and 'airflow.providers.dummy.hooks.dummy.DummyHook2'."
- ) in self._caplog.records[0].message
+ ) in msg
def test_providers_manager_register_plugins(self):
providers_manager = ProvidersManager()
diff --git a/airflow-core/tests/unit/cli/commands/test_task_command.py b/airflow-core/tests/unit/cli/commands/test_task_command.py
index faad02503e4..04ae1540bef 100644
--- a/airflow-core/tests/unit/cli/commands/test_task_command.py
+++ b/airflow-core/tests/unit/cli/commands/test_task_command.py
@@ -21,7 +21,6 @@ import contextlib
import io
import json
import logging
-import logging.config
import os
import shutil
from argparse import ArgumentParser
@@ -34,7 +33,6 @@ import pytest
from airflow.cli import cli_parser
from airflow.cli.commands import task_command
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
from airflow.configuration import conf
from airflow.exceptions import DagRunNotFound
from airflow.models import DagBag, DagRun, TaskInstance
@@ -84,7 +82,6 @@ class TestCliTasks:
@classmethod
@pytest.fixture(autouse=True)
def setup_class(cls):
- logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
parse_and_sync_to_db(os.devnull, include_examples=True)
cls.parser = cli_parser.get_parser()
clear_db_runs()
diff --git a/airflow-core/tests/unit/core/test_logging_config.py b/airflow-core/tests/unit/core/test_logging_config.py
deleted file mode 100644
index 72864b066b3..00000000000
--- a/airflow-core/tests/unit/core/test_logging_config.py
+++ /dev/null
@@ -1,367 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import contextlib
-import importlib
-import logging
-import os
-import pathlib
-import sys
-import tempfile
-from unittest.mock import patch
-
-import pytest
-
-from airflow.configuration import conf
-
-from tests_common.test_utils.config import conf_vars
-from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker
-
-pytestmark = skip_if_force_lowest_dependencies_marker
-
-SETTINGS_FILE_VALID = """
-LOGGING_CONFIG = {
- 'version': 1,
- 'disable_existing_loggers': False,
- 'formatters': {
- 'airflow.task': {
- 'format': '[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s'
- },
- },
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'formatter': 'airflow.task',
- 'stream': 'ext://sys.stdout'
- },
- 'task': {
- 'class': 'logging.StreamHandler',
- 'formatter': 'airflow.task',
- 'stream': 'ext://sys.stdout'
- },
- },
- 'loggers': {
- 'airflow.task': {
- 'handlers': ['task'],
- 'level': 'INFO',
- 'propagate': False,
- },
- }
-}
-"""
-
-SETTINGS_FILE_INVALID = """
-LOGGING_CONFIG = {
- 'version': 1,
- 'disable_existing_loggers': False,
- 'formatters': {
- 'airflow.task': {
- 'format': '[%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s'
- },
- },
- 'handlers': {
- 'console': {
- 'class': 'logging.StreamHandler',
- 'formatter': 'airflow.task',
- 'stream': 'ext://sys.stdout'
- }
- },
- 'loggers': {
- 'airflow': {
- 'handlers': ['file.handler'], # this handler does not exists
- 'level': 'INFO',
- 'propagate': False
- }
- }
-}
-"""
-
-SETTINGS_FILE_EMPTY = """
-# Other settings here
-"""
-
-SETTINGS_DEFAULT_NAME = "custom_airflow_local_settings"
-
-
-def reset_logging():
- """Reset Logging"""
- manager = logging.root.manager
- manager.disabled = logging.NOTSET
- airflow_loggers = [
- logger for logger_name, logger in manager.loggerDict.items() if logger_name.startswith("airflow")
- ]
- for logger in airflow_loggers:
- if isinstance(logger, logging.Logger):
- logger.setLevel(logging.NOTSET)
- logger.propagate = True
- logger.disabled = False
- logger.filters.clear()
- handlers = logger.handlers.copy()
- for handler in handlers:
- # Copied from `logging.shutdown`.
- try:
- handler.acquire()
- handler.flush()
- handler.close()
- except (OSError, ValueError):
- pass
- finally:
- handler.release()
- logger.removeHandler(handler)
-
-
-@contextlib.contextmanager
-def settings_context(content, directory=None, name="LOGGING_CONFIG"):
- """
- Sets a settings file and puts it in the Python classpath
-
- :param content:
- The content of the settings file
- :param directory: the directory
- :param name: str
- """
- initial_logging_config = os.environ.get("AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS", "")
- try:
- settings_root = tempfile.mkdtemp()
- filename = f"{SETTINGS_DEFAULT_NAME}.py"
- if directory:
- # Create the directory structure with __init__.py
- dir_path = os.path.join(settings_root, directory)
- pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
-
- basedir = settings_root
- for part in directory.split("/"):
- open(os.path.join(basedir, "__init__.py"), "w").close()
- basedir = os.path.join(basedir, part)
- open(os.path.join(basedir, "__init__.py"), "w").close()
-
- # Replace slashes by dots
- module = directory.replace("/", ".") + "." + SETTINGS_DEFAULT_NAME + "." + name
- settings_file = os.path.join(dir_path, filename)
- else:
- module = SETTINGS_DEFAULT_NAME + "." + name
- settings_file = os.path.join(settings_root, filename)
-
- with open(settings_file, "w") as handle:
- handle.writelines(content)
- sys.path.append(settings_root)
-
- # Using environment vars instead of conf_vars so value is accessible
- # to parent and child processes when using 'spawn' for multiprocessing.
- os.environ["AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS"] = module
- yield settings_file
-
- finally:
- os.environ["AIRFLOW__LOGGING__LOGGING_CONFIG_CLASS"] = initial_logging_config
- sys.path.remove(settings_root)
-
-
-class TestLoggingSettings:
- # Make sure that the configure_logging is not cached
- def setup_method(self):
- self.old_modules = dict(sys.modules)
-
- def teardown_method(self):
- # Remove any new modules imported during the test run. This lets us
- # import the same source files for more than one test.
- from airflow.config_templates import airflow_local_settings
- from airflow.logging_config import configure_logging
-
- for mod in list(sys.modules):
- if mod not in self.old_modules:
- del sys.modules[mod]
-
- reset_logging()
- importlib.reload(airflow_local_settings)
- configure_logging()
-
- # When we try to load an invalid config file, we expect an error
- def test_loading_invalid_local_settings(self):
- from airflow.logging_config import configure_logging, log
-
- with settings_context(SETTINGS_FILE_INVALID):
- with patch.object(log, "error") as mock_info:
- # Load config
- with pytest.raises(ValueError):
- configure_logging()
-
- mock_info.assert_called_once_with(
- "Unable to load the config, contains a configuration error."
- )
-
- def test_loading_valid_complex_local_settings(self):
- # Test what happens when the config is somewhere in a subfolder
- module_structure = "etc.airflow.config"
- dir_structure = module_structure.replace(".", "/")
- with settings_context(SETTINGS_FILE_VALID, dir_structure):
- from airflow.logging_config import configure_logging, log
-
- with patch.object(log, "info") as mock_info:
- configure_logging()
- mock_info.assert_any_call(
- "Successfully imported user-defined logging config from %s",
- f"etc.airflow.config.{SETTINGS_DEFAULT_NAME}.LOGGING_CONFIG",
- )
-
- # When we load an empty file, it should go to default
- def test_loading_no_local_settings(self):
- with settings_context(SETTINGS_FILE_EMPTY):
- from airflow.logging_config import configure_logging
-
- with pytest.raises(ImportError):
- configure_logging()
-
- def test_1_9_config(self):
- from airflow.logging_config import configure_logging
-
- with conf_vars({("logging", "task_log_reader"): "file.task"}):
- with pytest.warns(DeprecationWarning, match=r"file.task"):
- configure_logging()
- assert conf.get("logging", "task_log_reader") == "task"
-
- def test_loading_remote_logging_with_wasb_handler(self):
- """Test if logging can be configured successfully for Azure Blob Storage"""
- pytest.importorskip(
- "airflow.providers.microsoft.azure", reason="'microsoft.azure' provider not installed"
- )
- import airflow.logging_config
- from airflow.config_templates import airflow_local_settings
- from airflow.providers.microsoft.azure.log.wasb_task_handler import WasbRemoteLogIO
-
- with conf_vars(
- {
- ("logging", "remote_logging"): "True",
- ("logging", "remote_log_conn_id"): "some_wasb",
- ("logging", "remote_base_log_folder"): "wasb://some-folder",
- }
- ):
- importlib.reload(airflow_local_settings)
- airflow.logging_config.configure_logging()
-
- assert isinstance(airflow.logging_config.REMOTE_TASK_LOG, WasbRemoteLogIO)
-
- @pytest.mark.parametrize(
- "remote_base_log_folder, log_group_arn",
- [
- (
- "cloudwatch://arn:aws:logs:aaaa:bbbbb:log-group:ccccc",
- "arn:aws:logs:aaaa:bbbbb:log-group:ccccc",
- ),
- (
- "cloudwatch://arn:aws:logs:aaaa:bbbbb:log-group:aws/ccccc",
- "arn:aws:logs:aaaa:bbbbb:log-group:aws/ccccc",
- ),
- (
- "cloudwatch://arn:aws:logs:aaaa:bbbbb:log-group:/aws/ecs/ccccc",
- "arn:aws:logs:aaaa:bbbbb:log-group:/aws/ecs/ccccc",
- ),
- ],
- )
- def test_log_group_arns_remote_logging_with_cloudwatch_handler(
- self, remote_base_log_folder, log_group_arn
- ):
- """Test if the correct ARNs are configured for Cloudwatch"""
- import airflow.logging_config
- from airflow.config_templates import airflow_local_settings
- from airflow.providers.amazon.aws.log.cloudwatch_task_handler import CloudWatchRemoteLogIO
-
- with conf_vars(
- {
- ("logging", "remote_logging"): "True",
- ("logging", "remote_log_conn_id"): "some_cloudwatch",
- ("logging", "remote_base_log_folder"): remote_base_log_folder,
- }
- ):
- importlib.reload(airflow_local_settings)
- airflow.logging_config.configure_logging()
-
- remote_io = airflow.logging_config.REMOTE_TASK_LOG
- assert isinstance(remote_io, CloudWatchRemoteLogIO)
- assert remote_io.log_group_arn == log_group_arn
-
- def test_loading_remote_logging_with_gcs_handler(self):
- """Test if logging can be configured successfully for GCS"""
- import airflow.logging_config
- from airflow.config_templates import airflow_local_settings
- from airflow.providers.google.cloud.log.gcs_task_handler import GCSRemoteLogIO
-
- with conf_vars(
- {
- ("logging", "remote_logging"): "True",
- ("logging", "remote_log_conn_id"): "some_gcs",
- ("logging", "remote_base_log_folder"): "gs://some-folder",
- ("logging", "google_key_path"): "/gcs-key.json",
- (
- "logging",
- "remote_task_handler_kwargs",
- ): '{"delete_local_copy": true, "project_id": "test-project", "gcp_keyfile_dict": {},"scopes": ["https://www.googleapis.com/auth/devstorage.read_write"]}',
- }
- ):
- importlib.reload(airflow_local_settings)
- airflow.logging_config.configure_logging()
-
- assert isinstance(airflow.logging_config.REMOTE_TASK_LOG, GCSRemoteLogIO)
- assert getattr(airflow.logging_config.REMOTE_TASK_LOG, "delete_local_copy") is True
- assert getattr(airflow.logging_config.REMOTE_TASK_LOG, "project_id") == "test-project"
- assert getattr(airflow.logging_config.REMOTE_TASK_LOG, "gcp_keyfile_dict") == {}
- assert getattr(airflow.logging_config.REMOTE_TASK_LOG, "scopes") == [
- "https://www.googleapis.com/auth/devstorage.read_write"
- ]
- assert getattr(airflow.logging_config.REMOTE_TASK_LOG, "gcp_key_path") == "/gcs-key.json"
-
- def test_loading_remote_logging_with_kwargs(self):
- """Test if logging can be configured successfully with kwargs"""
- pytest.importorskip("airflow.providers.amazon", reason="'amazon' provider not installed")
- import airflow.logging_config
- from airflow.config_templates import airflow_local_settings
- from airflow.providers.amazon.aws.log.s3_task_handler import S3RemoteLogIO
-
- with conf_vars(
- {
- ("logging", "remote_logging"): "True",
- ("logging", "remote_log_conn_id"): "some_s3",
- ("logging", "remote_base_log_folder"): "s3://some-folder",
- ("logging", "remote_task_handler_kwargs"): '{"delete_local_copy": true}',
- }
- ):
- importlib.reload(airflow_local_settings)
- airflow.logging_config.configure_logging()
-
- task_log = airflow.logging_config.REMOTE_TASK_LOG
- assert isinstance(task_log, S3RemoteLogIO)
- assert getattr(task_log, "delete_local_copy") is True
-
- def test_loading_remote_logging_with_hdfs_handler(self):
- """Test if logging can be configured successfully for HDFS"""
- pytest.importorskip("airflow.providers.apache.hdfs", reason="'apache.hdfs' provider not installed")
- import airflow.logging_config
- from airflow.config_templates import airflow_local_settings
- from airflow.providers.apache.hdfs.log.hdfs_task_handler import HdfsRemoteLogIO
-
- with conf_vars(
- {
- ("logging", "remote_logging"): "True",
- ("logging", "remote_log_conn_id"): "some_hdfs",
- ("logging", "remote_base_log_folder"): "hdfs://some-folder",
- }
- ):
- importlib.reload(airflow_local_settings)
- airflow.logging_config.configure_logging()
-
- assert isinstance(airflow.logging_config.REMOTE_TASK_LOG, HdfsRemoteLogIO)
diff --git a/airflow-core/tests/unit/core/test_stats.py b/airflow-core/tests/unit/core/test_stats.py
index 10752d16fba..9c5b96289bf 100644
--- a/airflow-core/tests/unit/core/test_stats.py
+++ b/airflow-core/tests/unit/core/test_stats.py
@@ -376,11 +376,11 @@ class TestPatternValidatorConfigOption:
@conf_vars({**stats_on, **block_list, ("metrics", "metrics_allow_list"): "baz,qux"})
def test_setting_allow_and_block_logs_warning(self, caplog):
- importlib.reload(airflow.stats)
-
- assert isinstance(airflow.stats.Stats.statsd, statsd.StatsClient)
- assert type(airflow.stats.Stats.instance.metrics_validator) is PatternAllowListValidator
with caplog.at_level(logging.WARNING):
+ importlib.reload(airflow.stats)
+
+ assert isinstance(airflow.stats.Stats.statsd, statsd.StatsClient)
+ assert type(airflow.stats.Stats.instance.metrics_validator) is PatternAllowListValidator
assert "Ignoring metrics_block_list" in caplog.text
diff --git a/airflow-core/tests/unit/dag_processing/test_manager.py b/airflow-core/tests/unit/dag_processing/test_manager.py
index 02121f0194e..24b173468d9 100644
--- a/airflow-core/tests/unit/dag_processing/test_manager.py
+++ b/airflow-core/tests/unit/dag_processing/test_manager.py
@@ -28,7 +28,6 @@ import textwrap
import time
from collections import deque
from datetime import datetime, timedelta
-from logging.config import dictConfig
from pathlib import Path
from socket import socket, socketpair
from unittest import mock
@@ -41,7 +40,6 @@ from sqlalchemy import func, select
from uuid6 import uuid7
from airflow.callbacks.callback_requests import DagCallbackRequest
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
from airflow.dag_processing.bundles.manager import DagBundlesManager
from airflow.dag_processing.manager import (
DagFileInfo,
@@ -115,7 +113,6 @@ class TestDagFileProcessorManager:
yield
def setup_method(self):
- dictConfig(DEFAULT_LOGGING_CONFIG)
clear_db_assets()
clear_db_runs()
clear_db_serialized_dags()
diff --git a/airflow-core/tests/unit/models/test_baseoperator.py b/airflow-core/tests/unit/models/test_baseoperator.py
index b4f7197583e..59e5924a4da 100644
--- a/airflow-core/tests/unit/models/test_baseoperator.py
+++ b/airflow-core/tests/unit/models/test_baseoperator.py
@@ -122,16 +122,6 @@ class TestBaseOperator:
assert op_no_dag.start_date.tzinfo
assert op_no_dag.end_date.tzinfo
- # ensure the default logging config is used for this test, no matter what ran before
- @pytest.mark.usefixtures("reset_logging_config")
- def test_logging_propogated_by_default(self, caplog):
- """Test that when set_context hasn't been called that log records are emitted"""
- BaseOperator(task_id="test").log.warning("test")
- # This looks like "how could it fail" but this actually checks that the handler called `emit`. Testing
- # the other case (that when we have set_context it goes to the file is harder to achieve without
- # leaking a lot of state)
- assert caplog.messages == ["test"]
-
def test_resume_execution(self):
op = BaseOperator(task_id="hi")
with pytest.raises(TaskDeferralTimeout):
diff --git a/airflow-core/tests/unit/utils/log/test_log_reader.py b/airflow-core/tests/unit/utils/log/test_log_reader.py
index 5ca675f8d08..2e718921cfc 100644
--- a/airflow-core/tests/unit/utils/log/test_log_reader.py
+++ b/airflow-core/tests/unit/utils/log/test_log_reader.py
@@ -18,10 +18,10 @@ from __future__ import annotations
import copy
import datetime
-import logging
import os
import sys
import tempfile
+import types
from typing import TYPE_CHECKING
from unittest import mock
@@ -78,16 +78,26 @@ class TestLogView:
@pytest.fixture(autouse=True)
def configure_loggers(self, log_dir, settings_folder):
- logging_config = copy.deepcopy(DEFAULT_LOGGING_CONFIG)
- logging_config["handlers"]["task"]["base_log_folder"] = log_dir
- settings_file = os.path.join(settings_folder, "airflow_local_settings_test.py")
- with open(settings_file, "w") as handle:
- new_logging_file = f"LOGGING_CONFIG = {logging_config}"
- handle.writelines(new_logging_file)
+ logging_config = {**DEFAULT_LOGGING_CONFIG}
+ logging_config["handlers"] = {**logging_config["handlers"]}
+ logging_config["handlers"]["task"] = {
+ **logging_config["handlers"]["task"],
+ "base_log_folder": log_dir,
+ }
+
+ mod = types.SimpleNamespace()
+ mod.LOGGING_CONFIG = logging_config
+
+ # "Inject" a fake module into sys so it loads it without needing to write valid python code
+ sys.modules["airflow_local_settings_test"] = mod
+
with conf_vars({("logging", "logging_config_class"): "airflow_local_settings_test.LOGGING_CONFIG"}):
settings.configure_logging()
- yield
- logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
+ try:
+ yield
+ finally:
+ del sys.modules["airflow_local_settings_test"]
+ settings.configure_logging()
@pytest.fixture(autouse=True)
def prepare_log_files(self, log_dir):
diff --git a/airflow-core/tests/unit/utils/test_log_handlers.py b/airflow-core/tests/unit/utils/test_log_handlers.py
index 5813c22819e..8f72c579c1f 100644
--- a/airflow-core/tests/unit/utils/test_log_handlers.py
+++ b/airflow-core/tests/unit/utils/test_log_handlers.py
@@ -36,6 +36,7 @@ from pydantic import TypeAdapter
from pydantic.v1.utils import deep_update
from requests.adapters import Response
+from airflow import settings
from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
from airflow.executors import executor_constants, executor_loader
from airflow.jobs.job import Job
@@ -63,7 +64,7 @@ from airflow.utils.types import DagRunType
from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.markers import skip_if_force_lowest_dependencies_marker
-pytestmark = pytest.mark.db_test
+pytestmark = [pytest.mark.db_test, pytest.mark.xfail()]
DEFAULT_DATE = datetime(2016, 1, 1)
TASK_LOGGER = "airflow.task"
@@ -90,8 +91,7 @@ class TestFileTaskLogHandler:
session.query(TaskInstance).delete()
def setup_method(self):
- logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
- logging.root.disabled = False
+ settings.configure_logging()
self.clean_up()
# We use file task handler by default.
diff --git a/airflow-core/tests/unit/utils/test_logging_mixin.py b/airflow-core/tests/unit/utils/test_logging_mixin.py
index 14e3532074b..4ad65db7fb1 100644
--- a/airflow-core/tests/unit/utils/test_logging_mixin.py
+++ b/airflow-core/tests/unit/utils/test_logging_mixin.py
@@ -93,18 +93,6 @@ class TestLoggingMixin:
assert DummyClass().log.name == "unit.utils.test_logging_mixin.DummyClass"
- def test_logger_name_is_root_when_logger_name_is_empty_string(self):
- """
- Ensure that when `_logger_name` is set as an empty string, the resulting logger name is an empty
- string too, which result in a logger with 'root' as name.
- Note: Passing an empty string to `logging.getLogger` will create a logger with name 'root'.
- """
-
- class EmptyStringLogger(LoggingMixin):
- _logger_name: str | None = ""
-
- assert EmptyStringLogger().log.name == "root"
-
def test_log_config_logger_name_correctly_prefix_logger_name(self):
"""
Ensure that when a class has `_log_config_logger_name`, it is used as prefix in the final logger
diff --git a/airflow-core/tests/unit/utils/test_process_utils.py b/airflow-core/tests/unit/utils/test_process_utils.py
index 7edc5f28b73..47ffc9245f3 100644
--- a/airflow-core/tests/unit/utils/test_process_utils.py
+++ b/airflow-core/tests/unit/utils/test_process_utils.py
@@ -103,19 +103,22 @@ class TestReapProcessGroup:
class TestExecuteInSubProcess:
def test_should_print_all_messages1(self, caplog):
execute_in_subprocess(["bash", "-c", "echo CAT; echo KITTY;"])
- msgs = [record.getMessage() for record in caplog.records]
- assert msgs == ["Executing cmd: bash -c 'echo CAT; echo KITTY;'", "Output:", "CAT", "KITTY"]
+ assert caplog.messages == [
+ "Executing cmd: bash -c 'echo CAT; echo KITTY;'",
+ "Output:",
+ "CAT",
+ "KITTY",
+ ]
def test_should_print_all_messages_from_cwd(self, caplog, tmp_path):
execute_in_subprocess(["bash", "-c", "echo CAT; pwd; echo KITTY;"], cwd=str(tmp_path))
- msgs = [record.getMessage() for record in caplog.records]
assert [
"Executing cmd: bash -c 'echo CAT; pwd; echo KITTY;'",
"Output:",
"CAT",
str(tmp_path),
"KITTY",
- ] == msgs
+ ] == caplog.messages
def test_using_env_works(self, caplog):
execute_in_subprocess(["bash", "-c", 'echo "My value is ${VALUE}"'], env=dict(VALUE="1"))
diff --git a/airflow-core/tests/unit/utils/test_task_handler_with_custom_formatter.py b/airflow-core/tests/unit/utils/test_task_handler_with_custom_formatter.py
deleted file mode 100644
index 118e442d0c1..00000000000
--- a/airflow-core/tests/unit/utils/test_task_handler_with_custom_formatter.py
+++ /dev/null
@@ -1,115 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-from __future__ import annotations
-
-import logging
-
-import pytest
-
-from airflow.config_templates.airflow_local_settings import DEFAULT_LOGGING_CONFIG
-from airflow.models.taskinstance import TaskInstance
-from airflow.providers.standard.operators.empty import EmptyOperator
-from airflow.utils.log.logging_mixin import set_context
-from airflow.utils.state import DagRunState
-from airflow.utils.timezone import datetime
-from airflow.utils.types import DagRunType
-
-from tests_common.test_utils.config import conf_vars
-from tests_common.test_utils.db import clear_db_runs
-from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS
-
-if AIRFLOW_V_3_0_PLUS:
- from airflow.utils.types import DagRunTriggeredByType
-
-pytestmark = pytest.mark.db_test
-
-
-DEFAULT_DATE = datetime(2019, 1, 1)
-TASK_HANDLER = "task"
-TASK_HANDLER_CLASS = "airflow.utils.log.task_handler_with_custom_formatter.TaskHandlerWithCustomFormatter"
-PREV_TASK_HANDLER = DEFAULT_LOGGING_CONFIG["handlers"]["task"]
-
-DAG_ID = "task_handler_with_custom_formatter_dag"
-TASK_ID = "task_handler_with_custom_formatter_task"
-
-
-@pytest.fixture(scope="module", autouse=True)
-def custom_task_log_handler_config():
- DEFAULT_LOGGING_CONFIG["handlers"]["task"] = {
- "class": TASK_HANDLER_CLASS,
- "formatter": "airflow",
- "stream": "sys.stdout",
- }
- logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
- logging.root.disabled = False
- yield
- DEFAULT_LOGGING_CONFIG["handlers"]["task"] = PREV_TASK_HANDLER
- logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
-
-
-@pytest.fixture
-def task_instance(dag_maker):
- with dag_maker(DAG_ID, start_date=DEFAULT_DATE, serialized=True) as dag:
- task = EmptyOperator(task_id=TASK_ID)
- triggered_by_kwargs = {"triggered_by": DagRunTriggeredByType.TEST} if AIRFLOW_V_3_0_PLUS else {}
- dagrun = dag_maker.create_dagrun(
- state=DagRunState.RUNNING,
- logical_date=DEFAULT_DATE,
- run_type=DagRunType.MANUAL,
- data_interval=dag.timetable.infer_manual_data_interval(run_after=DEFAULT_DATE),
- **triggered_by_kwargs,
- )
- ti = TaskInstance(task=task, run_id=dagrun.run_id)
- ti.log.disabled = False
- yield ti
- clear_db_runs()
-
-
-def assert_prefix_once(task_instance: TaskInstance, prefix: str) -> None:
- handler = next((h for h in task_instance.log.handlers if h.name == TASK_HANDLER), None)
- assert handler is not None, "custom task log handler not set up correctly"
- assert handler.formatter is not None, "custom task log formatter not set up correctly"
- previous_formatter = handler.formatter
- expected_format = f"{prefix}:{handler.formatter._fmt}"
- set_context(task_instance.log, task_instance)
- assert expected_format == handler.formatter._fmt
- handler.setFormatter(previous_formatter)
-
-
-def assert_prefix_multiple(task_instance: TaskInstance, prefix: str) -> None:
- handler = next((h for h in task_instance.log.handlers if h.name == TASK_HANDLER), None)
- assert handler is not None, "custom task log handler not set up correctly"
- assert handler.formatter is not None, "custom task log formatter not set up correctly"
- previous_formatter = handler.formatter
- expected_format = f"{prefix}:{handler.formatter._fmt}"
- set_context(task_instance.log, task_instance)
- set_context(task_instance.log, task_instance)
- set_context(task_instance.log, task_instance)
- assert expected_format == handler.formatter._fmt
- handler.setFormatter(previous_formatter)
-
-
-def test_custom_formatter_default_format(task_instance):
- """The default format provides no prefix."""
- assert_prefix_once(task_instance, "")
-
-
-@conf_vars({("logging", "task_log_prefix_template"): "{{ ti.dag_id }}-{{ ti.task_id }}"})
-def test_custom_formatter_custom_format_not_affected_by_config(task_instance):
- """Certifies that the prefix is only added once, even after repeated calls"""
- assert_prefix_multiple(task_instance, f"{DAG_ID}-{TASK_ID}")
diff --git a/devel-common/src/tests_common/pytest_plugin.py b/devel-common/src/tests_common/pytest_plugin.py
index 0edee542cd6..9acd6bcad41 100644
--- a/devel-common/src/tests_common/pytest_plugin.py
+++ b/devel-common/src/tests_common/pytest_plugin.py
@@ -18,8 +18,8 @@
from __future__ import annotations
import importlib
-import itertools
import json
+import logging
import os
import platform
import re
@@ -1667,13 +1667,9 @@ def create_log_template(request):
@pytest.fixture
def reset_logging_config():
- import logging.config
+ from airflow.logging_config import configure_logging
- from airflow import settings
- from airflow.utils.module_loading import import_string
-
- logging_config = import_string(settings.LOGGING_CLASS_PATH)
- logging.config.dictConfig(logging_config)
+ configure_logging()
@pytest.fixture(scope="session", autouse=True)
@@ -1954,7 +1950,7 @@ def add_expected_folders_to_pythonpath():
@pytest.fixture
-def cap_structlog():
+def cap_structlog(request, monkeypatch):
"""
Test that structlog messages are logged.
@@ -1969,56 +1965,46 @@ def cap_structlog():
...
... assert "not logged" not in cap_structlog # not in works too
"""
- import structlog.testing
- from structlog import configure, get_config
-
- class LogCapture(structlog.testing.LogCapture):
- # Partial comparison -- only check keys passed in, or the "event"/message if a single value is given
- def __contains__(self, target):
- if not isinstance(target, dict):
- target = {"event": target}
-
- def predicate(e):
- def check_one(key, want):
- try:
- val = e.get(key)
- if isinstance(want, re.Pattern):
- return want.match(val)
- return val == want
- except Exception:
- return False
-
- return all(itertools.starmap(check_one, target.items()))
+ import structlog.stdlib
+ from structlog import DropEvent, configure, get_config
- return any(predicate(e) for e in self.entries)
+ from tests_common.test_utils.logs import StructlogCapture
- def __getitem__(self, i):
- return self.entries[i]
-
- def __iter__(self):
- return iter(self.entries)
-
- def __repr__(self):
- return repr(self.entries)
-
- @property
- def text(self):
- """All the event text as a single multi-line string."""
- return "\n".join(e["event"] for e in self.entries)
-
- cap = LogCapture()
+ cap = StructlogCapture()
# Modify `_Configuration.default_processors` set via `configure` but always
# keep the list instance intact to not break references held by bound
# loggers.
processors = get_config()["processors"]
old_processors = processors.copy()
+
+ # And modify the stdlib logging to capture too
+ handler = logging.root.handlers[0]
+ if not isinstance(handler.formatter, structlog.stdlib.ProcessorFormatter):
+ raise AssertionError(
+ f"{type(handler.formatter)} is not an instance of structlog.stblid.ProcessorFormatter"
+ )
+
+ std_formatter = structlog.stdlib.ProcessorFormatter(
+ foreign_pre_chain=handler.formatter.foreign_pre_chain,
+ pass_foreign_args=True,
+ processor=cap,
+ )
+
+ def stdlib_filter(record):
+ with suppress(DropEvent):
+ std_formatter.format(record)
+ return False
+
try:
# clear processors list and use LogCapture for testing
processors.clear()
processors.append(cap)
configure(processors=processors)
+ monkeypatch.setattr(handler, "level", logging.DEBUG)
+ monkeypatch.setattr(handler, "filters", [stdlib_filter])
yield cap
finally:
+ cap._finalize()
# remove LogCapture and restore original processors
processors.clear()
processors.extend(old_processors)
@@ -2033,6 +2019,8 @@ def override_caplog(request):
This is in an effort to reduce flakiness from caplog related tests where one test file can change log
behaviour and bleed in to affecting other test files
"""
+ yield request.getfixturevalue("cap_structlog")
+ return
# We need this `_ispytest` so it doesn't warn about using private
fixture = pytest.LogCaptureFixture(request.node, _ispytest=True)
yield fixture
diff --git a/devel-common/src/tests_common/test_utils/logs.py b/devel-common/src/tests_common/test_utils/logs.py
index 03cbe0b55b9..aa999cb5a28 100644
--- a/devel-common/src/tests_common/test_utils/logs.py
+++ b/devel-common/src/tests_common/test_utils/logs.py
@@ -17,11 +17,20 @@
# under the License.
from __future__ import annotations
+import itertools
import json
+import logging
+import sys
+import traceback
+from contextlib import ExitStack, contextmanager
+from typing import TYPE_CHECKING, NoReturn
from airflow.models import Log
from airflow.sdk.execution_time.secrets_masker import DEFAULT_SENSITIVE_FIELDS
+if TYPE_CHECKING:
+ from structlog.typing import EventDict, WrappedLogger
+
def _masked_value_check(data, sensitive_fields):
"""
@@ -69,3 +78,211 @@ def check_last_log(session, dag_id, event, logical_date, expected_extra=None, ch
_masked_value_check(extra_json, DEFAULT_SENSITIVE_FIELDS)
session.query(Log).delete()
+
+
+class StructlogCapture:
+ """
+ Test that structlog messages are logged.
+
+ This extends the feature built in to structlog to make it easier to find if a message is logged.
+
+ >>> def test_something(cap_structlog):
+ ... log.info("some event", field1=False, field2=[1, 2])
+ ... log.info("some event", field1=True)
+ ... assert "some_event" in cap_structlog # a string searches on `event` field
+ ... assert {"event": "some_event", "field1": True} in cap_structlog # Searches only on passed fields
+ ... assert {"field2": [1, 2]} in cap_structlog
+ ...
+ ... assert "not logged" not in cap_structlog # not in works too
+
+ This fixture class will also manage the log level of stdlib loggers via ``at_level`` and ``set_level``.
+ """
+
+ # This class is a manual mixing of pytest's LogCaptureFixture and structlog's LogCapture class, but
+ # tailored to Airflow's "send all logs via structlog" approach
+
+ _logger: str | None = None
+ """The logger we specifically want to capture log messages from"""
+
+ def __init__(self):
+ self.entries = []
+ self._initial_logger_levels: dict[str | None, int] = {}
+
+ def _finalize(self) -> None:
+ """
+ Finalize the fixture.
+
+ This restores the log levels and the disabled logging levels changed by :meth:`set_level`.
+ """
+ from airflow._logging.structlog import PER_LOGGER_LEVELS
+
+ for logger_name, level in self._initial_logger_levels.items():
+ logger = logging.getLogger(logger_name)
+ logger.setLevel(level)
+ if level is logging.NOTSET:
+ del PER_LOGGER_LEVELS[logger_name]
+ else:
+ PER_LOGGER_LEVELS[logger_name] = level
+
+ def __contains__(self, target):
+ import operator
+
+ if isinstance(target, str):
+
+ def predicate(e):
+ return e["event"] == target
+ elif isinstance(target, dict):
+ # Partial comparison -- only check keys passed in
+ get = operator.itemgetter(*target.keys())
+ want = tuple(target.values())
+
+ def predicate(e):
+ try:
+ return get(e) == want
+ except Exception:
+ return False
+ else:
+ raise TypeError(f"Can't search logs using {type(target)}")
+
+ return any(predicate(e) for e in self.entries)
+
+ def __getitem__(self, i):
+ return self.entries[i]
+
+ def __iter__(self):
+ return iter(self.entries)
+
+ def __repr__(self):
+ return repr(self.entries)
+
+ def __call__(self, logger: WrappedLogger, method_name: str, event_dict: EventDict) -> NoReturn:
+ from structlog import DropEvent
+ from structlog._log_levels import map_method_name
+
+ from airflow._logging.structlog import (
+ NamedBytesLogger,
+ NamedWriteLogger,
+ )
+
+ logger_name = (
+ event_dict.get("logger_name")
+ or event_dict.get("logger")
+ or (isinstance(logger, (NamedBytesLogger, NamedWriteLogger)) and logger.name)
+ or ""
+ )
+ if not self._logger or logger_name.startswith(self._logger):
+ event_dict["log_level"] = map_method_name(method_name)
+
+ # Capture the current exception. This mirrors the "ExceptionRenderer", but much more minimal for
+ # testing
+ if event_dict.get("exc_info") is True:
+ event_dict["exc_info"] = sys.exc_info()
+ self.entries.append(event_dict)
+
+ raise DropEvent
+
+ @property
+ def text(self):
+ """All the event text as a single multi-line string."""
+
+ def format(e):
+ yield e["event"] + "\n"
+ if exc_info := e.get("exc_info"):
+ yield from traceback.format_exception(*exc_info)
+
+ return "".join(itertools.chain.from_iterable(map(format, self.entries)))
+
+ # These next fns make it duck-type the same as Pytests "caplog" fixture
+ @property
+ def messages(self):
+ """All the event messages as a list."""
+ return [e["event"] for e in self.entries]
+
+ def _force_enable_logging(self, level: int, logger_obj: logging.Logger) -> int:
+ """
+ Enable the desired logging level if the global level was disabled via ``logging.disabled``.
+
+ Only enables logging levels greater than or equal to the requested ``level``.
+
+ Does nothing if the desired ``level`` wasn't disabled.
+
+ :param level:
+ The logger level caplog should capture.
+ All logging is enabled if a non-standard logging level string is supplied.
+ Valid level strings are in :data:`logging._nameToLevel`.
+ :param logger_obj: The logger object to check.
+
+ :return: The original disabled logging level.
+ """
+ original_disable_level: int = logger_obj.manager.disable
+
+ if not logger_obj.isEnabledFor(level):
+ # Each level is `10` away from other levels.
+ # https://docs.python.org/3/library/logging.html#logging-levels
+ disable_level = max(level - 10, logging.NOTSET)
+ logging.disable(disable_level)
+
+ return original_disable_level
+
+ @contextmanager
+ def at_level(self, level: str | int, logger=None):
+ from airflow._logging.structlog import NAME_TO_LEVEL, PER_LOGGER_LEVELS
+
+ if isinstance(level, str):
+ level = NAME_TO_LEVEL[level.lower()]
+
+ # Since we explicitly set the level of the "airflow" logger in our config, we want to set that by
+ # default if the test auithor didn't ask for this at a specific logger to be set (otherwise we only
+ # set the root logging level, which doesn't have any affect if sub loggers have explicit levels set)
+ keys: tuple[str, ...] = (logger or "",)
+ if not logger:
+ keys += ("airflow",)
+
+ def _reset(logger, key, level, orig_hdlr_level):
+ logger.setLevel(level)
+ if level is logging.NOTSET:
+ del PER_LOGGER_LEVELS[key]
+ else:
+ PER_LOGGER_LEVELS[key] = level
+ if logger.handlers:
+ hdlr = logger.handlers[0]
+ hdlr.setLevel(orig_hdlr_level)
+
+ cm = ExitStack()
+ for key in keys:
+ old = PER_LOGGER_LEVELS.get(key, logging.NOTSET)
+ PER_LOGGER_LEVELS[key] = level
+ stdlogger = logging.getLogger(key)
+ stdlogger.setLevel(level)
+ hdlr = orig_hdlr_level = None
+ if stdlogger.handlers:
+ hdlr = stdlogger.handlers[0]
+ orig_hdlr_level = hdlr.level
+ hdlr.setLevel(level)
+ cm.callback(_reset, stdlogger, key, old, orig_hdlr_level)
+
+ with cm:
+ yield self
+
+ def set_level(self, level: str | int, logger=None):
+ from airflow._logging.structlog import NAME_TO_LEVEL, PER_LOGGER_LEVELS
+
+ # Set the global level
+ if isinstance(level, str):
+ level = NAME_TO_LEVEL[level.lower()]
+
+ key = logger or ""
+
+ stdlogger = logging.getLogger(key)
+ self._initial_logger_levels[key] = PER_LOGGER_LEVELS.get(key, logging.NOTSET)
+
+ PER_LOGGER_LEVELS[key] = level
+ stdlogger.setLevel(level)
+ self._logger = logger
+
+ def clear(self):
+ self.entries = []
+
+ @property
+ def records(self):
+ return self.entries
diff --git a/task-sdk/src/airflow/sdk/definitions/_internal/node.py b/task-sdk/src/airflow/sdk/definitions/_internal/node.py
index 21fa4ede5b1..3cb087202df 100644
--- a/task-sdk/src/airflow/sdk/definitions/_internal/node.py
+++ b/task-sdk/src/airflow/sdk/definitions/_internal/node.py
@@ -17,16 +17,18 @@
from __future__ import annotations
-import logging
import re
from abc import ABCMeta, abstractmethod
from collections.abc import Iterable, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any
+import structlog
+
from airflow.sdk.definitions._internal.mixins import DependencyMixin
if TYPE_CHECKING:
+ from airflow._logging.structlog import AirflowFilteringBoundLogger
from airflow.sdk.definitions.abstractoperator import Operator
from airflow.sdk.definitions.dag import DAG
from airflow.sdk.definitions.edges import EdgeModifier
@@ -81,7 +83,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
_log_config_logger_name: str | None = None
_logger_name: str | None = None
- _cached_logger: logging.Logger | None = None
+ _cached_logger: AirflowFilteringBoundLogger | None = None
def __init__(self):
self.upstream_task_ids = set()
@@ -112,7 +114,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
return "_in_memory_dag_"
@property
- def log(self) -> logging.Logger:
+ def log(self) -> AirflowFilteringBoundLogger:
"""
Get a logger for this node.
@@ -137,7 +139,7 @@ class DAGNode(DependencyMixin, metaclass=ABCMeta):
else self._log_config_logger_name
)
- self._cached_logger = logging.getLogger(logger_name)
+ self._cached_logger = structlog.get_logger(logger_name)
return self._cached_logger
@property
diff --git a/task-sdk/src/airflow/sdk/definitions/dag.py b/task-sdk/src/airflow/sdk/definitions/dag.py
index 9e728d68128..941cda2db35 100644
--- a/task-sdk/src/airflow/sdk/definitions/dag.py
+++ b/task-sdk/src/airflow/sdk/definitions/dag.py
@@ -1043,29 +1043,6 @@ class DAG:
from airflow.utils.state import DagRunState, State, TaskInstanceState
from airflow.utils.types import DagRunTriggeredByType, DagRunType
- if TYPE_CHECKING:
- from airflow.models.taskinstance import TaskInstance
-
- def add_logger_if_needed(ti: TaskInstance):
- """
- Add a formatted logger to the task instance.
-
- This allows all logs to surface to the command line, instead of into
- a task file. Since this is a local test run, it is much better for
- the user to see logs in the command line, rather than needing to
- search for a log file.
-
- :param ti: The task instance that will receive a logger.
- """
- format = logging.Formatter("[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s")
- handler = logging.StreamHandler(sys.stdout)
- handler.level = logging.INFO
- handler.setFormatter(format)
- # only add log handler once
- if not any(isinstance(h, logging.StreamHandler) for h in ti.log.handlers):
- log.debug("Adding Streamhandler to taskinstance %s", ti.task_id)
- ti.log.addHandler(handler)
-
exit_stack = ExitStack()
if conn_file_path or variable_file_path:
@@ -1189,7 +1166,6 @@ class DAG:
else:
# Run the task locally
try:
- add_logger_if_needed(ti)
if mark_success:
ti.set_state(State.SUCCESS)
log.info("[DAG TEST] Marking success for %s on %s", ti.task, ti.logical_date)
diff --git a/task-sdk/tests/task_sdk/bases/test_operator.py b/task-sdk/tests/task_sdk/bases/test_operator.py
index b9a4c14e317..2eed3065b96 100644
--- a/task-sdk/tests/task_sdk/bases/test_operator.py
+++ b/task-sdk/tests/task_sdk/bases/test_operator.py
@@ -698,6 +698,15 @@ class TestBaseOperator:
task.render_template_fields(context={"foo": "whatever", "bar": "whatever"})
assert mock_jinja_env.call_count == 1
+ # ensure the default logging config is used for this test, no matter what ran before
+ @pytest.mark.usefixtures("reset_logging_config")
+ def test_logging_propogated_by_default(self, caplog):
+ BaseOperator(task_id="test").log.warning("test")
+ # This looks like "how could it fail" but this actually checks that the handler called `emit`. Testing
+ # the other case (that when we have set_context it goes to the file is harder to achieve without
+ # leaking a lot of state)
+ assert caplog.messages == ["test"]
+
def test_init_subclass_args():
class InitSubclassOp(BaseOperator):