You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2025/12/18 13:47:06 UTC
(airflow) branch glue-pass-params-v2 updated (049cf1e1c77 -> 758753706b8)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch glue-pass-params-v2
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 049cf1e1c77 glue pass openlineage params
add acfcba15c84 Fix incorrect backfill duration calculation in Grid view (#58813)
add f1325133187 Fix Execution API refresh token (#58782)
add 1f6202df9e9 feat(param): add source to Param (#58615)
add 9917a04c0ce Add mprocs upgrade check to upgrade_important_versions (#58723)
add e0aa80c4401 Improved how relative imports are allowed for shared distributions (#58824)
add 2a84037dbad docs(asset): enhance asset extra documentation (#58769)
add a291225dbe3 Fix links in AGENTS.md (#58774)
add e1247676a70 Add retry handling for Kubernetes API client operations in KubernetesPodOperator (#58778)
add 52975053074 Add CODEOWNERS entry for Airflow CLI (#58784)
add bbf5b8fdf71 Helm chart: start Redis directly, not via shell (#58790)
add 1914cff318c feat: add PYTHON_LTO build arg for FIPS compliance (#58583)
add 160db5bd84b Fix static checks, tailing whitespace (#58835)
add 1f4f28d05f1 Small update to `README_RELEASE_PROVIDERS.md` (#58818)
add f697a0b0ee7 fix Preserve variable value formatting in edit dialog (#58757)
add db7689f25ae CI: Upgrade important CI environment (#58839)
add f5d6a7c6bad Fix mypy errors for RayHook (#58843)
add 7ccd8824b20 Fix lower bound dependency to common-compat provider (#58833)
add ed10c596661 Add 31 missing airflowctl integration test commands (#58701)
add 65aec2d640a Fail PR when common compat changes and there is no # use next version (#58840)
add 4a61eebddb5 fix poll_sleep is ignored in DataflowStartFlexTemplateOperator definition (#58846)
add 8e18380fb08 Remove global from listener unit tests (#58838)
add d409ba9a275 Make the launcher a cached_property and minor change to improve how pod name is retrieved (#58646)
add 7ddc52a3e5a Unclog shared conf library tests from importing from airflow core (#58851)
add f1032188570 Fix(UI): Improve mobile responsiveness of Dashboard sections (#58811)
add 60bf28d687e Remove global from breeze testing commands (#58854)
add ea9e1ec3b5e Fix typos and improve phrasing in unit testing guide (#58765)
add 33cb64afbd2 fix airflowignore negation does not work in subfolders (#58740)
add 0c3e920fcd4 Align all linebreaks in ci-amd-arm.yml (#58855)
add 9e6c15fd18b Remove global from breeze PROVIDER_DEPENDENCIES (#58857)
add 6a86f258fc1 update zizmor prek hook to version 1.18.0 (#58860)
add b89ab85817d Handle false positive tests for airflowctl integration tests (#58859)
add 302bda12367 fix(helm): allow additional PodDisruptionBudget config properties (#58864)
add af27f43fe48 Add back `SimpleAllAdminMiddleware` (#58829)
add caafddae739 Add database cleanup docs to Helm productions docs (#58707)
add fc706353947 Bugfix/remove global from breeze (#58866)
add 65d681c7228 Update healthcheck command syntax for celery worker (#58861)
add b48c9096f72 Make cleanup cronjob conditional on kubernetes executor (#58695)
add 90dbb8ffc70 Remove global from prek ci scripts (#58872)
add 53687e3d1b9 Synchronize the list of people who can publish docs and build images (#58876)
add a7fc9c69452 fix(docs): handle array parameter flattening in _get_params (#56389)
add 95da754a17c Update providers metadata 2025-11-30 (#58875)
add 80ca9f51a86 Add Pre-commit check for airflowctl tests (#58856)
add d4fec16e398 Add additional clarifying docs for TaskGroup params (#58511)
add b702a9cd055 Auto install pnpm for UI e2e tests in breeze command (#58845)
add bca0183ad93 sdk: Refactor `get_python_source` to strip comments (#50327) (#57782)
add dc88d1e82a9 Add prompt to breeze container (#58879)
add 8545d3c578a Docs: update UIAlert import path and usage for v3 (#58089) (#58090)
add 68558f343a2 Fix mypy errors in models (#58728)
add 0adfc5bd7c8 BUGFIX: trigger ui parameter field is dict when param.value is null (#58682)
add 20c44354f21 Return dag tags in alphabetical order (#58883)
add 9c4ebf337fc Use `HTTP_422_UNPROCESSABLE_CONTENT` instead of `HTTP_422_UNPROCESSABLE_ENTITY` (#58828)
add 13baf17b933 Fix mypy error in `airflow-core/src/airflow/models/dag` (#58902)
add aa7cb1d7f1b Catch and log pandas import errors (#58744)
add c9d0fab18d5 Select deadlines by id so that deadlines associated with other dagruns of same run_id are not pruned. (#58574)
add 32afc282eab Move BaseSecretsBackend to shared library for client server separation (#58621)
add b0057c29de1 Prepare ad-hoc release for 2025-12-01 wave of providers (#58878)
add b30bfa34fae Remove global from openlineage provider (#58868)
add 1c142ec5e64 Support optional scope in OAuth token request (#58871)
add 29ace334ac9 chore: Move OpenLineage methods to BaseSQLOperator (#58897)
add c116edcd5c7 Fix `breeze setup autocomplete` references (#58898)
add 9ad99263314 feat(google): add enable_open_telemetry_tracing to PubSubHook and PubSubPublishMessageOperator (#58766)
add 43606160c76 fix string to datetime pydantic and test (#58791)
add 78cc12f77f0 Small updates to release process (PMC provider check) (#58913)
add 7f81bc0f46f Improve XCom documentation by clarifying usage of XCOM_RETURN_KEY constant (#58356)
add 6c50b62ab72 Improve startup information printed when Breeze starts (#58926)
add 2c7de9768ff Adding Task SDK integration tests for Xcom operations (#58638)
add 1c53e0844d5 Fix the serialization issue of AwaitMessageTrigger with Redis. (#58746)
add d490d09fc11 CI: Upgrade important CI environment (#58915)
add c5e70a92210 Fix mypy errors in sql_to_s3 due to pandas stub bump (#58923)
add baec49a13bb Fix LocalExecutor memory spike by applying gc.freeze (#58365)
add e8480587885 nit: rename TriggerDagRunOperator._defer to deferrable (#58925)
add 22af27ea5a7 Fix backfill max_active_runs race condition with concurrent schedulers (#58807)
add bcda5080b8f Do not remove .pyc and .pyo files after building Python (#58944)
add c811833d32e Update badges of Airflow: add prek and use nicer table formatting (#58930)
add 7f07c312319 UI Unify trash icons (#58953)
add 1f1cbec33eb Fix remaining mypy issues in amazon provider (#58950)
add 74c2044f303 Add translations for PL. (#58940)
add 586fe930791 Fix exception when logging stdout with a custom %-fmt string. (#58959)
add 46472e5fbba Fix depreccated DagBag import from models.dagbag (#58938)
add 1f575ab700a tests: Add OL system tests for deferrable TriggerDagRunOperator (#58933)
add b95fca4dbdf Bump minimum prek version to 0.2.0 (#58952)
add 438dbfd8e06 Add license header to test_percent_formatter.py (#58975)
add 28e08fc9970 Make shared timezone library standalone by getting rid of `airflow.*` imports (#58888)
add 8ab288a4d50 Fix Liveness / Readiness / Startup probe path for Airflow 3.x #58129 (#58734)
add 27b5e58a61e Update providers metadata 2025-12-02 (#58978)
add a5f1c4edf51 Update asset manager to be partition-aware (#58289)
add fe99a32f27d Version cap fastapi due to is_coroutine changes in 0.123.5 (#58980)
add 1e8d0a198ed [main] Upgrade important CI environment (#58976)
add 24e682c593c Fix assets used only as inlets being incorrectly orphaned (#58303) (#58368)
add 729e74be2e0 Add pandas as an optional dependency to the AMPP (#58984)
add dcc88f1d4ec fix(i18n): Trasnalte trigger as 觸發器, triggerer as 觸發者 and dagRun.triggeredBy as 觸發來源 (#58988)
add 5aa90c0cda3 Fix NotMapped exception when clearing task instances with downstream/upstream (#58922)
add f3af770842d Implement timetables in SDK (#58669)
add 08813da0d69 Add prek check before compiling assets in start-airflow (#58979)
add ab8820e2d71 Fix go back button from fab iframe (#58997)
add 4970ea2978e Fix task instance and runs tooltips in Grid view (#58359)
add 1b30bf90a58 Bump the fab-ui-package-updates group across 1 directory with 2 updates (#58954)
add 76fe18d9883 Adding Task SDK integration tests for asset operations (#59000)
add c97dbbdf263 Refactor FavoriteDags layout for improved responsiveness (#59014)
add b653efa9556 Refactor Dashboard layout and Health component responsiveness (#59015)
add c02ae15f953 Bump the core-ui-package-updates group across 1 directory with 10 updates (#59009)
add 28e6122df05 Remove global from celery provider (#58869)
add 2017714aebe Retry on `RedshiftSQLHook.get_conn` (#59017)
add 729571b1a8d Update the documentation for the LocalExecutor (#58990)
add ab71473d29e Update logging documentation for Azure Blob Storage (#59016)
add 4a5451632ea Fix CloudwatchTaskHandler display error (#54054)
add 02c49379153 fix(sftp): add default port to SFTPHookAsync (#58945)
add 3a9fad1667d Decouple xcom public API from using XcomEncoder (#58900)
add caa3d0d21e9 provide a clear naming and description for the attribute caching get_template_context (#59023)
add 430f0e08d30 Fixing LLM providers tests (#59031)
add d024a46ce7d Change task log source display to hidden by default (#58749)
add 0a1e798fb71 Add UI/API performance tips (#59004)
add ba7e2fb4685 feat: added toasters for permission denied (#57966) (#58016)
add d3bc5b96e8f Move HTTPAuthorizationCredentials import from TYPE_CHECKING block to (#59048)
add d3fecc2bffc Update prek badge URL in README (#59055)
add 8ec7601eab6 Fix TI overview trend count link (#59059)
add 462f71fa6eb i18n(Ko): add missing translations in common.json (Dec 4) (#59039)
add f0b2ae95a61 i18n(Ko): add missing translations in hitl.json (#59037)
add 36ecfe20b82 i18n(Ko): add missing translations in assets.json (Nov 26) (#58709)
add fe26bfe0575 Uncomment Python version pin in `check-lazy-logging` (#59047)
add cd255d7bffe Bump ruff to 0.14.8, uv to 0.9.15 (#59058)
add c63de7b0701 i18n(ko): translating in components.json (Dec 4) (#59040)
add 5bf9983746d i18n(Ko) : add mission translations in dags.json (Dec 4) (#59041)
add be38f92ed8f Fix readme from generated code (#59070)
add 52526b3f605 Link AIP-76 todos to issues (#59063)
add 337aee896a9 Move the traces and metrics code under a common observability package (#56187)
add 65ca7d337d4 chore: Adjust AirflowRunFacet and docs after moving OL methods to BaseSQLOperator (#58903)
add 056efdbc1ad keycloak: enable client_secret retrieval from secrets backend (#59065)
add b698b7b5c6b Update scheduler to create dag runs for partitioned-asset-driven dags (#59006)
add 64102a08080 Shorten the path to import conf from airflow task sdk (#59029)
add 8b63199e1fb CI: Upgrade important CI environment (#59101)
add 0ecd95f7dd1 docs(INTHEWILD.md): add Numberly to the list of companies using Apache Airflow (#59098)
add 52987f42ffc Limit urllib3 to < 2.6.0 to handle kubernetes client breaking with it (#59108)
add d51e5723c63 Add sensitive keycloak value to fix failing tests. (#59107)
add 15aeed9a8ae Rename dag to serdag where appropriate in scheduler (#59118)
add 28b89563a19 Fix flaky test_next_dagrun_after_fake_scheduled_previous (#59035)
add 5c942e6c67c Bump python 3.13 to 3.13.11 (#59125)
add 1d6bd9d8add Add dag bundles to `airflow info` command (#59124)
add 40ee29ce3cd Limit urllib3 to <2.6.0 during latest boto tests (#59130)
add 5845599cb74 Fix newly introduced mypy error (#59112)
add 97cf3c786af fix flaky TestEmrCreateJobFlowOperator (#59128)
add f496fc035ba Update include @noeunkim for Korean translations (#59096)
add 519e1ef7000 i18n(Ko): add missing translation in components.json (Dec 5) (#59076)
add 6b4867e8169 Update uv version to 0.9.16 (#59138)
add 0ee0dafd956 Move Self to observability shared package (#59140)
add bc38090507e CI: Upgrade important CI environment (#59144)
add 24a334ff1ba Fix typo in Failed to Trigger DAG -> Failed to Trigger Dag (#59077)
add 236dab6406e Breeze exit on incompatible arguments (#59148)
add a129c2cbd71 Also limit urllib3 to < 2.6.0 in k8s tests - for use in v3-1-tests (#59152)
add f0d0b2fb993 Add script that verifies consistency: Release Page with Release Calendar (#59142)
add bdabf46d964 Remove global statement from Fab provider (#59018)
add bc7ce10582a Fix breeze check-release-files Command for Core+Task SDK checks (#59159)
add 53a721cc59d improved logging & exitcode check (#59146)
add ad6961dc447 Fix airflow/task-sdk relase PMC checks (#59164)
add 5c41f2ee293 Prevent dag processor crash on encountering excel files in the DAG directory (#59069)
add 8879eb8cf23 Fix mypy error on version check in dev (#59171)
add d46a9d12065 Fix inconsistent Dag hashes when template fields contain unordered dicts (#59091)
add f58548101b8 AIP 67 - Multi-Team: Update LocalExecutor to support running in a multi-team config (#59021)
add 80b49311651 Correct header parameter description in HTTP operator (#59105)
add 0bf581e2a28 Handle mid line chunking when writing docker stream to logs (#59071)
add 2b38b194633 Fix DagRun.queued_at not updating when clearing (#59066)
add 908e6c28435 Create an explicit control for createUserJob (#56057)
add 7e9787265ac Allow teams to use global executors by default (#59116)
add 7446e41feb5 Fix ingress notes (#59122)
add 31e4381eff1 helm: correctly derive celery sync_parallelism from scheduler CPU limits (#58733)
add 0f8d79a4852 fix uv venv fail without direct internet access (#59046)
add ef14b9a618f update missing catalan translations (#59110)
add d96ff621ae5 Fix finally return handling (#58998)
add 3935c86a69d fix: scheduler, triggerer, worker airflow components have inconsistent log volume (#56418)
add 15ddf8d1b9a Close German translation gaps for full UI translation 2025-12-07 (#59162)
add fffab39f996 Drop Airflow 2 Support in Edge Provider (#59143)
add 6208a479924 Implement get_config in LocalFilesystemBackend (#59062)
add 9023c38adc4 Add Horizontal Pod Autoscaler (HPA) Support for Airflow API Server (#52392)
add 25179391926 docs: Fix broken permalink icon (#58763)
add 501f724b64f Update release candidate commands (#59186)
add d9c826eb77f Repurpose imports from sdk for sdk tests for sentry (#59034)
add 3b8ed158d94 CHG: fix address (#59193)
add 7510322789e feat: display logged in within UserSettingsButton (#58981)
add c2b20ae7c58 Align the term Dag in all translations (#59153)
add bfb99f5f524 Add backcompat for exceptions in providers (#58727)
add 38db2defa23 Show asset extra in asset list (#59195)
add 92d2f87572a Add toggle functionality to Dags state filters (#59089)
add 217fc5f59a6 Remove global from providers amazon tests (#59178)
add 85b65eb3961 Fix misleading error message when GitHook creation fails (#59084)
add 0beaf2d3cca Adding retries and pool to Weaviate dags to handle connection latency issues (#59198)
add a59d2ecdc39 chore: use OL macros instead of building OL ids from scratch (#59197)
add dd31202e8c4 Convert the exclusion on urllib3 to != for 2.6.0 (#59203)
add 68275ced21c Remove tags from AWS system tests (#59206)
add a178d00aa5e Remove team ID and use team name as PK (#59109)
add 51064ba2409 Fix mypy error in `scheduler_job_runner.py` (#59204)
add 479acd8a77d i18n(Ko): add missing translation (Dec 8) (#59199)
add 6d57acfa779 Remove labels from unexisted sections (#59213)
add 4ea1ebf56c8 Added paginated_run method to KiotaRequestAdapterHook in MSGraph (#57536)
add cfd82c7f9c5 Add newsfragment for 52357 PR (#59214)
add 442979423f7 Fix outdated dependency documentation (#58970)
add 84568391f21 Add is_watcher field to TriggerDetails TypedDict (#59161)
add b20d1f62cd5 Fix broken syntax highlighting in AIR rules note section (#59188)
add 0bf6f82d3d6 Fix side-effect of running version suffix replacements (#58937)
add 3c2163729d3 Fix composer sys test (#59200)
add 95658615b68 Keycloak CLI: add dry run functionality (#59134)
add 216e7a722f7 Add max_trigger_to_select_per_loop conf to respect Triggerer HA setup (#58803)
add 697ad4a3d40 Import empty operator from compat module for AF2/AF3 support (#59222)
add a0b3a3b8b48 Update BulkDeleteAction to use generic type (#59207)
add be94a60ecc7 Remove unused functions in Core expandinput (#59225)
add 087c656cf2a Completing stubbed tests for AssetEvent operations (#59182)
add 4438c28bee1 Remove UUID case for sqlalachemy 2 from tests (#59230)
add 7218cf04581 Fix links for DurationChart (#59095)
add ac6e893af73 Add common: namespace to translation keys in AssetsList component for consistency. (#59227)
add 787b8186b62 docs: adjust misleading docs (#59228)
add 42d82cbc17a Permit `airflow db migrate -r` with an empty database (#59205)
add a6c9066120a Fix incorrect state query parameter for task_instances (#59086)
add c12afbdda46 Fix migration to add TI primary key for sqlite dialect (#59241)
add 3b74a8d6920 Complete Hebrew translation 2025-12-09 (#59133)
add 740da666b58 Don't depend upon FastAPI inside Task-SDK client (#59250)
add 12a66dbccab Keycloak: implement connection pooling (#59252)
add 11e127ba5fc Create `create_token` method in FAB auth manager (#59245)
add 10558d8d154 Conditionally import from task sdk (#59254)
add 861cf546cf0 Close the gaps in PL translation 2025-12-09 (#59251)
add af5e469f8c2 Passwords from stdin in Keycloak Provider (#59119)
add dac53df3ade Fix airflow version in migration (#59240)
add 8e3f689e14e Fix remote logging S3 connection retrieval in supervisor context (#59247)
add 38cf2147567 Prepare providers release 2025-12-09 (#59249)
add afd53fd8b4c Fetch asset event source dag run (#59090)
add 91ae1733791 Add wildcard support for dag_id and dag_run_id in bulk ti endpoint (#57441)
add 4983b2a0236 Extract shared "module_loading" distribution (#59139)
add 1e7e40ee595 Add CI workflow for UI e2e tests (#58901)
add 0b6d2589df0 Fix an odd import of pendulum from sqlalchemy_utils instead of elsewhere. (#59258)
add 274e096e6df Fix flaky test_get_last_ti_filters_removed_tasks and Pydantic v2 discriminator error to fix CI (#59273)
add 612a290a34e Enhance code view to support search and diff (#55467)
add ffb180b9ee4 Update main as Apache Airflow 3.1.4 has been released (#59268)
add f1fdcdc0249 Return 403 when the Keycloak access token is expired (#59281)
add 925376664f8 A few more missing compat imports in example dags (#59283)
add d32091cd4f4 Check team boundaries in variables (#58905)
add 0c794d01a46 Update version and add changelog for 3.1.4 Python Client Release (#59274)
add 006b7e8683a Ensure that git-sync actually runs when dags.gitSync.enabled=true and dags.persistence.enabled=true (#59123)
add d739a989cd2 Add TdLoadOperator to execute data transfers using Teradata Parallel Transporter (TPT) tdload utility (#58918)
add 505881858ec Translation gaps in main TR (#59168)
add 98fee1ecd66 Fix release process issues after going through 2025-12-09 release (#59261)
add 9f87af96fd9 Add uvicorn_logging_level config option to control API server access logs (#55405) (#56062)
add 6e3e51201e4 Fix XCom key handling when keys contain special characters like slash (#58344)
add 7c6fcecc535 fix chrome failure (#59300)
add 3966d0afb2f Check for Deprecated SQLA Query Class Usage in Prevent Deprecated SQLA Hook (#55274)
add f1f1feb9dbb update readme with new contribution section (#59312)
add d8604c4bb2f Update i18n freeze instructions (#59136)
add 09350e3d6b2 Add theme config (#58411)
add e0b02c9bb3a Cleanup input components (#59287)
add 0224303bc68 Fix backfill run_on_latest_version defaulting to False instead of True (#59304)
add 95adec475f9 Add hints for claim validation to auth manager documentation (#59320)
add c9e190fb254 Add todo re exception handling in dagrun create loop (#59121)
add a27dd7be521 Update json to JSON for consistency in translations (#59323)
add f4cd07129d8 Bump the fab-ui-package-updates group across 1 directory with 3 updates (#59242)
add 2bacaea711e `ISSUE_TRIAGE_PROCESS.rst` fix link: spelling and whitespace (#59340)
add dfeca8274e0 Update manager.py (#59330)
add 81dee2f4fe1 Optimize K8s API usage for watching events (#59080)
add 310e1181cb5 Fix _format_schema_for_description to handle missing "fields" key gracefully (#59343)
add 23ab73adf0a build: upgrade uv to 0.9.17, prek to 0.2.21, ruff to 0.14.9 (#59350)
add 8b604105c97 add pull for docker images + complie UI assets for workflow_dispatch (#59315)
add cd4c768a0f3 nit: bump OL client dependency to 1.41 (#59321)
add acce763164f Add toaster notifications for Connection Test (#59354)
add 5fdee096fe8 Fix toast notification description text truncation (#59356)
add 8256ee16d1d Add task group ID filtering support to task instance query (#58092)
add b0cb5c9c0df Returning `destination_cloud_storage_uris` from `BigQueryToGCSOperator` (#59367)
add 2ff8a72572a Fix various Airflow 2 import issues revealed by previous fixes (#59341)
add 626af79f547 Fix Internal Server Error in API server due to KeycloakPostError when refreshing user in Keycloak provider (#59361)
add 1a7c5769340 Add `team_name` to connection public APIs (#59336)
add 220023740a8 Proposes clarification about information sent to the devlist (#59335)
add 1c865093072 Update main as Airflow 3.1.5 has been released (#59379)
add 3b6a24bba11 Fix XCom directory creation logic in Kubernetes decorator (#56545) (#59347)
add 93dfcde9446 Add documentation for Airflow Ctl integration tests in contributing-docs (#59387)
add f128c840a89 fix flaky test_dags_clear (#59394)
add 9eba1b85ae3 test(e2e): align DAG naming to Dag for consistency (#59401)
add 8ae779d7287 Update link and typo to Apache Mentoring Program in communication guidelines (#59403)
add cd832362930 aDd cooldown days for prek autoupgrade. (#59395)
add 93454f0a8b1 Adding Japanese translations for UI (i18n) (#59313)
add 652d4ee983f docs: Improve unit-testing docs with runnable pytest examples for Operators (#58969)
add 52441b4faa0 i18n(Ko): add missing translation (Dec 12) (#59345)
add 900e6920642 Add support for disabling SA token automount for Scheduler (#59156)
add 2887405a25d Add `delete_active_pod` cleanup option and corresponding unit tests (#59160)
add 83c8bd93366 Chart: update default Airflow version to 3.1.5 (#59271)
add 2b92c976390 Fix deferred mode KubernetesPodOperator: fast-fail pod start errors (ErrImagePull/ImagePullBackOff) (#59010)
add 92fd810e017 Extend airflow_ti_running metrics by scheduled, queued and deferred (#58819)
add d7dad134e11 Update providers metadata (#59388)
add 355c2144473 Limit aiosqlite to !=0.22.0 as it is hanging pytest session of ours (#59406)
add ab803b8e389 Remove Legacy Query Object Usage from test_assets.py (#59412)
add 9d60935ca63 CI: Upgrade important CI environment (#59416)
add 8a109dbcb11 Update issue templates regarding Airflow 2 and misc. (#59384)
add 41e023d74ae build: upgrade mypy to 1.19.1 (#59446)
add 737b3bafcd1 Fix Old RC removal logic and add test for the function (#59438)
add a9f095e6fc4 Infer the RC from the target version during release. (#59455)
add e1adf0b1eba Refactor deprecated SQLA query openlineage provider (#59448)
add 75e222575c8 Pnpm upgrade to 10.x and prevent script execution (#59466)
add 1ba1732caf0 Add Refresh Token logic to auth manager docs (#54196)
add 1d99e8dd850 Add pdbs for triggerer and workers (#59068)
add c4b17598a36 Add @typescript-eslint dev dependencies for version 8.50.0 (#59484)
add 551ad63002c Amend updated breeze param (#59483)
add d34942ef1fa Remove old task SDK RC when creating RC (#59459)
add bef558dfb20 Update the remove old releases function to include task-sdk (#59468)
add 809f59722db Refactor scheduler_dag_execution_timing.py to use SQLA2 (#59419)
add f74c105741d Fix GitHub Actions workflow: remove unsupported --previous-release option (#59499)
add 178205f756c Refactor deprecated SQLA models/test_dagrun.py (#59453)
add 6bc5fe925ed Docs: fix typo in documentation overview (#59452)
add 690ad9d8486 Use `get_dialect_name` instead of `session.get_bind().dialect.name` (#59492)
add 5ee00b317b2 Fix logout route in Keycloak provider so a KeycloakostError doesn't lead to Internal Server Error in API server (#59382)
add 97b30f412b2 Fix message of _read_from_logs_server when status_code is 403 (#59489)
add 35f74854453 scheduler test: filter captured messages to skip SQLA debug logs (#59493)
add ce580dc730f Fix Theme Configs (#59473)
add 215ff14d0d3 Add translation for version compare selection placeholder (#59346)
add d28c8febd7e Fix `KubernetesPodOperator` deferrable mode with env-defined connection (#41706) (#58841)
add 7934a6953a0 `issue-59385`: Removing unneeded call the `SnowflakeSqlApiHook` (#59470)
add 0bbc9f12171 fix(i18n): plugins should be translated as 外掛 instead of 插件 in Taiwanese Mandarian (#59513)
add fbbce53abd8 Add pre-commit hook to verify version consistency (#59509)
add 2cc1fa91fdb Fix clear task instance dialog tasks states (#59363)
add ab5ba5ca053 Adaptions for custom auth manager example in documentation (#59355)
add 4bfd1215941 Fix svn push step to add both versions in one commit (#59505)
add 7ba8dbfd088 Add ability to add, edit, and delete XComs directly from UI (#58921)
add 55c42fa7fd6 Manually bump packages (#59073)
add c81c570ecfb add ui e2e condtion for prod image build (#59523)
add 55a4ecd6be6 Consolidate more components (#59338)
add c998c941697 Update release version to 3.1.1 and improve error handling in SVN access (#59531)
add 31f953b0c8d bug fix (#59292)
add dd3c93e239c Remove redundant InvokeModel calls (#59527)
add 4713f72525d Bump Sphinx Airflow theme to 0.3.0 (#59538)
add d6095c5f2f0 Split SDK and serialized asset classes (#58993)
add f6343b164ab Feat e2e test for Dags pagination (#59400)
add 9fc231e237c Minor cleanups removing SDK references from Core (#59491)
add 908058951c5 Use exclude_defaults to prevent explicitly empty fields to be in request (#57718)
add 6059bec62d4 Add Rahul Vats as UI code owner (#59548)
add 7b30cc0fb66 Refactor Dag trigger e2e tests into dags-list.spec (#59542)
add 3af4d289cd2 Move Serialization/Deserialization (serde) to task SDK (#58992)
add c3f0fd477cc Refactor/sqla2 providers(celery, kubernetes, databricks, mysql) to remove SQLA query usage (#59537)
add 5c25ccc0010 Update multiple test files to remove SQLAlchemy query usage (#59501)
add e7a1e57fd16 [BUGFIX] Flush session() before processing Event Buffer (#59314)
add 357b7fbdd22 Use textwrap.dedent in all cases in conf tests (#59536)
add 875a427ab3f Update README.md "DAG" to "Dag" for consistency (#59554)
add aed9e04b83d Update guide duration from 5 to 15 minutes (#59552)
add ebefb4f145c Add Rahul to codeowners (#59565)
add d0bb4ad34d5 Update Dockerfile (#59569)
add c541c637c9f UI: Add data interval override option for manual DAG runs (#57342)
add fe5d81a9224 Revert "Use exclude_defaults to prevent explicitly empty fields to be in request (#57718)" (#59578)
add e659539c002 Align prevent rerun messages in line 38 and 68 in i18n dags.json (#59562)
add 1c0b4936d40 Fix Dag Processor logging crash (#59317)
add e2cd7c609b1 Add log record when listening dag is partitioned but run has no key (#59375)
add f8d1e8831d5 Add missing fields to SQLValueCheckOperator docstring (#59409)
add 1c703068a51 [Chart] Fix rendering condition of "git_sync_ssh_key_volume" (#59418)
add 50e10d667bb Make Edge provider SQLA2 compatible (#59414)
add dcaad6008e2 Add sqs hook tests (#59586)
add cb3d5b7d243 Remove unecessary eslint ignores in TriggerDAGForm (#59579)
add 667b112bca7 Speed up dag parsing for google gen_ai_generative_model dag (#59587)
add 940c7bb970e Use sdk shared library references in serde source code (#59591)
new 758753706b8 glue pass openlineage params
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (049cf1e1c77)
\
N -- N -- N refs/heads/glue-pass-params-v2 (758753706b8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/CODEOWNERS | 11 +-
.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml | 29 +-
.github/ISSUE_TEMPLATE/2-feature_request.yml | 17 +-
.../3-airflow_providers_bug_report.yml | 19 +-
.../4-airflow_helmchart_bug_report.yml | 17 +-
.../ISSUE_TEMPLATE/5-airflow_doc_issue_report.yml | 4 +
.github/actions/breeze/action.yml | 2 +-
.github/actions/install-prek/action.yml | 4 +-
.github/boring-cyborg.yml | 3 +
.github/workflows/additional-prod-image-tests.yml | 40 +
.github/workflows/airflow-distributions-tests.yml | 2 +-
.github/workflows/basic-tests.yml | 10 +-
.github/workflows/ci-amd-arm.yml | 24 +-
.github/workflows/publish-docs-to-s3.yml | 4 +-
.github/workflows/release_dockerhub_image.yml | 7 +-
.github/workflows/ui-e2e-tests.yml | 159 +
.pre-commit-config.yaml | 50 +-
.rat-excludes | 3 +-
AGENTS.md | 4 +-
Dockerfile | 39 +-
Dockerfile.ci | 47 +-
INTHEWILD.md | 1 +
ISSUE_TRIAGE_PROCESS.rst | 2 +-
README.md | 56 +-
RELEASE_NOTES.rst | 85 +
airflow-core/.pre-commit-config.yaml | 19 +-
.../logging-monitoring/metrics.rst | 9 +
.../administration-and-deployment/scheduler.rst | 4 +-
.../docs/authoring-and-scheduling/assets.rst | 50 +-
.../docs/authoring-and-scheduling/deferring.rst | 13 +
.../docs/authoring-and-scheduling/serializers.rst | 6 +-
airflow-core/docs/best-practices.rst | 102 +-
airflow-core/docs/conf.py | 4 +
.../docs/core-concepts/auth-manager/index.rst | 22 +-
airflow-core/docs/core-concepts/dags.rst | 3 +
airflow-core/docs/core-concepts/debug.rst | 2 +-
airflow-core/docs/core-concepts/executor/local.rst | 23 +-
airflow-core/docs/core-concepts/xcoms.rst | 10 +-
airflow-core/docs/howto/customize-ui.rst | 97 +-
.../docs/howto/docker-compose/docker-compose.yaml | 6 +-
airflow-core/docs/howto/index.rst | 1 +
airflow-core/docs/howto/performance.rst | 55 +
.../howto/run-with-self-signed-certificate.rst | 2 +-
airflow-core/docs/howto/set-up-database.rst | 2 +-
airflow-core/docs/img/airflow_erd.sha256 | 2 +-
airflow-core/docs/img/airflow_erd.svg | 4954 +++++++++++---------
.../exmaple_theme_configuration_dark_mode.png | Bin 0 -> 60070 bytes
.../exmaple_theme_configuration_light_mode.png | Bin 0 -> 60098 bytes
airflow-core/docs/installation/dependencies.rst | 22 +-
.../docs/installation/supported-versions.rst | 2 +-
.../docs/installation/upgrading_to_airflow3.rst | 2 +-
airflow-core/docs/migrations-ref.rst | 9 +-
airflow-core/docs/public-airflow-interface.rst | 3 +
airflow-core/newsfragments/58337.feature.rst | 1 +
airflow-core/pyproject.toml | 15 +-
airflow-core/src/airflow/__init__.py | 7 +-
airflow-core/src/airflow/_shared/module_loading | 1 +
airflow-core/src/airflow/_shared/observability | 1 +
airflow-core/src/airflow/_shared/secrets_backend | 1 +
.../api_fastapi/auth/managers/base_auth_manager.py | 13 +-
.../managers/simple/middleware.py} | 25 +-
.../auth/managers/simple/ui/package-lock.json | 1653 +++----
.../auth/managers/simple/ui/package.json | 16 +-
.../auth/managers/simple/ui/pnpm-lock.yaml | 322 +-
.../src/airflow/api_fastapi/common/parameters.py | 69 +-
.../src/airflow/api_fastapi/common/types.py | 71 +
.../src/airflow/api_fastapi/compat.py | 20 +-
.../src/airflow/api_fastapi/core_api/app.py | 5 +
.../api_fastapi/core_api/datamodels/backfills.py | 1 +
.../api_fastapi/core_api/datamodels/common.py | 3 +-
.../api_fastapi/core_api/datamodels/connections.py | 2 +
.../api_fastapi/core_api/datamodels/dags.py | 6 +
.../core_api/datamodels/task_instances.py | 2 +
.../api_fastapi/core_api/datamodels/ui/common.py | 4 +-
.../api_fastapi/core_api/datamodels/ui/config.py | 3 +-
.../api_fastapi/core_api/datamodels/ui/teams.py | 3 -
.../api_fastapi/core_api/datamodels/variables.py | 5 +-
.../api_fastapi/core_api/openapi/_private_ui.yaml | 55 +-
.../core_api/openapi/v2-rest-api-generated.yaml | 132 +-
.../core_api/routes/public/backfills.py | 1 +
.../api_fastapi/core_api/routes/public/dags.py | 5 +-
.../core_api/routes/public/task_instances.py | 7 +-
.../core_api/routes/public/variables.py | 8 +-
.../api_fastapi/core_api/routes/public/xcom.py | 81 +-
.../api_fastapi/core_api/routes/ui/config.py | 23 +
.../api_fastapi/core_api/routes/ui/teams.py | 2 +-
.../src/airflow/api_fastapi/core_api/security.py | 3 +-
.../core_api/services/public/task_instances.py | 353 +-
.../core_api/services/ui/connections.py | 4 +-
.../api_fastapi/core_api/services/ui/task_group.py | 5 +-
.../src/airflow/api_fastapi/execution_api/app.py | 37 +-
.../execution_api/datamodels/taskinstance.py | 5 +-
.../src/airflow/api_fastapi/execution_api/deps.py | 80 +-
.../api_fastapi/execution_api/routes/__init__.py | 4 +-
.../api_fastapi/execution_api/routes/dag_runs.py | 135 +-
.../execution_api/routes/task_instances.py | 47 +-
.../api_fastapi/execution_api/routes/variables.py | 19 +-
.../api_fastapi/execution_api/routes/xcoms.py | 212 +-
.../api_fastapi/execution_api/versions/__init__.py | 5 +
.../execution_api/versions/v2025_12_08.py} | 31 +-
airflow-core/src/airflow/assets/evaluation.py | 41 +-
airflow-core/src/airflow/assets/manager.py | 180 +-
airflow-core/src/airflow/cli/cli_config.py | 7 +-
.../src/airflow/cli/commands/api_server_command.py | 9 +-
.../src/airflow/cli/commands/daemon_utils.py | 2 +-
.../src/airflow/cli/commands/dag_command.py | 23 +-
.../src/airflow/cli/commands/db_manager_command.py | 2 +-
.../src/airflow/cli/commands/info_command.py | 7 +-
.../src/airflow/cli/commands/team_command.py | 17 +-
.../src/airflow/config_templates/config.yml | 83 +-
airflow-core/src/airflow/configuration.py | 5 +-
.../src/airflow/dag_processing/bundles/base.py | 5 +-
.../src/airflow/dag_processing/bundles/manager.py | 23 +-
.../src/airflow/dag_processing/collection.py | 42 +-
airflow-core/src/airflow/dag_processing/manager.py | 13 +-
.../src/airflow/dag_processing/processor.py | 2 +-
.../src/airflow/example_dags/example_assets.py | 4 +-
.../example_dags/example_outlet_event_extra.py | 6 +
.../src/airflow/example_dags/example_xcom.py | 3 +-
airflow-core/src/airflow/exceptions.py | 54 +-
.../src/airflow/executors/base_executor.py | 15 +-
.../src/airflow/executors/executor_loader.py | 2 +-
.../src/airflow/executors/local_executor.py | 75 +-
.../src/airflow/jobs/dag_processor_job_runner.py | 2 +-
airflow-core/src/airflow/jobs/job.py | 4 +-
.../src/airflow/jobs/scheduler_job_runner.py | 208 +-
.../src/airflow/jobs/triggerer_job_runner.py | 9 +-
airflow-core/src/airflow/listeners/spec/asset.py | 8 +-
airflow-core/src/airflow/logging_config.py | 2 +-
.../versions/0000_2_6_2_squashed_migrations.py | 848 ++++
.../0001_2_7_0_add_index_to_task_instance_table.py | 4 +-
...3_0_0_add_uuid_primary_key_to_task_instance_.py | 22 +-
.../versions/0094_3_2_0_remove_team_id.py | 148 +
airflow-core/src/airflow/models/__init__.py | 7 +-
airflow-core/src/airflow/models/asset.py | 53 +-
airflow-core/src/airflow/models/callback.py | 2 +-
airflow-core/src/airflow/models/connection.py | 29 +-
airflow-core/src/airflow/models/dag.py | 37 +-
airflow-core/src/airflow/models/dagrun.py | 7 +-
airflow-core/src/airflow/models/dagwarning.py | 4 +-
airflow-core/src/airflow/models/deadline.py | 2 +-
airflow-core/src/airflow/models/expandinput.py | 15 +-
airflow-core/src/airflow/models/pool.py | 15 +-
airflow-core/src/airflow/models/serialized_dag.py | 16 +-
airflow-core/src/airflow/models/taskinstance.py | 117 +-
airflow-core/src/airflow/models/team.py | 31 +-
airflow-core/src/airflow/models/trigger.py | 9 +-
airflow-core/src/airflow/models/variable.py | 105 +-
airflow-core/src/airflow/models/xcom.py | 24 +-
airflow-core/src/airflow/models/xcom_arg.py | 18 +-
.../airflow/{metrics => observability}/__init__.py | 0
.../{_shared => observability/metrics}/__init__.py | 0
.../observability/metrics/datadog_logger.py | 40 +
.../airflow/observability/metrics/otel_logger.py | 43 +
.../airflow/observability/metrics/statsd_logger.py | 63 +
.../src/airflow/{ => observability}/stats.py | 10 +-
airflow-core/src/airflow/observability/trace.py | 122 +
.../common => observability/traces}/__init__.py | 0
.../traces/otel_tracer.py} | 30 +-
airflow-core/src/airflow/plugins_manager.py | 4 +-
airflow-core/src/airflow/providers_manager.py | 2 +-
airflow-core/src/airflow/secrets/base_secrets.py | 82 +-
.../src/airflow/secrets/environment_variables.py | 9 +-
.../src/airflow/secrets/local_filesystem.py | 41 +-
airflow-core/src/airflow/secrets/metastore.py | 13 +-
airflow-core/src/airflow/serialization/__init__.py | 12 +
airflow-core/src/airflow/serialization/decoders.py | 141 +
.../airflow/serialization/definitions/assets.py | 309 ++
.../src/airflow/serialization/definitions/param.py | 12 +-
airflow-core/src/airflow/serialization/encoders.py | 342 ++
airflow-core/src/airflow/serialization/helpers.py | 50 +-
.../airflow/serialization/serialized_objects.py | 292 +-
.../airflow/serialization/serializers/bignum.py | 2 +-
.../airflow/serialization/serializers/builtin.py | 2 +-
.../airflow/serialization/serializers/datetime.py | 2 +-
.../airflow/serialization/serializers/deltalake.py | 2 +-
.../airflow/serialization/serializers/iceberg.py | 2 +-
.../serialization/serializers/kubernetes.py | 2 +-
.../src/airflow/serialization/serializers/numpy.py | 2 +-
.../airflow/serialization/serializers/pandas.py | 2 +-
.../airflow/serialization/serializers/pydantic.py | 2 +-
.../airflow/serialization/serializers/timezone.py | 2 +-
.../src/airflow/serialization/stringify.py | 141 +
airflow-core/src/airflow/settings.py | 14 +-
airflow-core/src/airflow/task/priority_strategy.py | 2 +-
.../src/airflow/ti_deps/deps/prev_dagrun_dep.py | 3 +-
airflow-core/src/airflow/timetables/_cron.py | 4 +-
airflow-core/src/airflow/timetables/_delta.py | 15 +
airflow-core/src/airflow/timetables/assets.py | 31 +-
airflow-core/src/airflow/timetables/base.py | 90 +-
airflow-core/src/airflow/timetables/events.py | 15 +-
airflow-core/src/airflow/timetables/interval.py | 25 +-
airflow-core/src/airflow/timetables/simple.py | 106 +-
airflow-core/src/airflow/timetables/trigger.py | 70 +-
airflow-core/src/airflow/traces/utils.py | 114 -
airflow-core/src/airflow/triggers/callback.py | 2 +-
.../src/airflow/ui/openapi-gen/queries/common.ts | 6 +-
.../ui/openapi-gen/queries/ensureQueryData.ts | 8 +-
.../src/airflow/ui/openapi-gen/queries/prefetch.ts | 8 +-
.../src/airflow/ui/openapi-gen/queries/queries.ts | 33 +-
.../src/airflow/ui/openapi-gen/queries/suspense.ts | 8 +-
.../airflow/ui/openapi-gen/requests/schemas.gen.ts | 128 +-
.../ui/openapi-gen/requests/services.gen.ts | 41 +-
.../airflow/ui/openapi-gen/requests/types.gen.ts | 79 +-
airflow-core/src/airflow/ui/package.json | 77 +-
airflow-core/src/airflow/ui/playwright.config.ts | 1 -
airflow-core/src/airflow/ui/pnpm-lock.yaml | 2673 +++++------
airflow-core/src/airflow/ui/public/i18n/README.md | 10 +-
.../airflow/ui/public/i18n/locales/ca/assets.json | 4 +
.../airflow/ui/public/i18n/locales/ca/common.json | 7 +-
.../ui/public/i18n/locales/ca/components.json | 10 +
.../src/airflow/ui/public/i18n/locales/ca/dag.json | 12 +
.../airflow/ui/public/i18n/locales/ca/dags.json | 5 +
.../airflow/ui/public/i18n/locales/ca/hitl.json | 3 +-
.../ui/public/i18n/locales/de/components.json | 9 +-
.../airflow/ui/public/i18n/locales/en/admin.json | 6 +
.../airflow/ui/public/i18n/locales/en/assets.json | 1 +
.../airflow/ui/public/i18n/locales/en/browse.json | 30 +-
.../airflow/ui/public/i18n/locales/en/common.json | 13 +-
.../ui/public/i18n/locales/en/components.json | 8 +
.../airflow/ui/public/i18n/locales/en/dags.json | 2 +-
.../airflow/ui/public/i18n/locales/he/assets.json | 7 +
.../airflow/ui/public/i18n/locales/he/browse.json | 4 -
.../airflow/ui/public/i18n/locales/he/common.json | 10 +-
.../ui/public/i18n/locales/he/components.json | 10 +
.../src/airflow/ui/public/i18n/locales/he/dag.json | 12 +
.../airflow/ui/public/i18n/locales/he/dags.json | 5 +
.../airflow/ui/public/i18n/locales/he/hitl.json | 5 +-
.../ui/public/i18n/locales/hu/components.json | 4 +-
.../src/airflow/ui/public/i18n/locales/hu/dag.json | 4 +-
.../airflow/ui/public/i18n/locales/ja/admin.json | 164 +
.../airflow/ui/public/i18n/locales/ja/assets.json | 35 +
.../airflow/ui/public/i18n/locales/ja/browse.json | 22 +
.../airflow/ui/public/i18n/locales/ja/common.json | 334 ++
.../ui/public/i18n/locales/ja/components.json | 150 +
.../src/airflow/ui/public/i18n/locales/ja/dag.json | 166 +
.../airflow/ui/public/i18n/locales/ja/dags.json | 101 +
.../ui/public/i18n/locales/ja/dashboard.json | 45 +
.../airflow/ui/public/i18n/locales/ja/hitl.json | 38 +
.../airflow/ui/public/i18n/locales/ja/tasks.json | 10 +
.../airflow/ui/public/i18n/locales/ko/assets.json | 2 +
.../airflow/ui/public/i18n/locales/ko/browse.json | 4 -
.../airflow/ui/public/i18n/locales/ko/common.json | 12 +-
.../ui/public/i18n/locales/ko/components.json | 12 +-
.../airflow/ui/public/i18n/locales/ko/dags.json | 5 +
.../airflow/ui/public/i18n/locales/ko/hitl.json | 3 +-
.../ui/public/i18n/locales/nl/components.json | 4 +-
.../airflow/ui/public/i18n/locales/pl/assets.json | 1 +
.../airflow/ui/public/i18n/locales/pl/browse.json | 4 -
.../airflow/ui/public/i18n/locales/pl/common.json | 4 +
.../ui/public/i18n/locales/pl/components.json | 3 +
.../airflow/ui/public/i18n/locales/pt/assets.json | 10 +-
.../airflow/ui/public/i18n/locales/pt/browse.json | 2 +-
.../airflow/ui/public/i18n/locales/pt/common.json | 38 +-
.../ui/public/i18n/locales/pt/components.json | 28 +-
.../src/airflow/ui/public/i18n/locales/pt/dag.json | 12 +-
.../airflow/ui/public/i18n/locales/pt/dags.json | 16 +-
.../ui/public/i18n/locales/pt/dashboard.json | 28 +-
.../airflow/ui/public/i18n/locales/th/common.json | 2 +-
.../src/airflow/ui/public/i18n/locales/th/dag.json | 4 +-
.../airflow/ui/public/i18n/locales/tr/assets.json | 5 +
.../airflow/ui/public/i18n/locales/tr/common.json | 7 +-
.../ui/public/i18n/locales/tr/components.json | 10 +
.../src/airflow/ui/public/i18n/locales/tr/dag.json | 12 +
.../airflow/ui/public/i18n/locales/tr/dags.json | 5 +
.../airflow/ui/public/i18n/locales/tr/hitl.json | 3 +-
.../ui/public/i18n/locales/zh-CN/components.json | 2 +-
.../ui/public/i18n/locales/zh-TW/common.json | 10 +-
.../ui/public/i18n/locales/zh-TW/dashboard.json | 2 +-
.../ui/src/components/Assets/AssetEvents.tsx | 2 +-
.../ui/src/components/Assets/TriggeredRuns.tsx | 4 +-
.../ui/src/components/Banner/BackfillBanner.tsx | 4 +-
.../src/airflow/ui/src/components/BasicTooltip.tsx | 133 +
.../ui/src/components/Clear/Run/ClearRunDialog.tsx | 15 +-
.../TaskInstance/ClearGroupTaskInstanceDialog.tsx | 17 +-
.../ClearTaskInstanceConfirmationDialog.tsx | 6 +-
.../Clear/TaskInstance/ClearTaskInstanceDialog.tsx | 13 +-
.../src/airflow/ui/src/components/ConfigForm.tsx | 3 +-
.../DagWarningsModal.tsx => DAGWarningsModal.tsx} | 4 +-
.../ui/src/components/DagActions/ParseDag.tsx | 3 +-
.../src/components/DagActions/RunBackfillForm.tsx | 51 +-
.../src/airflow/ui/src/components/DagRunInfo.tsx | 2 +-
.../src/airflow/ui/src/components/DeleteDialog.tsx | 4 +-
.../ui/src/components/DisplayMarkdownButton.tsx | 4 +-
.../airflow/ui/src/components/DurationChart.tsx | 17 +-
.../ui/src/components/EditableMarkdownButton.tsx | 4 +-
.../ui/src/components/FilterBar/FilterPill.tsx | 45 +-
.../components/FilterBar/filters/DateFilter.tsx | 23 +-
.../FilterBar/filters/DateRangeCalendar.tsx | 6 +-
.../FilterBar/filters/DateRangeFilter.tsx | 186 +-
.../components/FilterBar/filters/NumberFilter.tsx | 65 +-
.../components/FilterBar/filters/SelectFilter.tsx | 88 +-
.../FilterBar/filters/TextSearchFilter.tsx | 23 +-
.../airflow/ui/src/components/FilterBar/index.ts | 2 +-
.../src/components/FlexibleForm/FieldDropdown.tsx | 2 +-
.../src/components/FlexibleForm/FieldSelector.tsx | 10 +-
.../src/airflow/ui/src/components/HoverTooltip.tsx | 63 -
.../airflow/ui/src/components/LimitedItemsList.tsx | 4 +-
.../src/components/MarkAs/Run/MarkRunAsDialog.tsx | 4 +-
.../TaskInstance/MarkTaskInstanceAsDialog.tsx | 4 +-
.../airflow/ui/src/components/SearchBar.test.tsx | 3 +-
.../src/airflow/ui/src/components/SearchBar.tsx | 26 +-
.../ui/src/components/TaskInstanceTooltip.tsx | 5 +-
.../airflow/ui/src/components/TrendCountChart.tsx | 2 +-
.../TriggerDag/TriggerDAGAdvancedOptions.tsx | 84 +
.../src/components/TriggerDag/TriggerDAGForm.tsx | 191 +-
.../src/components/TriggerDag/TriggerDAGModal.tsx | 1 +
.../src/components/ui/ActionBar/CloseTrigger.tsx | 4 +-
.../airflow/ui/src/components/ui/ActionButton.tsx | 4 +-
.../src/airflow/ui/src/components/ui/Alert.tsx | 4 +-
.../src/airflow/ui/src/components/ui/Button.tsx | 52 -
.../ui/src/components/ui/Dialog/CloseTrigger.tsx | 4 +-
.../ui/src/components/ui/FileUpload/Dropzone.tsx | 43 -
.../ui/src/components/ui/FileUpload/FileInput.tsx | 54 -
.../ui/src/components/ui/FileUpload/Item.tsx | 61 -
.../ui/src/components/ui/FileUpload/List.tsx | 51 -
.../ui/src/components/ui/FileUpload/Root.tsx | 35 -
.../airflow/ui/src/components/ui/InputGroup.tsx | 57 -
.../ui/src/components/ui/Popover/CloseTrigger.tsx | 4 +-
.../src/airflow/ui/src/components/ui/RadioCard.tsx | 55 +-
.../ui/src/components/ui/ResizableWrapper.tsx | 7 +-
.../ui/src/components/ui/Select/Content.tsx | 2 +-
.../ui/src/components/ui/Select/Trigger.tsx | 4 +-
.../ui/src/components/ui/Toaster/Toaster.tsx | 4 +-
.../src/airflow/ui/src/components/ui/index.ts | 3 -
.../index.ts => context/ChakraCustomProvider.tsx} | 30 +-
airflow-core/src/airflow/ui/src/i18n/config.ts | 1 +
.../ui/src/layouts/Details/DetailsLayout.tsx | 11 +-
.../ui/src/layouts/Details/Grid/GridButton.tsx | 66 +-
.../airflow/ui/src/layouts/Details/Grid/GridTI.tsx | 65 +-
.../ui/src/layouts/Details/PanelButtons.tsx | 35 +-
.../ui/src/layouts/Details/TaskStreamFilter.tsx | 3 +-
.../ui/src/layouts/Nav/UserSettingsButton.tsx | 15 +
airflow-core/src/airflow/ui/src/main.tsx | 15 +-
.../airflow/ui/src/pages/AssetsList/AssetsList.tsx | 46 +-
.../ui/src/pages/AssetsList/DependencyPopover.tsx | 4 +-
.../ui/src/pages/Connections/Connections.tsx | 3 +-
.../pages/Connections/DeleteConnectionButton.tsx | 10 +-
.../pages/Connections/DeleteConnectionsButton.tsx | 8 +-
.../ui/src/pages/Dag/Calendar/CalendarCell.tsx | 15 +-
.../ui/src/pages/Dag/Calendar/CalendarTooltip.tsx | 104 +-
.../src/airflow/ui/src/pages/Dag/Code/Code.tsx | 290 +-
.../ui/src/pages/Dag/Code/CodeDiffViewer.tsx | 87 +
.../ui/src/pages/Dag/Code/VersionCompareSelect.tsx | 112 +
.../pages/Dag/Tasks/TaskFilters/TaskFilters.tsx | 3 +-
.../ui/src/pages/DagsList/AssetSchedule.tsx | 4 +-
.../src/pages/DagsList/DagsFilters/DagsFilters.tsx | 14 +-
.../src/airflow/ui/src/pages/DagsList/DagsList.tsx | 3 +-
.../airflow/ui/src/pages/Dashboard/Dashboard.tsx | 4 +-
.../Dashboard/FavoriteDags/FavoriteDagCard.tsx | 4 +-
.../pages/Dashboard/FavoriteDags/FavoriteDags.tsx | 22 +-
.../ui/src/pages/Dashboard/Health/Health.tsx | 2 +-
.../HistoricalMetrics/HistoricalMetrics.tsx | 6 +-
.../Dashboard/HistoricalMetrics/MetricSection.tsx | 3 +-
.../pages/Dashboard/Stats/DAGImportErrorsModal.tsx | 4 +-
.../Dashboard/Stats/PluginImportErrorsModal.tsx | 4 +-
.../src/airflow/ui/src/pages/Events/Events.tsx | 6 +-
.../airflow/ui/src/pages/Pools/AddPoolButton.tsx | 4 +-
.../src/airflow/ui/src/pages/Pools/PoolForm.tsx | 3 +-
.../src/airflow/ui/src/pages/Pools/Pools.tsx | 3 +-
.../src/airflow/ui/src/pages/Run/Header.tsx | 2 +-
airflow-core/src/airflow/ui/src/pages/Security.tsx | 23 +-
.../ui/src/pages/Task/Overview/Overview.tsx | 6 +-
.../ui/src/pages/TaskInstance/AssetEvents.tsx | 6 +-
.../airflow/ui/src/pages/TaskInstance/Header.tsx | 2 +-
.../ui/src/pages/TaskInstance/Logs/Logs.test.tsx | 22 +-
.../ui/src/pages/TaskInstance/Logs/Logs.tsx | 2 +-
.../src/pages/TaskInstance/Logs/TaskLogHeader.tsx | 9 +-
.../ui/src/pages/TaskInstances/TaskInstances.tsx | 3 +-
.../src/pages/Variables/DeleteVariablesButton.tsx | 8 +-
.../src/pages/Variables/ImportVariablesButton.tsx | 4 +-
.../ui/src/pages/Variables/ImportVariablesForm.tsx | 55 +-
.../Variables/ManageVariable/AddVariableButton.tsx | 4 +-
.../ManageVariable/DeleteVariableButton.tsx | 10 +-
.../ManageVariable/EditVariableButton.tsx | 13 +-
.../Variables/ManageVariable/VariableForm.tsx | 3 +-
.../airflow/ui/src/pages/Variables/Variables.tsx | 3 +-
.../XCom/AddXComButton.tsx} | 42 +-
.../airflow/ui/src/pages/XCom/DeleteXComButton.tsx | 96 +
.../ParseDag.tsx => pages/XCom/EditXComButton.tsx} | 50 +-
.../src/airflow/ui/src/pages/XCom/XCom.tsx | 56 +-
.../src/airflow/ui/src/pages/XCom/XComModal.tsx | 213 +
.../ui/src/queries/useConnectionTypeMeta.ts | 103 +-
.../src/airflow/ui/src/queries/useDagParams.ts | 12 +-
.../airflow/ui/src/queries/useDagTagsInfinite.ts | 18 +-
.../ui/src/queries/useRefreshOnNewDagRuns.ts | 4 +-
.../airflow/ui/src/queries/useTestConnection.ts | 21 +-
.../src/airflow/ui/src/queries/useTrigger.ts | 28 +-
airflow-core/src/airflow/ui/src/theme.ts | 728 +--
airflow-core/src/airflow/ui/tests/e2e/README.md | 161 +-
.../src/airflow/ui/tests/e2e/pages/DagsPage.ts | 46 +-
.../{dag-trigger.spec.ts => dags-list.spec.ts} | 52 +-
airflow-core/src/airflow/utils/context.py | 15 +-
airflow-core/src/airflow/utils/db.py | 7 +-
airflow-core/src/airflow/utils/db_manager.py | 2 +-
airflow-core/src/airflow/utils/file.py | 6 +-
airflow-core/src/airflow/utils/helpers.py | 56 +-
airflow-core/src/airflow/utils/json.py | 37 +-
.../src/airflow/utils/log/file_task_handler.py | 4 +-
airflow-core/src/airflow/utils/serve_logs/core.py | 7 +-
.../src/airflow/utils/serve_logs/log_server.py | 2 +-
airflow-core/src/airflow/utils/sqlalchemy.py | 6 +-
.../tests/integration/otel/dags/otel_test_dag.py | 4 +-
.../dags/otel_test_dag_with_pause_between_tasks.py | 4 +-
.../otel/dags/otel_test_dag_with_pause_in_task.py | 4 +-
airflow-core/tests/integration/otel/test_otel.py | 29 +-
.../tests/unit/always/test_project_structure.py | 1 -
airflow-core/tests/unit/always/test_secrets.py | 10 +-
.../unit/always/test_secrets_local_filesystem.py | 75 +
.../auth/managers/simple/test_middleware.py | 61 +
.../auth/managers/test_base_auth_manager.py | 6 +-
.../unit/api_fastapi/common/test_exceptions.py | 15 +-
.../tests/unit/api_fastapi/common/test_types.py | 76 +
.../core_api/routes/public/test_assets.py | 39 +-
.../core_api/routes/public/test_connections.py | 84 +-
.../core_api/routes/public/test_dag_run.py | 38 +-
.../core_api/routes/public/test_dags.py | 125 +-
.../core_api/routes/public/test_event_logs.py | 53 +-
.../core_api/routes/public/test_hitl.py | 4 +
.../core_api/routes/public/test_task_instances.py | 195 +-
.../core_api/routes/public/test_tasks.py | 38 +-
.../core_api/routes/public/test_variables.py | 45 +-
.../core_api/routes/public/test_xcom.py | 135 +-
.../api_fastapi/core_api/routes/ui/test_config.py | 21 +
.../api_fastapi/core_api/routes/ui/test_grid.py | 8 +-
.../api_fastapi/core_api/routes/ui/test_teams.py | 5 -
.../services/public/test_task_instances.py | 340 +-
.../execution_api/versions/head/test_dag_runs.py | 105 +-
.../execution_api/versions/head/test_router.py | 8 +-
.../versions/head/test_task_instances.py | 23 +-
.../versions/v2025_11_07}/__init__.py | 0
.../versions/v2025_11_07/test_dag_runs.py | 62 +
airflow-core/tests/unit/assets/test_evaluation.py | 117 +-
airflow-core/tests/unit/assets/test_manager.py | 25 +-
.../unit/cli/commands/test_api_server_command.py | 2 +
.../tests/unit/cli/commands/test_asset_command.py | 5 +-
.../unit/cli/commands/test_backfill_command.py | 6 +-
.../tests/unit/cli/commands/test_dag_command.py | 1 -
.../tests/unit/cli/commands/test_db_command.py | 2 +-
.../tests/unit/cli/commands/test_info_command.py | 2 +-
.../tests/unit/cli/commands/test_team_command.py | 28 +-
airflow-core/tests/unit/core/test_configuration.py | 302 +-
.../tests/unit/core/test_example_dags_system.py | 2 +-
.../bundles/test_dag_bundle_manager.py | 51 +
.../tests/unit/dag_processing/test_collection.py | 25 +-
.../tests/unit/dag_processing/test_processor.py | 2 +
.../tests/unit/executors/test_local_executor.py | 118 +-
.../executors/test_local_executor_check_workers.py | 4 +-
airflow-core/tests/unit/jobs/test_base_job.py | 4 +-
airflow-core/tests/unit/jobs/test_scheduler_job.py | 264 +-
airflow-core/tests/unit/jobs/test_triggerer_job.py | 6 +-
.../tests/unit/listeners/asset_listener.py | 9 +-
airflow-core/tests/unit/listeners/full_listener.py | 36 +-
.../tests/unit/listeners/lifecycle_listener.py | 26 +-
.../tests/unit/listeners/test_listeners.py | 20 +-
airflow-core/tests/unit/models/test_asset.py | 8 +-
airflow-core/tests/unit/models/test_cleartasks.py | 9 +-
airflow-core/tests/unit/models/test_connection.py | 7 +-
airflow-core/tests/unit/models/test_dag.py | 60 +-
airflow-core/tests/unit/models/test_dagrun.py | 70 +-
airflow-core/tests/unit/models/test_dagwarning.py | 3 +-
airflow-core/tests/unit/models/test_pool.py | 9 +-
.../tests/unit/models/test_renderedtifields.py | 32 +-
.../tests/unit/models/test_serialized_dag.py | 53 +
.../tests/unit/models/test_taskinstance.py | 172 +-
airflow-core/tests/unit/models/test_team.py | 50 +-
airflow-core/tests/unit/models/test_timestamp.py | 5 +-
airflow-core/tests/unit/models/test_trigger.py | 91 +
airflow-core/tests/unit/models/test_variable.py | 77 +-
airflow-core/tests/unit/models/test_xcom.py | 2 +-
.../unit/observability}/__init__.py | 0
.../unit/observability/metrics}/__init__.py | 0
.../{core => observability/metrics}/test_stats.py | 124 +-
.../unit/observability/traces}/__init__.py | 0
.../traces}/test_otel_tracer.py | 36 +-
.../tests/unit/plugins/test_plugin_ignore.py | 11 +-
.../tests/unit/plugins/test_plugins_manager.py | 2 +-
.../unit/serialization/definitions}/__init__.py | 0
.../unit/serialization/definitions/test_assets.py | 97 +
.../serialization/serializers/test_serializers.py | 14 +-
.../unit/serialization/test_dag_serialization.py | 37 +-
.../unit/serialization/test_serialized_objects.py | 39 +-
.../tests/unit/serialization/test_stringify.py | 237 +
.../ti_deps/deps/test_runnable_exec_date_dep.py | 5 +-
.../tests/unit/timetables/test_assets_timetable.py | 161 +-
.../tests/unit/timetables/test_events_timetable.py | 4 +-
airflow-core/tests/unit/utils/test_cli_util.py | 5 +-
airflow-core/tests/unit/utils/test_db_cleanup.py | 10 +-
airflow-core/tests/unit/utils/test_edgemodifier.py | 3 +-
airflow-core/tests/unit/utils/test_file.py | 65 +
.../tests/unit/utils/test_module_loading.py | 35 -
.../tests/airflowctl_tests/conftest.py | 55 +-
.../airflowctl_tests/fixtures/pools_export.json | 38 +
.../fixtures/test_connections.json | 11 +
.../airflowctl_tests/fixtures/test_pools.json | 8 +
.../airflowctl_tests/fixtures/test_variables.json | 7 +
.../fixtures/variables_export.json | 8 +
.../airflowctl_tests/test_airflowctl_commands.py | 33 +-
airflow-ctl/.pre-commit-config.yaml | 12 +-
.../src/airflowctl/api/datamodels/generated.py | 135 +-
.../tests/airflow_e2e_tests/conftest.py | 2 +-
chart/.pre-commit-config.yaml | 4 +-
chart/Chart.yaml | 18 +-
chart/README.md | 5 +
chart/docs/conf.py | 35 +-
chart/docs/index.rst | 1 +
chart/docs/production-guide.rst | 198 +
chart/docs/service-account-token-examples.rst | 434 ++
chart/files/pod-template-file.kubernetes-helm-yaml | 2 +-
chart/files/statsd-mappings.yml | 21 +
chart/newsfragments/52357.improvement.rst | 1 +
chart/newsfragments/53350.significant.rst | 3 -
chart/newsfragments/59156.significant.rst | 7 +
chart/newsfragments/59271.significant.rst | 3 +
chart/templates/NOTES.txt | 17 +-
chart/templates/_helpers.yaml | 25 +
.../api-server/api-server-deployment.yaml | 6 +-
.../api-server-hpa.yaml} | 35 +-
chart/templates/cleanup/cleanup-cronjob.yaml | 3 +-
.../templates/cleanup/cleanup-serviceaccount.yaml | 4 +-
.../dag-processor/dag-processor-deployment.yaml | 2 +-
chart/templates/jobs/create-user-job.yaml | 2 +-
chart/templates/rbac/pod-launcher-role.yaml | 1 +
chart/templates/redis/redis-statefulset.yaml | 6 +-
.../templates/scheduler/scheduler-deployment.yaml | 36 +
.../scheduler/scheduler-serviceaccount.yaml | 4 +-
.../templates/triggerer/triggerer-deployment.yaml | 5 +
.../triggerer-poddisruptionbudget.yaml} | 27 +-
.../workers/worker-celery-serviceaccount.yaml | 4 +-
chart/templates/workers/worker-deployment.yaml | 5 +
.../workers/worker-kubernetes-serviceaccount.yaml | 4 +-
.../worker-poddisruptionbudget.yaml} | 27 +-
chart/templates/workers/worker-serviceaccount.yaml | 2 +-
chart/values.schema.json | 204 +-
chart/values.yaml | 74 +-
clients/python/CHANGELOG.md | 7 +
clients/python/pyproject.toml | 2 +-
clients/python/version.txt | 2 +-
contributing-docs/02_how_to_communicate.rst | 27 +-
contributing-docs/03_contributors_quick_start.rst | 12 +-
.../03a_contributors_quick_start_beginners.rst | 4 +-
contributing-docs/05_pull_requests.rst | 4 +-
contributing-docs/09_testing.rst | 3 +
.../13_airflow_dependencies_and_extras.rst | 9 +
contributing-docs/testing/airflow_ctl_tests.rst | 47 +
contributing-docs/testing/unit_tests.rst | 755 ++-
dev/README_RELEASE_AIRFLOW.md | 150 +-
dev/README_RELEASE_PROVIDERS.md | 8 +-
dev/airflow_perf/scheduler_dag_execution_timing.py | 9 +-
dev/breeze/doc/01_installation.rst | 2 +-
dev/breeze/doc/ci/02_images.md | 4 +-
.../doc/images/output_release-management.txt | 2 +-
...tput_release-management_check-release-files.svg | 26 +-
...tput_release-management_check-release-files.txt | 2 +-
.../output_release-management_start-release.svg | 12 +-
.../output_release-management_start-release.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 2 +-
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
.../doc/images/output_testing_ui-e2e-tests.svg | 80 +-
.../doc/images/output_testing_ui-e2e-tests.txt | 2 +-
dev/breeze/pyproject.toml | 4 +-
.../src/airflow_breeze/commands/ci_commands.py | 2 +-
.../src/airflow_breeze/commands/common_options.py | 6 +-
.../airflow_breeze/commands/developer_commands.py | 15 +-
.../src/airflow_breeze/commands/main_command.py | 2 +-
.../commands/release_candidate_command.py | 81 +-
.../src/airflow_breeze/commands/release_command.py | 187 +-
.../commands/release_management_commands.py | 50 +-
.../commands/release_management_commands_config.py | 2 +-
.../src/airflow_breeze/commands/sbom_commands.py | 4 +-
.../airflow_breeze/commands/testing_commands.py | 167 +-
.../commands/testing_commands_config.py | 8 +
.../src/airflow_breeze/commands/ui_commands.py | 24 +-
dev/breeze/src/airflow_breeze/global_constants.py | 103 +-
.../src/airflow_breeze/params/shell_params.py | 8 +
.../prepare_providers/provider_documentation.py | 19 +-
.../airflow_breeze/utils/check_release_files.py | 8 +-
dev/breeze/src/airflow_breeze/utils/ci_group.py | 10 +-
.../airflow_breeze/utils/docker_command_utils.py | 6 +-
.../airflow_breeze/utils/docker_compose_utils.py | 170 +
.../src/airflow_breeze/utils/md5_build_check.py | 7 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 24 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 1 +
.../airflow_breeze/utils/provider_dependencies.py | 98 +-
.../src/airflow_breeze/utils/publish_docs_to_s3.py | 17 +-
dev/breeze/src/airflow_breeze/utils/recording.py | 11 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 36 +
.../src/airflow_breeze/utils/selective_checks.py | 108 +-
.../src/airflow_breeze/utils/shared_options.py | 31 +-
dev/breeze/src/airflow_breeze/utils/visuals.py | 2 +-
dev/breeze/tests/test_packages.py | 46 +-
.../tests/test_pytest_args_for_test_types.py | 1 +
dev/breeze/tests/test_release_candidate_command.py | 517 ++
dev/breeze/tests/test_release_command.py | 519 ++
dev/breeze/tests/test_selective_checks.py | 163 +-
dev/breeze/uv.lock | 659 +--
dev/pyproject.toml | 3 +
dev/verify_release_calendar.py | 600 +++
devel-common/pyproject.toml | 6 +-
devel-common/sphinx_design/static/custom.css | 26 +-
.../src/sphinx_exts/providers_extensions.py | 49 +-
.../sphinx_exts/templates/openlineage.rst.jinja2 | 41 +-
devel-common/src/tests_common/pytest_plugin.py | 52 +-
devel-common/src/tests_common/test_utils/db.py | 30 +-
.../src/tests_common/test_utils/providers.py | 2 +-
docker-stack-docs/build.rst | 24 +
docker-stack-docs/changelog.rst | 12 +
.../extending/add-pypi-packages-uv/Dockerfile | 2 +-
.../test_docker_compose_quick_start.py | 4 +
docs/README.md | 2 +-
docs/spelling_wordlist.txt | 4 +
generated/PYPI_README.md | 38 +-
generated/provider_metadata.json | 830 +++-
go-sdk/.pre-commit-config.yaml | 4 +-
go-sdk/pkg/edgeapi/client.go | 2 +-
.../helm_tests/airflow_aux/test_airflow_common.py | 3 +
.../helm_tests/airflow_aux/test_annotations.py | 4 +-
.../airflow_aux/test_basic_helm_chart.py | 10 +-
.../helm_tests/airflow_aux/test_cleanup_pods.py | 84 +-
.../tests/helm_tests/airflow_aux/test_configmap.py | 73 +
.../helm_tests/airflow_aux/test_create_user_job.py | 50 +
.../helm_tests/airflow_core/test_api_server.py | 9 +
.../helm_tests/airflow_core/test_dag_processor.py | 10 +-
.../helm_tests/airflow_core/test_pdb_triggerer.py | 57 +
.../helm_tests/airflow_core/test_pdb_worker.py | 57 +
.../helm_tests/airflow_core/test_scheduler.py | 664 ++-
.../helm_tests/airflow_core/test_triggerer.py | 32 +
.../tests/helm_tests/airflow_core/test_worker.py | 32 +
.../helm_tests/apiserver/test_hpa_apiserver.py | 139 +
helm-tests/tests/helm_tests/security/test_rbac.py | 2 +
.../helm_tests/security/test_security_context.py | 30 +-
kubernetes-tests/pyproject.toml | 4 +
providers/.last_release_date.txt | 2 +-
providers/.pre-commit-config.yaml | 2 +-
providers/MANAGING_PROVIDERS_LIFECYCLE.rst | 2 +-
providers/airbyte/README.rst | 8 +-
providers/airbyte/docs/changelog.rst | 11 +
providers/airbyte/docs/index.rst | 8 +-
providers/airbyte/provider.yaml | 3 +-
providers/airbyte/pyproject.toml | 8 +-
.../src/airflow/providers/airbyte/__init__.py | 2 +-
.../src/airflow/providers/airbyte/hooks/airbyte.py | 3 +-
.../airflow/providers/airbyte/operators/airbyte.py | 3 +-
.../airflow/providers/airbyte/sensors/airbyte.py | 3 +-
.../tests/unit/airbyte/hooks/test_airbyte.py | 2 +-
.../tests/unit/airbyte/sensors/test_airbyte.py | 2 +-
providers/alibaba/README.rst | 8 +-
providers/alibaba/docs/changelog.rst | 11 +
providers/alibaba/docs/index.rst | 8 +-
providers/alibaba/provider.yaml | 3 +-
providers/alibaba/pyproject.toml | 8 +-
.../src/airflow/providers/alibaba/__init__.py | 2 +-
.../alibaba/cloud/hooks/analyticdb_spark.py | 3 +-
.../airflow/providers/alibaba/cloud/hooks/oss.py | 3 +-
.../alibaba/cloud/operators/analyticdb_spark.py | 3 +-
.../providers/alibaba/cloud/sensors/oss_key.py | 4 +-
.../cloud/operators/test_analyticdb_spark.py | 2 +-
.../unit/alibaba/cloud/sensors/test_oss_key.py | 2 +-
providers/amazon/README.rst | 9 +-
providers/amazon/docs/changelog.rst | 32 +
providers/amazon/docs/index.rst | 8 +-
providers/amazon/docs/operators/bedrock.rst | 12 +-
providers/amazon/provider.yaml | 3 +-
providers/amazon/pyproject.toml | 12 +-
.../src/airflow/providers/amazon/__init__.py | 2 +-
.../amazon/aws/auth_manager/avp/facade.py | 2 +-
.../src/airflow/providers/amazon/aws/bundles/s3.py | 2 +-
.../src/airflow/providers/amazon/aws/exceptions.py | 2 +-
.../aws/executors/aws_lambda/lambda_executor.py | 3 +-
.../amazon/aws/executors/batch/batch_executor.py | 3 +-
.../amazon/aws/executors/ecs/ecs_executor.py | 3 +-
.../airflow/providers/amazon/aws/hooks/athena.py | 2 +-
.../providers/amazon/aws/hooks/athena_sql.py | 2 +-
.../airflow/providers/amazon/aws/hooks/base_aws.py | 8 +-
.../providers/amazon/aws/hooks/batch_client.py | 2 +-
.../providers/amazon/aws/hooks/batch_waiters.py | 2 +-
.../airflow/providers/amazon/aws/hooks/chime.py | 2 +-
.../providers/amazon/aws/hooks/comprehend.py | 2 +-
.../airflow/providers/amazon/aws/hooks/datasync.py | 4 +-
.../airflow/providers/amazon/aws/hooks/dynamodb.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/ec2.py | 2 +-
.../aws/hooks/elasticache_replication_group.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/emr.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/glue.py | 11 +-
.../providers/amazon/aws/hooks/glue_catalog.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/logs.py | 12 +-
.../providers/amazon/aws/hooks/quicksight.py | 2 +-
.../src/airflow/providers/amazon/aws/hooks/rds.py | 2 +-
.../providers/amazon/aws/hooks/redshift_sql.py | 13 +-
.../src/airflow/providers/amazon/aws/hooks/s3.py | 3 +-
.../providers/amazon/aws/hooks/sagemaker.py | 2 +-
.../amazon/aws/hooks/sagemaker_unified_studio.py | 3 +-
.../src/airflow/providers/amazon/aws/links/emr.py | 2 +-
.../amazon/aws/log/cloudwatch_task_handler.py | 70 +-
.../providers/amazon/aws/operators/appflow.py | 2 +-
.../providers/amazon/aws/operators/athena.py | 2 +-
.../providers/amazon/aws/operators/batch.py | 2 +-
.../providers/amazon/aws/operators/bedrock.py | 2 +-
.../providers/amazon/aws/operators/comprehend.py | 2 +-
.../providers/amazon/aws/operators/datasync.py | 3 +-
.../airflow/providers/amazon/aws/operators/dms.py | 2 +-
.../airflow/providers/amazon/aws/operators/ec2.py | 2 +-
.../airflow/providers/amazon/aws/operators/ecs.py | 2 +-
.../airflow/providers/amazon/aws/operators/eks.py | 3 +-
.../airflow/providers/amazon/aws/operators/emr.py | 3 +-
.../providers/amazon/aws/operators/eventbridge.py | 2 +-
.../airflow/providers/amazon/aws/operators/glue.py | 2 +-
.../providers/amazon/aws/operators/glue_crawler.py | 2 +-
.../amazon/aws/operators/glue_databrew.py | 2 +-
.../amazon/aws/operators/kinesis_analytics.py | 2 +-
.../amazon/aws/operators/lambda_function.py | 2 +-
.../airflow/providers/amazon/aws/operators/mwaa.py | 2 +-
.../providers/amazon/aws/operators/neptune.py | 2 +-
.../airflow/providers/amazon/aws/operators/rds.py | 2 +-
.../amazon/aws/operators/redshift_cluster.py | 2 +-
.../amazon/aws/operators/redshift_data.py | 2 +-
.../airflow/providers/amazon/aws/operators/s3.py | 2 +-
.../providers/amazon/aws/operators/sagemaker.py | 2 +-
.../aws/operators/sagemaker_unified_studio.py | 3 +-
.../amazon/aws/operators/step_function.py | 2 +-
.../amazon/aws/secrets/secrets_manager.py | 3 +-
.../amazon/aws/secrets/systems_manager.py | 3 +-
.../airflow/providers/amazon/aws/sensors/athena.py | 2 +-
.../airflow/providers/amazon/aws/sensors/batch.py | 2 +-
.../providers/amazon/aws/sensors/bedrock.py | 2 +-
.../providers/amazon/aws/sensors/comprehend.py | 2 +-
.../airflow/providers/amazon/aws/sensors/dms.py | 2 +-
.../airflow/providers/amazon/aws/sensors/ec2.py | 2 +-
.../airflow/providers/amazon/aws/sensors/ecs.py | 2 +-
.../airflow/providers/amazon/aws/sensors/eks.py | 3 +-
.../airflow/providers/amazon/aws/sensors/emr.py | 4 +-
.../providers/amazon/aws/sensors/glacier.py | 2 +-
.../airflow/providers/amazon/aws/sensors/glue.py | 2 +-
.../amazon/aws/sensors/glue_catalog_partition.py | 2 +-
.../providers/amazon/aws/sensors/glue_crawler.py | 2 +-
.../amazon/aws/sensors/kinesis_analytics.py | 2 +-
.../amazon/aws/sensors/lambda_function.py | 2 +-
.../airflow/providers/amazon/aws/sensors/mwaa.py | 2 +-
.../amazon/aws/sensors/opensearch_serverless.py | 2 +-
.../providers/amazon/aws/sensors/quicksight.py | 2 +-
.../airflow/providers/amazon/aws/sensors/rds.py | 2 +-
.../amazon/aws/sensors/redshift_cluster.py | 2 +-
.../src/airflow/providers/amazon/aws/sensors/s3.py | 3 +-
.../providers/amazon/aws/sensors/sagemaker.py | 2 +-
.../amazon/aws/sensors/sagemaker_unified_studio.py | 3 +-
.../airflow/providers/amazon/aws/sensors/sqs.py | 2 +-
.../providers/amazon/aws/sensors/step_function.py | 2 +-
.../providers/amazon/aws/transfers/gcs_to_s3.py | 3 +-
.../amazon/aws/transfers/redshift_to_s3.py | 3 +-
.../amazon/aws/transfers/s3_to_dynamodb.py | 3 +-
.../amazon/aws/transfers/s3_to_redshift.py | 3 +-
.../providers/amazon/aws/transfers/s3_to_sql.py | 3 +-
.../providers/amazon/aws/transfers/sql_to_s3.py | 10 +-
.../airflow/providers/amazon/aws/triggers/ecs.py | 2 +-
.../airflow/providers/amazon/aws/triggers/eks.py | 2 +-
.../providers/amazon/aws/triggers/sagemaker.py | 2 +-
.../airflow/providers/amazon/aws/triggers/sqs.py | 2 +-
.../airflow/providers/amazon/aws/utils/__init__.py | 2 +-
.../amazon/aws/utils/connection_wrapper.py | 2 +-
.../airflow/providers/amazon/aws/utils/waiter.py | 2 +-
.../amazon/aws/utils/waiter_with_logging.py | 2 +-
.../tests/system/amazon/aws/example_appflow.py | 8 +-
.../tests/system/amazon/aws/example_appflow_run.py | 1 -
.../tests/system/amazon/aws/example_athena.py | 1 -
.../system/amazon/aws/example_azure_blob_to_s3.py | 1 -
.../tests/system/amazon/aws/example_batch.py | 1 -
.../tests/system/amazon/aws/example_bedrock.py | 49 +-
.../amazon/aws/example_bedrock_batch_inference.py | 20 +-
.../aws/example_bedrock_retrieve_and_generate.py | 25 +-
.../system/amazon/aws/example_cloudformation.py | 1 -
.../tests/system/amazon/aws/example_comprehend.py | 1 -
.../aws/example_comprehend_document_classifier.py | 1 -
.../tests/system/amazon/aws/example_datasync.py | 1 -
.../amazon/tests/system/amazon/aws/example_dms.py | 1 -
.../tests/system/amazon/aws/example_dynamodb.py | 1 -
.../system/amazon/aws/example_dynamodb_to_s3.py | 3 +-
.../amazon/tests/system/amazon/aws/example_ec2.py | 1 -
.../amazon/tests/system/amazon/aws/example_ecs.py | 1 -
.../tests/system/amazon/aws/example_ecs_fargate.py | 1 -
.../system/amazon/aws/example_eks_templated.py | 1 -
.../aws/example_eks_with_fargate_in_one_step.py | 1 -
.../amazon/aws/example_eks_with_fargate_profile.py | 1 -
.../aws/example_eks_with_nodegroup_in_one_step.py | 1 -
.../amazon/aws/example_eks_with_nodegroups.py | 1 -
.../amazon/tests/system/amazon/aws/example_emr.py | 1 -
.../tests/system/amazon/aws/example_emr_eks.py | 1 -
.../amazon/aws/example_emr_notebook_execution.py | 1 -
.../system/amazon/aws/example_emr_serverless.py | 1 -
.../tests/system/amazon/aws/example_eventbridge.py | 1 -
.../tests/system/amazon/aws/example_ftp_to_s3.py | 1 -
.../tests/system/amazon/aws/example_gcs_to_s3.py | 1 -
.../system/amazon/aws/example_glacier_to_gcs.py | 1 -
.../amazon/tests/system/amazon/aws/example_glue.py | 1 -
.../system/amazon/aws/example_glue_data_quality.py | 1 -
...xample_glue_data_quality_with_recommendation.py | 1 -
.../amazon/aws/example_google_api_sheets_to_s3.py | 1 -
.../amazon/aws/example_google_api_youtube_to_s3.py | 1 -
.../system/amazon/aws/example_hive_to_dynamodb.py | 1 -
.../tests/system/amazon/aws/example_http_to_s3.py | 8 +-
.../amazon/aws/example_imap_attachment_to_s3.py | 1 -
.../system/amazon/aws/example_kinesis_analytics.py | 1 -
.../tests/system/amazon/aws/example_lambda.py | 1 -
.../tests/system/amazon/aws/example_local_to_s3.py | 1 -
.../tests/system/amazon/aws/example_mongo_to_s3.py | 1 -
.../amazon/tests/system/amazon/aws/example_mwaa.py | 2 +-
.../system/amazon/aws/example_mwaa_airflow2.py | 1 -
.../tests/system/amazon/aws/example_neptune.py | 1 -
.../tests/system/amazon/aws/example_quicksight.py | 1 -
.../tests/system/amazon/aws/example_rds_event.py | 1 -
.../tests/system/amazon/aws/example_rds_export.py | 1 -
.../system/amazon/aws/example_rds_instance.py | 1 -
.../system/amazon/aws/example_rds_snapshot.py | 1 -
.../tests/system/amazon/aws/example_redshift.py | 1 -
.../amazon/aws/example_redshift_s3_transfers.py | 1 -
.../amazon/tests/system/amazon/aws/example_s3.py | 1 -
.../system/amazon/aws/example_s3_to_dynamodb.py | 1 -
.../tests/system/amazon/aws/example_s3_to_ftp.py | 1 -
.../tests/system/amazon/aws/example_s3_to_sftp.py | 1 -
.../tests/system/amazon/aws/example_s3_to_sql.py | 1 -
.../tests/system/amazon/aws/example_sagemaker.py | 3 +-
.../amazon/aws/example_sagemaker_endpoint.py | 1 -
.../amazon/aws/example_sagemaker_notebook.py | 1 -
.../amazon/aws/example_sagemaker_pipeline.py | 1 -
.../amazon/aws/example_sagemaker_unified_studio.py | 1 -
.../system/amazon/aws/example_salesforce_to_s3.py | 1 -
.../tests/system/amazon/aws/example_sftp_to_s3.py | 1 -
.../amazon/tests/system/amazon/aws/example_sns.py | 1 -
.../tests/system/amazon/aws/example_sql_to_s3.py | 1 -
.../amazon/tests/system/amazon/aws/example_sqs.py | 1 -
.../amazon/tests/system/amazon/aws/example_ssm.py | 12 +-
.../system/amazon/aws/example_step_functions.py | 1 -
.../amazon/aws/tests/test_aws_auth_manager.py | 13 +-
.../amazon/tests/system/amazon/aws/utils/k8s.py | 6 +-
.../amazon/aws/auth_manager/avp/test_facade.py | 2 +-
.../tests/unit/amazon/aws/bundles/test_s3.py | 2 +-
.../executors/aws_lambda/test_lambda_executor.py | 2 +-
.../aws/executors/batch/test_batch_executor.py | 2 +-
.../amazon/aws/executors/ecs/test_ecs_executor.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_base_aws.py | 2 +-
.../unit/amazon/aws/hooks/test_batch_client.py | 2 +-
.../unit/amazon/aws/hooks/test_batch_waiters.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_chime.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_comprehend.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_datasync.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_dynamodb.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_ec2.py | 2 +-
.../hooks/test_elasticache_replication_group.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_emr.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_glue.py | 86 +-
.../unit/amazon/aws/hooks/test_glue_catalog.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_quicksight.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_rds.py | 2 +-
.../unit/amazon/aws/hooks/test_redshift_sql.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_s3.py | 2 +-
.../tests/unit/amazon/aws/hooks/test_sagemaker.py | 2 +-
.../aws/hooks/test_sagemaker_unified_studio.py | 2 +-
.../amazon/tests/unit/amazon/aws/hooks/test_sqs.py | 132 +-
.../unit/amazon/aws/hooks/test_step_function.py | 2 +-
.../amazon/tests/unit/amazon/aws/links/test_emr.py | 2 +-
.../amazon/aws/log/test_cloudwatch_task_handler.py | 36 +-
.../unit/amazon/aws/log/test_s3_task_handler.py | 2 +-
.../tests/unit/amazon/aws/operators/test_athena.py | 2 +-
.../tests/unit/amazon/aws/operators/test_batch.py | 2 +-
.../unit/amazon/aws/operators/test_datasync.py | 2 +-
.../tests/unit/amazon/aws/operators/test_dms.py | 2 +-
.../tests/unit/amazon/aws/operators/test_ec2.py | 2 +-
.../tests/unit/amazon/aws/operators/test_ecs.py | 2 +-
.../tests/unit/amazon/aws/operators/test_eks.py | 3 +-
.../amazon/aws/operators/test_emr_add_steps.py | 2 +-
.../amazon/aws/operators/test_emr_containers.py | 2 +-
.../aws/operators/test_emr_create_job_flow.py | 3 +-
.../aws/operators/test_emr_modify_cluster.py | 2 +-
.../aws/operators/test_emr_notebook_execution.py | 2 +-
.../amazon/aws/operators/test_emr_serverless.py | 2 +-
.../aws/operators/test_emr_terminate_job_flow.py | 2 +-
.../unit/amazon/aws/operators/test_eventbridge.py | 2 +-
.../tests/unit/amazon/aws/operators/test_glue.py | 2 +-
.../amazon/aws/operators/test_kinesis_analytics.py | 2 +-
.../amazon/aws/operators/test_lambda_function.py | 2 +-
.../unit/amazon/aws/operators/test_neptune.py | 2 +-
.../tests/unit/amazon/aws/operators/test_rds.py | 2 +-
.../amazon/aws/operators/test_redshift_cluster.py | 2 +-
.../amazon/aws/operators/test_redshift_data.py | 2 +-
.../tests/unit/amazon/aws/operators/test_s3.py | 2 +-
.../amazon/aws/operators/test_sagemaker_base.py | 2 +-
.../aws/operators/test_sagemaker_endpoint.py | 2 +-
.../operators/test_sagemaker_endpoint_config.py | 2 +-
.../amazon/aws/operators/test_sagemaker_model.py | 2 +-
.../aws/operators/test_sagemaker_pipeline.py | 2 +-
.../aws/operators/test_sagemaker_processing.py | 2 +-
.../aws/operators/test_sagemaker_training.py | 2 +-
.../aws/operators/test_sagemaker_transform.py | 2 +-
.../amazon/aws/operators/test_sagemaker_tuning.py | 2 +-
.../aws/operators/test_sagemaker_unified_studio.py | 2 +-
.../amazon/aws/operators/test_step_function.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_athena.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_batch.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_bedrock.py | 2 +-
.../unit/amazon/aws/sensors/test_comprehend.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_dms.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_ec2.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_ecs.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_eks.py | 3 +-
.../tests/unit/amazon/aws/sensors/test_emr_base.py | 2 +-
.../unit/amazon/aws/sensors/test_emr_containers.py | 2 +-
.../unit/amazon/aws/sensors/test_emr_job_flow.py | 2 +-
.../aws/sensors/test_emr_notebook_execution.py | 2 +-
.../aws/sensors/test_emr_serverless_application.py | 2 +-
.../amazon/aws/sensors/test_emr_serverless_job.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_emr_step.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_glacier.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_glue.py | 2 +-
.../aws/sensors/test_glue_catalog_partition.py | 2 +-
.../unit/amazon/aws/sensors/test_glue_crawler.py | 2 +-
.../amazon/aws/sensors/test_glue_data_quality.py | 2 +-
.../amazon/aws/sensors/test_kinesis_analytics.py | 2 +-
.../amazon/aws/sensors/test_lambda_function.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_mwaa.py | 2 +-
.../aws/sensors/test_opensearch_serverless.py | 2 +-
.../unit/amazon/aws/sensors/test_quicksight.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_rds.py | 2 +-
.../amazon/aws/sensors/test_redshift_cluster.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_s3.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_automl.py | 2 +-
.../unit/amazon/aws/sensors/test_sagemaker_base.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_endpoint.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_pipeline.py | 2 +-
.../aws/sensors/test_sagemaker_processing.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_training.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_transform.py | 2 +-
.../amazon/aws/sensors/test_sagemaker_tuning.py | 2 +-
.../aws/sensors/test_sagemaker_unified_studio.py | 2 +-
.../tests/unit/amazon/aws/sensors/test_sqs.py | 2 +-
.../unit/amazon/aws/sensors/test_step_function.py | 2 +-
.../amazon/aws/transfers/test_redshift_to_s3.py | 2 +-
.../amazon/aws/transfers/test_s3_to_dynamodb.py | 2 +-
.../amazon/aws/transfers/test_s3_to_redshift.py | 2 +-
.../unit/amazon/aws/transfers/test_s3_to_sql.py | 2 +-
.../unit/amazon/aws/transfers/test_sql_to_s3.py | 3 +-
.../tests/unit/amazon/aws/triggers/test_ecs.py | 2 +-
.../tests/unit/amazon/aws/triggers/test_eks.py | 2 +-
.../tests/unit/amazon/aws/triggers/test_glue.py | 2 +-
.../tests/unit/amazon/aws/triggers/test_neptune.py | 2 +-
.../tests/unit/amazon/aws/triggers/test_ssm.py | 2 +-
.../amazon/aws/utils/test_connection_wrapper.py | 2 +-
.../tests/unit/amazon/aws/utils/test_waiter.py | 2 +-
.../amazon/aws/utils/test_waiter_with_logging.py | 2 +-
providers/apache/beam/README.rst | 38 +-
providers/apache/beam/docs/changelog.rst | 11 +
providers/apache/beam/docs/index.rst | 27 +-
providers/apache/beam/provider.yaml | 3 +-
providers/apache/beam/pyproject.toml | 10 +-
.../src/airflow/providers/apache/beam/__init__.py | 2 +-
.../airflow/providers/apache/beam/hooks/beam.py | 4 +-
.../providers/apache/beam/operators/beam.py | 4 +-
.../beam/tests/unit/apache/beam/hooks/test_beam.py | 2 +-
.../tests/unit/apache/beam/operators/test_beam.py | 3 +-
providers/apache/druid/README.rst | 8 +-
providers/apache/druid/docs/changelog.rst | 11 +
providers/apache/druid/docs/index.rst | 8 +-
providers/apache/druid/provider.yaml | 3 +-
providers/apache/druid/pyproject.toml | 8 +-
.../src/airflow/providers/apache/druid/__init__.py | 2 +-
.../airflow/providers/apache/druid/hooks/druid.py | 3 +-
.../tests/unit/apache/druid/hooks/test_druid.py | 2 +-
providers/apache/flink/README.rst | 16 +-
providers/apache/flink/docs/changelog.rst | 11 +
providers/apache/flink/docs/index.rst | 7 +-
providers/apache/flink/provider.yaml | 3 +-
providers/apache/flink/pyproject.toml | 14 +-
.../src/airflow/providers/apache/flink/__init__.py | 2 +-
.../apache/flink/sensors/flink_kubernetes.py | 3 +-
.../apache/flink/sensors/test_flink_kubernetes.py | 2 +-
providers/apache/hdfs/README.rst | 34 +-
providers/apache/hdfs/docs/changelog.rst | 11 +
providers/apache/hdfs/docs/index.rst | 25 +-
providers/apache/hdfs/provider.yaml | 3 +-
providers/apache/hdfs/pyproject.toml | 14 +-
.../src/airflow/providers/apache/hdfs/__init__.py | 2 +-
.../airflow/providers/apache/hdfs/hooks/webhdfs.py | 3 +-
providers/apache/hive/README.rst | 8 +-
providers/apache/hive/docs/changelog.rst | 11 +
providers/apache/hive/docs/index.rst | 8 +-
providers/apache/hive/provider.yaml | 3 +-
providers/apache/hive/pyproject.toml | 8 +-
.../src/airflow/providers/apache/hive/__init__.py | 2 +-
.../airflow/providers/apache/hive/hooks/hive.py | 3 +-
.../providers/apache/hive/operators/hive_stats.py | 3 +-
.../providers/apache/hive/transfers/s3_to_hive.py | 3 +-
.../hive/tests/unit/apache/hive/hooks/test_hive.py | 3 +-
.../unit/apache/hive/operators/test_hive_stats.py | 2 +-
.../hive/sensors/test_named_hive_partition.py | 2 +-
.../unit/apache/hive/transfers/test_s3_to_hive.py | 2 +-
providers/apache/kafka/README.rst | 22 +-
providers/apache/kafka/docs/changelog.rst | 11 +
providers/apache/kafka/docs/index.rst | 21 +-
providers/apache/kafka/provider.yaml | 3 +-
providers/apache/kafka/pyproject.toml | 10 +-
.../src/airflow/providers/apache/kafka/__init__.py | 2 +-
.../providers/apache/kafka/hooks/consume.py | 2 +-
.../providers/apache/kafka/operators/consume.py | 5 +-
.../providers/apache/kafka/operators/produce.py | 5 +-
.../apache/kafka/triggers/await_message.py | 4 +-
.../unit/apache/kafka/operators/test_consume.py | 2 +-
.../tests/unit/apache/kafka/sensors/test_kafka.py | 2 +-
providers/apache/kylin/README.rst | 8 +-
providers/apache/kylin/docs/changelog.rst | 11 +
providers/apache/kylin/docs/index.rst | 8 +-
providers/apache/kylin/provider.yaml | 3 +-
providers/apache/kylin/pyproject.toml | 8 +-
.../src/airflow/providers/apache/kylin/__init__.py | 2 +-
.../airflow/providers/apache/kylin/hooks/kylin.py | 3 +-
.../providers/apache/kylin/operators/kylin_cube.py | 3 +-
.../tests/unit/apache/kylin/hooks/test_kylin.py | 2 +-
.../unit/apache/kylin/operators/test_kylin_cube.py | 2 +-
providers/apache/livy/README.rst | 8 +-
providers/apache/livy/docs/changelog.rst | 11 +
providers/apache/livy/docs/index.rst | 8 +-
providers/apache/livy/provider.yaml | 3 +-
providers/apache/livy/pyproject.toml | 8 +-
.../src/airflow/providers/apache/livy/__init__.py | 2 +-
.../airflow/providers/apache/livy/hooks/livy.py | 2 +-
.../providers/apache/livy/operators/livy.py | 3 +-
.../livy/tests/unit/apache/livy/hooks/test_livy.py | 2 +-
.../tests/unit/apache/livy/operators/test_livy.py | 2 +-
providers/apache/pig/README.rst | 26 +-
providers/apache/pig/docs/changelog.rst | 11 +
providers/apache/pig/docs/index.rst | 17 +-
providers/apache/pig/provider.yaml | 3 +-
providers/apache/pig/pyproject.toml | 14 +-
.../src/airflow/providers/apache/pig/__init__.py | 2 +-
.../src/airflow/providers/apache/pig/hooks/pig.py | 3 +-
.../pig/tests/unit/apache/pig/hooks/test_pig.py | 2 +-
providers/apache/pinot/README.rst | 30 +-
providers/apache/pinot/docs/changelog.rst | 11 +
providers/apache/pinot/docs/index.rst | 21 +-
providers/apache/pinot/provider.yaml | 3 +-
providers/apache/pinot/pyproject.toml | 14 +-
.../src/airflow/providers/apache/pinot/__init__.py | 2 +-
.../airflow/providers/apache/pinot/hooks/pinot.py | 3 +-
.../tests/unit/apache/pinot/hooks/test_pinot.py | 2 +-
providers/apache/spark/README.rst | 8 +-
providers/apache/spark/docs/changelog.rst | 11 +
providers/apache/spark/docs/index.rst | 8 +-
providers/apache/spark/provider.yaml | 3 +-
providers/apache/spark/pyproject.toml | 8 +-
.../src/airflow/providers/apache/spark/__init__.py | 2 +-
.../providers/apache/spark/hooks/spark_jdbc.py | 2 +-
.../providers/apache/spark/hooks/spark_sql.py | 3 +-
.../providers/apache/spark/hooks/spark_submit.py | 3 +-
.../unit/apache/spark/hooks/test_spark_sql.py | 2 +-
.../unit/apache/spark/hooks/test_spark_submit.py | 2 +-
providers/arangodb/README.rst | 8 +-
providers/arangodb/docs/changelog.rst | 11 +
providers/arangodb/docs/index.rst | 8 +-
providers/arangodb/provider.yaml | 3 +-
providers/arangodb/pyproject.toml | 8 +-
.../src/airflow/providers/arangodb/__init__.py | 2 +-
.../airflow/providers/arangodb/hooks/arangodb.py | 3 +-
.../providers/arangodb/operators/arangodb.py | 3 +-
.../jira/docs/.latest-doc-only-change.txt | 2 +-
providers/atlassian/jira/docs/index.rst | 2 +-
providers/atlassian/jira/pyproject.toml | 2 +-
.../airflow/providers/atlassian/jira/hooks/jira.py | 3 +-
providers/celery/README.rst | 8 +-
providers/celery/docs/changelog.rst | 15 +
providers/celery/docs/index.rst | 8 +-
providers/celery/provider.yaml | 3 +-
providers/celery/pyproject.toml | 8 +-
.../src/airflow/providers/celery/__init__.py | 2 +-
.../providers/celery/executors/celery_executor.py | 3 +-
.../celery/executors/celery_executor_utils.py | 27 +-
.../providers/celery/executors/default_celery.py | 3 +-
.../integration/celery/test_celery_executor.py | 6 +-
.../unit/celery/executors/test_celery_executor.py | 2 +-
.../unit/celery/log_handlers/test_log_handlers.py | 5 +-
providers/cloudant/README.rst | 8 +-
providers/cloudant/docs/changelog.rst | 11 +
providers/cloudant/docs/index.rst | 8 +-
providers/cloudant/provider.yaml | 3 +-
providers/cloudant/pyproject.toml | 8 +-
.../src/airflow/providers/cloudant/__init__.py | 2 +-
.../airflow/providers/cloudant/hooks/cloudant.py | 3 +-
.../tests/unit/cloudant/hooks/test_cloudant.py | 2 +-
providers/cncf/kubernetes/README.rst | 9 +-
providers/cncf/kubernetes/docs/changelog.rst | 23 +
providers/cncf/kubernetes/docs/index.rst | 9 +-
providers/cncf/kubernetes/provider.yaml | 3 +-
providers/cncf/kubernetes/pyproject.toml | 12 +-
.../airflow/providers/cncf/kubernetes/__init__.py | 2 +-
.../backcompat/backwards_compat_converters.py | 2 +-
.../cncf/kubernetes/decorators/kubernetes.py | 9 +-
.../cncf/kubernetes/decorators/kubernetes_cmd.py | 5 +-
.../providers/cncf/kubernetes/exceptions.py | 6 +
.../kubernetes/executors/kubernetes_executor.py | 3 +-
.../executors/kubernetes_executor_utils.py | 2 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 105 +-
.../cncf/kubernetes/kubernetes_helper_functions.py | 10 +-
.../kubernetes/operators/custom_object_launcher.py | 2 +-
.../providers/cncf/kubernetes/operators/job.py | 3 +-
.../providers/cncf/kubernetes/operators/kueue.py | 2 +-
.../providers/cncf/kubernetes/operators/pod.py | 109 +-
.../cncf/kubernetes/operators/resource.py | 2 +-
.../cncf/kubernetes/operators/spark_kubernetes.py | 23 +-
.../kubernetes/resource_convert/env_variable.py | 2 +-
.../cncf/kubernetes/sensors/spark_kubernetes.py | 3 +-
.../cncf/kubernetes/template_rendering.py | 2 +-
.../providers/cncf/kubernetes/triggers/pod.py | 31 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 130 +-
.../backcompat/test_backwards_compat_converters.py | 2 +-
.../cncf/kubernetes/decorators/test_kubernetes.py | 47 +
.../kubernetes/decorators/test_kubernetes_cmd.py | 23 +
.../executors/test_kubernetes_executor.py | 2 +-
.../unit/cncf/kubernetes/hooks/test_kubernetes.py | 213 +-
.../operators/test_custom_object_launcher.py | 2 +-
.../unit/cncf/kubernetes/operators/test_job.py | 3 +-
.../unit/cncf/kubernetes/operators/test_kueue.py | 2 +-
.../unit/cncf/kubernetes/operators/test_pod.py | 89 +-
.../kubernetes/operators/test_spark_kubernetes.py | 101 +-
.../kubernetes/sensors/test_spark_kubernetes.py | 2 +-
.../unit/cncf/kubernetes/triggers/test_pod.py | 71 +-
.../unit/cncf/kubernetes/utils/test_pod_manager.py | 456 +-
providers/common/compat/.pre-commit-config.yaml | 2 +-
providers/common/compat/README.rst | 6 +-
providers/common/compat/docs/changelog.rst | 13 +
providers/common/compat/docs/index.rst | 6 +-
providers/common/compat/provider.yaml | 3 +-
providers/common/compat/pyproject.toml | 6 +-
.../airflow/providers/common/compat/__init__.py | 2 +-
.../common/compat/module_loading/__init__.py} | 38 +-
.../src/airflow/providers/common/compat/sdk.py | 19 +-
.../messaging/docs/.latest-doc-only-change.txt | 2 +-
providers/common/sql/README.rst | 23 +-
providers/common/sql/docs/changelog.rst | 14 +
providers/common/sql/docs/index.rst | 8 +-
providers/common/sql/docs/operators.rst | 23 +
providers/common/sql/provider.yaml | 3 +-
providers/common/sql/pyproject.toml | 11 +-
.../src/airflow/providers/common/sql/__init__.py | 2 +-
.../src/airflow/providers/common/sql/hooks/sql.py | 12 +-
.../common/sql/operators/generic_transfer.py | 3 +-
.../airflow/providers/common/sql/operators/sql.py | 214 +-
.../airflow/providers/common/sql/sensors/sql.py | 3 +-
.../airflow/providers/common/sql/triggers/sql.py | 3 +-
.../system/common/sql/example_sql_value_check.py} | 69 +-
.../tests/unit/common/sql/operators/test_sql.py | 33 +-
.../sql/tests/unit/common/sql/sensors/test_sql.py | 2 +-
providers/databricks/README.rst | 8 +-
providers/databricks/docs/changelog.rst | 12 +
providers/databricks/docs/index.rst | 8 +-
providers/databricks/provider.yaml | 3 +-
providers/databricks/pyproject.toml | 8 +-
.../src/airflow/providers/databricks/__init__.py | 2 +-
.../src/airflow/providers/databricks/exceptions.py | 2 +-
.../providers/databricks/hooks/databricks.py | 2 +-
.../providers/databricks/hooks/databricks_base.py | 3 +-
.../providers/databricks/hooks/databricks_sql.py | 2 +-
.../providers/databricks/operators/databricks.py | 3 +-
.../databricks/operators/databricks_repos.py | 3 +-
.../databricks/operators/databricks_sql.py | 3 +-
.../databricks/operators/databricks_workflow.py | 3 +-
.../databricks/plugins/databricks_workflow.py | 18 +-
.../providers/databricks/sensors/databricks.py | 3 +-
.../databricks/sensors/databricks_partition.py | 3 +-
.../providers/databricks/sensors/databricks_sql.py | 3 +-
.../providers/databricks/utils/databricks.py | 2 +-
.../airflow/providers/databricks/utils/mixins.py | 2 +-
.../providers/databricks/utils/openlineage.py | 88 +-
.../tests/unit/databricks/hooks/test_databricks.py | 2 +-
.../unit/databricks/hooks/test_databricks_base.py | 2 +-
.../unit/databricks/hooks/test_databricks_sql.py | 5 +-
.../unit/databricks/operators/test_databricks.py | 2 +-
.../databricks/operators/test_databricks_copy.py | 2 +-
.../databricks/operators/test_databricks_repos.py | 2 +-
.../operators/test_databricks_workflow.py | 2 +-
.../databricks/plugins/test_databricks_workflow.py | 6 +-
.../unit/databricks/sensors/test_databricks.py | 2 +-
.../sensors/test_databricks_partition.py | 2 +-
.../unit/databricks/sensors/test_databricks_sql.py | 2 +-
.../tests/unit/databricks/utils/test_databricks.py | 2 +-
.../tests/unit/databricks/utils/test_mixins.py | 2 +-
.../unit/databricks/utils/test_openlineage.py | 60 +-
providers/datadog/README.rst | 8 +-
providers/datadog/docs/changelog.rst | 11 +
providers/datadog/docs/index.rst | 8 +-
providers/datadog/provider.yaml | 3 +-
providers/datadog/pyproject.toml | 8 +-
.../src/airflow/providers/datadog/__init__.py | 2 +-
.../src/airflow/providers/datadog/hooks/datadog.py | 3 +-
.../airflow/providers/datadog/sensors/datadog.py | 3 +-
.../tests/unit/datadog/hooks/test_datadog.py | 2 +-
.../tests/unit/datadog/sensors/test_datadog.py | 2 +-
providers/dbt/cloud/README.rst | 8 +-
providers/dbt/cloud/docs/changelog.rst | 17 +
providers/dbt/cloud/docs/index.rst | 10 +-
providers/dbt/cloud/provider.yaml | 3 +-
providers/dbt/cloud/pyproject.toml | 8 +-
.../src/airflow/providers/dbt/cloud/__init__.py | 2 +-
.../src/airflow/providers/dbt/cloud/hooks/dbt.py | 2 +-
.../src/airflow/providers/dbt/cloud/sensors/dbt.py | 3 +-
.../providers/dbt/cloud/utils/openlineage.py | 79 +-
.../cloud/tests/unit/dbt/cloud/hooks/test_dbt.py | 2 +-
.../tests/unit/dbt/cloud/operators/test_dbt.py | 3 +-
.../cloud/tests/unit/dbt/cloud/sensors/test_dbt.py | 5 +-
.../tests/unit/dbt/cloud/utils/test_openlineage.py | 41 +-
providers/dingding/README.rst | 8 +-
providers/dingding/docs/changelog.rst | 11 +
providers/dingding/docs/index.rst | 8 +-
providers/dingding/provider.yaml | 3 +-
providers/dingding/pyproject.toml | 8 +-
.../src/airflow/providers/dingding/__init__.py | 2 +-
.../airflow/providers/dingding/hooks/dingding.py | 2 +-
providers/discord/README.rst | 8 +-
providers/discord/docs/changelog.rst | 11 +
providers/discord/docs/index.rst | 8 +-
providers/discord/provider.yaml | 3 +-
providers/discord/pyproject.toml | 8 +-
.../src/airflow/providers/discord/__init__.py | 2 +-
.../providers/discord/operators/discord_webhook.py | 2 +-
providers/docker/README.rst | 8 +-
providers/docker/docs/changelog.rst | 18 +
providers/docker/docs/index.rst | 8 +-
providers/docker/provider.yaml | 3 +-
providers/docker/pyproject.toml | 8 +-
.../src/airflow/providers/docker/__init__.py | 2 +-
.../airflow/providers/docker/decorators/docker.py | 2 +-
.../src/airflow/providers/docker/exceptions.py | 3 +-
.../src/airflow/providers/docker/hooks/docker.py | 3 +-
.../airflow/providers/docker/operators/docker.py | 22 +-
.../providers/docker/operators/docker_swarm.py | 2 +-
.../tests/unit/docker/decorators/test_docker.py | 2 +-
.../docker/tests/unit/docker/hooks/test_docker.py | 2 +-
.../tests/unit/docker/operators/test_docker.py | 30 +-
.../unit/docker/operators/test_docker_swarm.py | 2 +-
.../docker/tests/unit/docker/test_exceptions.py | 4 +-
providers/edge3/.pre-commit-config.yaml | 6 +-
providers/edge3/README.rst | 16 +-
providers/edge3/docs/architecture.rst | 4 +-
providers/edge3/docs/changelog.rst | 28 +
providers/edge3/docs/deployment.rst | 4 +-
providers/edge3/docs/index.rst | 18 +-
providers/edge3/docs/ui_plugin.rst | 6 +-
providers/edge3/provider.yaml | 3 +-
providers/edge3/pyproject.toml | 10 +-
.../edge3/src/airflow/providers/edge3/__init__.py | 6 +-
.../src/airflow/providers/edge3/cli/api_client.py | 35 +-
.../src/airflow/providers/edge3/cli/worker.py | 28 +-
.../edge3/example_dags/integration_test.py | 2 +-
.../providers/edge3/example_dags/win_test.py | 7 +-
.../providers/edge3/executors/edge_executor.py | 117 +-
.../airflow/providers/edge3/models/edge_worker.py | 4 +-
.../edge3/openapi/edge_worker_api_v1.yaml | 808 ----
.../edge3/plugins/edge_executor_plugin.py | 231 +-
.../providers/edge3/plugins/www/package.json | 2 +-
.../src/airflow/providers/edge3/version_compat.py | 2 -
.../src/airflow/providers/edge3/worker_api/auth.py | 46 +-
.../providers/edge3/worker_api/datamodels.py | 5 +-
.../edge3/worker_api/routes/_v2_compat.py | 144 -
.../edge3/worker_api/routes/_v2_routes.py | 237 -
.../providers/edge3/worker_api/routes/health.py | 2 +-
.../providers/edge3/worker_api/routes/jobs.py | 21 +-
.../providers/edge3/worker_api/routes/logs.py | 13 +-
.../providers/edge3/worker_api/routes/worker.py | 17 +-
.../{openapi => worker_api}/v2-edge-generated.yaml | 3 +-
.../edge3/tests/unit/edge3/cli/test_worker.py | 68 +-
.../unit/edge3/executors/test_edge_executor.py | 195 +-
.../edge3/plugins/test_edge_executor_plugin.py | 48 +-
.../unit/edge3/worker_api/routes/test_jobs.py | 10 +-
.../unit/edge3/worker_api/routes/test_logs.py | 6 +-
.../tests/unit/edge3/worker_api/routes/test_ui.py | 3 +-
.../unit/edge3/worker_api/routes/test_worker.py | 16 +-
providers/edge3/www-hash.txt | 2 +-
providers/elasticsearch/README.rst | 8 +-
providers/elasticsearch/docs/changelog.rst | 11 +
providers/elasticsearch/docs/index.rst | 8 +-
providers/elasticsearch/provider.yaml | 3 +-
providers/elasticsearch/pyproject.toml | 8 +-
.../airflow/providers/elasticsearch/__init__.py | 2 +-
.../providers/elasticsearch/log/es_task_handler.py | 5 +-
providers/fab/.pre-commit-config.yaml | 2 +-
providers/fab/README.rst | 8 +-
providers/fab/docs/auth-manager/access-control.rst | 2 +-
.../fab/docs/auth-manager/api-authentication.rst | 6 +-
providers/fab/docs/auth-manager/token.rst | 122 +-
providers/fab/docs/changelog.rst | 31 +
providers/fab/docs/index.rst | 8 +-
providers/fab/provider.yaml | 3 +-
providers/fab/pyproject.toml | 8 +-
.../fab/src/airflow/providers/fab/__init__.py | 2 +-
.../auth_manager/api_fastapi/datamodels/login.py | 7 -
.../openapi/v2-fab-auth-manager-generated.yaml | 22 +-
.../fab/auth_manager/api_fastapi/routes/login.py | 15 +-
.../fab/auth_manager/api_fastapi/services/login.py | 37 +-
.../providers/fab/auth_manager/fab_auth_manager.py | 32 +-
.../providers/fab/auth_manager/models/db.py | 2 +-
.../fab/auth_manager/security_manager/override.py | 7 +-
...y => 0000_1_4_0_create_ab_tables_if_missing.py} | 0
providers/fab/src/airflow/providers/fab/www/app.py | 12 +-
.../providers/fab/www/extensions/init_security.py | 2 +-
.../airflow/providers/fab/www/package-lock.json | 190 +-
.../fab/src/airflow/providers/fab/www/package.json | 9 +-
.../auth_manager/api_fastapi/routes/test_login.py | 8 +-
.../api_fastapi/services/test_login.py | 62 +-
.../unit/fab/auth_manager/test_fab_auth_manager.py | 45 +-
.../tests/unit/fab/auth_manager/test_security.py | 2 +-
.../unit/fab/db_manager/test_fab_db_manager.py | 2 +-
providers/fab/www-hash.txt | 2 +-
providers/facebook/README.rst | 8 +-
providers/facebook/docs/changelog.rst | 11 +
providers/facebook/docs/index.rst | 8 +-
providers/facebook/provider.yaml | 3 +-
providers/facebook/pyproject.toml | 8 +-
.../src/airflow/providers/facebook/__init__.py | 2 +-
.../airflow/providers/facebook/ads/hooks/ads.py | 3 +-
providers/git/README.rst | 8 +-
providers/git/docs/changelog.rst | 17 +
providers/git/docs/index.rst | 8 +-
providers/git/provider.yaml | 3 +-
providers/git/pyproject.toml | 8 +-
.../git/src/airflow/providers/git/__init__.py | 2 +-
.../git/src/airflow/providers/git/bundles/git.py | 7 +-
.../git/src/airflow/providers/git/hooks/git.py | 3 +-
providers/git/tests/unit/git/bundles/test_git.py | 37 +-
providers/git/tests/unit/git/hooks/test_git.py | 2 +-
providers/github/README.rst | 8 +-
providers/github/docs/changelog.rst | 11 +
providers/github/docs/index.rst | 8 +-
providers/github/provider.yaml | 3 +-
providers/github/pyproject.toml | 8 +-
.../src/airflow/providers/github/__init__.py | 2 +-
.../airflow/providers/github/operators/github.py | 3 +-
.../src/airflow/providers/github/sensors/github.py | 3 +-
.../github/tests/system/github/example_github.py | 2 +-
providers/google/README.rst | 10 +-
providers/google/docs/changelog.rst | 27 +-
providers/google/docs/index.rst | 10 +-
providers/google/provider.yaml | 3 +-
providers/google/pyproject.toml | 10 +-
.../src/airflow/providers/google/__init__.py | 2 +-
.../src/airflow/providers/google/ads/hooks/ads.py | 3 +-
.../airflow/providers/google/cloud/bundles/gcs.py | 2 +-
.../providers/google/cloud/hooks/alloy_db.py | 2 +-
.../providers/google/cloud/hooks/bigquery.py | 32 +-
.../providers/google/cloud/hooks/cloud_batch.py | 2 +-
.../providers/google/cloud/hooks/cloud_build.py | 2 +-
.../providers/google/cloud/hooks/cloud_composer.py | 2 +-
.../google/cloud/hooks/cloud_memorystore.py | 2 +-
.../providers/google/cloud/hooks/cloud_run.py | 2 +-
.../providers/google/cloud/hooks/cloud_sql.py | 2 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 3 +-
.../providers/google/cloud/hooks/compute.py | 2 +-
.../providers/google/cloud/hooks/compute_ssh.py | 2 +-
.../providers/google/cloud/hooks/datacatalog.py | 3 +-
.../providers/google/cloud/hooks/dataflow.py | 4 +-
.../providers/google/cloud/hooks/dataform.py | 2 +-
.../providers/google/cloud/hooks/datafusion.py | 2 +-
.../providers/google/cloud/hooks/dataplex.py | 2 +-
.../providers/google/cloud/hooks/dataproc.py | 2 +-
.../google/cloud/hooks/dataproc_metastore.py | 2 +-
.../airflow/providers/google/cloud/hooks/dlp.py | 2 +-
.../providers/google/cloud/hooks/functions.py | 2 +-
.../airflow/providers/google/cloud/hooks/gcs.py | 4 +-
.../airflow/providers/google/cloud/hooks/gdm.py | 2 +-
.../google/cloud/hooks/kubernetes_engine.py | 2 +-
.../airflow/providers/google/cloud/hooks/looker.py | 3 +-
.../providers/google/cloud/hooks/managed_kafka.py | 2 +-
.../providers/google/cloud/hooks/mlengine.py | 2 +-
.../airflow/providers/google/cloud/hooks/pubsub.py | 3 +
.../providers/google/cloud/hooks/spanner.py | 2 +-
.../providers/google/cloud/hooks/stackdriver.py | 2 +-
.../airflow/providers/google/cloud/hooks/tasks.py | 2 +-
.../providers/google/cloud/hooks/translate.py | 2 +-
.../google/cloud/hooks/vertex_ai/auto_ml.py | 3 +-
.../cloud/hooks/vertex_ai/batch_prediction_job.py | 2 +-
.../google/cloud/hooks/vertex_ai/custom_job.py | 2 +-
.../google/cloud/hooks/vertex_ai/feature_store.py | 2 +-
.../hooks/vertex_ai/hyperparameter_tuning_job.py | 2 +-
.../google/cloud/hooks/vertex_ai/model_service.py | 2 +-
.../google/cloud/hooks/vertex_ai/pipeline_job.py | 2 +-
.../providers/google/cloud/hooks/vertex_ai/ray.py | 2 +-
.../airflow/providers/google/cloud/hooks/vision.py | 2 +-
.../providers/google/cloud/log/gcs_task_handler.py | 2 +-
.../providers/google/cloud/operators/alloy_db.py | 2 +-
.../providers/google/cloud/operators/bigquery.py | 4 +-
.../google/cloud/operators/bigquery_dts.py | 2 +-
.../providers/google/cloud/operators/bigtable.py | 2 +-
.../google/cloud/operators/cloud_batch.py | 2 +-
.../google/cloud/operators/cloud_build.py | 2 +-
.../google/cloud/operators/cloud_composer.py | 2 +-
.../google/cloud/operators/cloud_logging_sink.py | 2 +-
.../providers/google/cloud/operators/cloud_run.py | 2 +-
.../providers/google/cloud/operators/cloud_sql.py | 3 +-
.../operators/cloud_storage_transfer_service.py | 2 +-
.../providers/google/cloud/operators/compute.py | 2 +-
.../providers/google/cloud/operators/dataflow.py | 3 +-
.../providers/google/cloud/operators/datafusion.py | 2 +-
.../providers/google/cloud/operators/dataplex.py | 2 +-
.../providers/google/cloud/operators/dataproc.py | 3 +-
.../google/cloud/operators/dataproc_metastore.py | 2 +-
.../providers/google/cloud/operators/datastore.py | 2 +-
.../providers/google/cloud/operators/functions.py | 2 +-
.../providers/google/cloud/operators/gcs.py | 3 +-
.../google/cloud/operators/kubernetes_engine.py | 3 +-
.../providers/google/cloud/operators/looker.py | 2 +-
.../google/cloud/operators/managed_kafka.py | 2 +-
.../providers/google/cloud/operators/pubsub.py | 9 +-
.../providers/google/cloud/operators/spanner.py | 2 +-
.../google/cloud/operators/speech_to_text.py | 2 +-
.../google/cloud/operators/text_to_speech.py | 2 +-
.../providers/google/cloud/operators/translate.py | 2 +-
.../google/cloud/operators/translate_speech.py | 2 +-
.../operators/vertex_ai/batch_prediction_job.py | 2 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 2 +-
.../google/cloud/operators/vertex_ai/dataset.py | 2 +-
.../operators/vertex_ai/experiment_service.py | 2 +-
.../cloud/operators/vertex_ai/feature_store.py | 2 +-
.../cloud/operators/vertex_ai/generative_model.py | 3 +-
.../vertex_ai/hyperparameter_tuning_job.py | 2 +-
.../cloud/operators/vertex_ai/pipeline_job.py | 2 +-
.../google/cloud/operators/vertex_ai/ray.py | 2 +-
.../google/cloud/secrets/secret_manager.py | 5 +-
.../providers/google/cloud/sensors/bigquery.py | 4 +-
.../providers/google/cloud/sensors/bigquery_dts.py | 3 +-
.../google/cloud/sensors/cloud_composer.py | 3 +-
.../sensors/cloud_storage_transfer_service.py | 3 +-
.../providers/google/cloud/sensors/dataflow.py | 3 +-
.../providers/google/cloud/sensors/dataform.py | 3 +-
.../providers/google/cloud/sensors/datafusion.py | 3 +-
.../providers/google/cloud/sensors/dataplex.py | 3 +-
.../providers/google/cloud/sensors/dataproc.py | 3 +-
.../google/cloud/sensors/dataproc_metastore.py | 3 +-
.../airflow/providers/google/cloud/sensors/gcs.py | 3 +-
.../providers/google/cloud/sensors/looker.py | 3 +-
.../providers/google/cloud/sensors/pubsub.py | 3 +-
.../cloud/sensors/vertex_ai/feature_store.py | 3 +-
.../providers/google/cloud/sensors/workflows.py | 3 +-
.../cloud/transfers/azure_fileshare_to_gcs.py | 3 +-
.../google/cloud/transfers/bigquery_to_gcs.py | 7 +-
.../google/cloud/transfers/cassandra_to_gcs.py | 2 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 2 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 2 +-
.../providers/google/cloud/transfers/gcs_to_gcs.py | 3 +-
.../google/cloud/transfers/gcs_to_local.py | 2 +-
.../google/cloud/transfers/gcs_to_sftp.py | 2 +-
.../providers/google/cloud/transfers/s3_to_gcs.py | 2 +-
.../google/cloud/transfers/sftp_to_gcs.py | 2 +-
.../providers/google/cloud/triggers/bigquery.py | 2 +-
.../google/cloud/triggers/cloud_composer.py | 2 +-
.../providers/google/cloud/triggers/cloud_run.py | 2 +-
.../triggers/cloud_storage_transfer_service.py | 2 +-
.../providers/google/cloud/triggers/dataproc.py | 2 +-
.../google/cloud/triggers/kubernetes_engine.py | 3 +-
.../providers/google/cloud/triggers/vertex_ai.py | 2 +-
.../google/cloud/utils/credentials_provider.py | 2 +-
.../google/cloud/utils/field_sanitizer.py | 2 +-
.../google/cloud/utils/field_validator.py | 2 +-
.../providers/google/common/hooks/base_google.py | 3 +-
.../google/common/hooks/operation_helpers.py | 2 +-
.../providers/google/firebase/hooks/firestore.py | 2 +-
.../google/firebase/operators/firestore.py | 2 +-
.../providers/google/leveldb/hooks/leveldb.py | 4 +-
.../marketing_platform/hooks/campaign_manager.py | 2 +-
.../google/marketing_platform/hooks/search_ads.py | 2 +-
.../operators/analytics_admin.py | 2 +-
.../operators/campaign_manager.py | 2 +-
.../marketing_platform/sensors/display_video.py | 3 +-
.../providers/google/suite/hooks/calendar.py | 2 +-
.../airflow/providers/google/suite/hooks/sheets.py | 2 +-
.../google/suite/transfers/gcs_to_gdrive.py | 2 +-
.../cloud_build/example_cloud_build_trigger.py | 2 +-
.../cloud/composer/example_cloud_composer.py | 4 +-
.../example_dataflow_native_python_async.py | 2 +-
.../example_dataflow_sensors_deferrable.py | 2 +-
.../gen_ai/example_gen_ai_generative_model.py | 38 +-
.../google/tests/unit/google/ads/hooks/test_ads.py | 2 +-
.../tests/unit/google/cloud/hooks/test_alloy_db.py | 2 +-
.../tests/unit/google/cloud/hooks/test_bigquery.py | 8 +-
.../unit/google/cloud/hooks/test_cloud_batch.py | 2 +-
.../unit/google/cloud/hooks/test_cloud_build.py | 2 +-
.../google/cloud/hooks/test_cloud_memorystore.py | 2 +-
.../unit/google/cloud/hooks/test_cloud_run.py | 2 +-
.../unit/google/cloud/hooks/test_cloud_sql.py | 2 +-
.../hooks/test_cloud_storage_transfer_service.py | 2 +-
.../test_cloud_storage_transfer_service_async.py | 2 +-
.../tests/unit/google/cloud/hooks/test_compute.py | 2 +-
.../unit/google/cloud/hooks/test_compute_ssh.py | 2 +-
.../unit/google/cloud/hooks/test_datacatalog.py | 3 +-
.../tests/unit/google/cloud/hooks/test_dataflow.py | 2 +-
.../tests/unit/google/cloud/hooks/test_dataform.py | 2 +-
.../unit/google/cloud/hooks/test_datafusion.py | 2 +-
.../tests/unit/google/cloud/hooks/test_dataproc.py | 2 +-
.../unit/google/cloud/hooks/test_datastore.py | 2 +-
.../tests/unit/google/cloud/hooks/test_dlp.py | 2 +-
.../unit/google/cloud/hooks/test_functions.py | 2 +-
.../tests/unit/google/cloud/hooks/test_gcs.py | 2 +-
.../tests/unit/google/cloud/hooks/test_gdm.py | 2 +-
.../google/cloud/hooks/test_kubernetes_engine.py | 2 +-
.../tests/unit/google/cloud/hooks/test_looker.py | 2 +-
.../tests/unit/google/cloud/hooks/test_mlengine.py | 2 +-
.../tests/unit/google/cloud/hooks/test_os_login.py | 2 +-
.../tests/unit/google/cloud/hooks/test_pubsub.py | 50 +-
.../tests/unit/google/cloud/hooks/test_vision.py | 2 +-
.../hooks/vertex_ai/test_batch_prediction_job.py | 2 +-
.../cloud/hooks/vertex_ai/test_custom_job.py | 2 +-
.../vertex_ai/test_hyperparameter_tuning_job.py | 2 +-
.../cloud/hooks/vertex_ai/test_pipeline_job.py | 2 +-
.../unit/google/cloud/openlineage/test_utils.py | 37 +-
.../unit/google/cloud/operators/test_alloy_db.py | 2 +-
.../unit/google/cloud/operators/test_bigquery.py | 12 +-
.../unit/google/cloud/operators/test_bigtable.py | 2 +-
.../google/cloud/operators/test_cloud_batch.py | 2 +-
.../google/cloud/operators/test_cloud_build.py | 2 +-
.../google/cloud/operators/test_cloud_composer.py | 2 +-
.../cloud/operators/test_cloud_logging_sink.py | 2 +-
.../unit/google/cloud/operators/test_cloud_run.py | 2 +-
.../unit/google/cloud/operators/test_cloud_sql.py | 2 +-
.../test_cloud_storage_transfer_service.py | 2 +-
.../unit/google/cloud/operators/test_compute.py | 2 +-
.../unit/google/cloud/operators/test_dataflow.py | 3 +-
.../unit/google/cloud/operators/test_datafusion.py | 2 +-
.../unit/google/cloud/operators/test_dataplex.py | 2 +-
.../unit/google/cloud/operators/test_dataproc.py | 98 +-
.../unit/google/cloud/operators/test_functions.py | 2 +-
.../cloud/operators/test_kubernetes_engine.py | 6 +-
.../unit/google/cloud/operators/test_looker.py | 2 +-
.../unit/google/cloud/operators/test_pubsub.py | 45 +-
.../unit/google/cloud/operators/test_spanner.py | 2 +-
.../google/cloud/operators/test_speech_to_text.py | 2 +-
.../google/cloud/operators/test_text_to_speech.py | 2 +-
.../cloud/operators/test_translate_speech.py | 2 +-
.../unit/google/cloud/operators/test_vertex_ai.py | 3 +-
.../google/cloud/secrets/test_secret_manager.py | 2 +-
.../unit/google/cloud/sensors/test_bigquery.py | 5 +-
.../unit/google/cloud/sensors/test_bigquery_dts.py | 2 +-
.../unit/google/cloud/sensors/test_bigtable.py | 2 +-
.../sensors/test_cloud_storage_transfer_service.py | 2 +-
.../unit/google/cloud/sensors/test_dataflow.py | 2 +-
.../unit/google/cloud/sensors/test_dataform.py | 2 +-
.../unit/google/cloud/sensors/test_datafusion.py | 2 +-
.../unit/google/cloud/sensors/test_dataplex.py | 2 +-
.../unit/google/cloud/sensors/test_dataproc.py | 2 +-
.../cloud/sensors/test_dataproc_metastore.py | 2 +-
.../tests/unit/google/cloud/sensors/test_gcs.py | 2 +-
.../tests/unit/google/cloud/sensors/test_looker.py | 2 +-
.../tests/unit/google/cloud/sensors/test_pubsub.py | 2 +-
.../unit/google/cloud/sensors/test_vertex_ai.py | 2 +-
.../unit/google/cloud/sensors/test_workflows.py | 2 +-
.../google/cloud/transfers/test_bigquery_to_gcs.py | 29 +-
.../google/cloud/transfers/test_gcs_to_bigquery.py | 2 +-
.../unit/google/cloud/transfers/test_gcs_to_gcs.py | 3 +-
.../google/cloud/transfers/test_gcs_to_local.py | 2 +-
.../google/cloud/transfers/test_gcs_to_sftp.py | 2 +-
.../unit/google/cloud/transfers/test_s3_to_gcs.py | 2 +-
.../google/cloud/transfers/test_sftp_to_gcs.py | 2 +-
.../test_cloud_storage_transfer_service.py | 2 +-
.../unit/google/cloud/triggers/test_vertex_ai.py | 2 +-
.../unit/google/cloud/utils/gcp_authenticator.py | 2 +-
.../cloud/utils/test_credentials_provider.py | 2 +-
.../unit/google/common/hooks/test_base_google.py | 2 +-
.../unit/google/firebase/hooks/test_firestore.py | 2 +-
.../operators/test_analytics_admin.py | 2 +-
.../sensors/test_display_video.py | 2 +-
.../tests/unit/google/suite/hooks/test_sheets.py | 2 +-
.../google/suite/transfers/test_gcs_to_gdrive.py | 2 +-
.../airflow/providers/hashicorp/secrets/vault.py | 3 +-
providers/http/README.rst | 8 +-
providers/http/docs/changelog.rst | 16 +
providers/http/docs/index.rst | 8 +-
providers/http/provider.yaml | 3 +-
providers/http/pyproject.toml | 8 +-
.../http/src/airflow/providers/http/__init__.py | 2 +-
.../http/src/airflow/providers/http/exceptions.py | 2 +-
.../http/src/airflow/providers/http/hooks/http.py | 3 +-
.../src/airflow/providers/http/operators/http.py | 5 +-
.../src/airflow/providers/http/sensors/http.py | 3 +-
.../src/airflow/providers/http/triggers/http.py | 2 +-
providers/http/tests/unit/http/hooks/test_http.py | 2 +-
.../http/tests/unit/http/operators/test_http.py | 2 +-
.../http/tests/unit/http/sensors/test_http.py | 2 +-
providers/imap/README.rst | 8 +-
providers/imap/docs/changelog.rst | 11 +
providers/imap/docs/index.rst | 8 +-
providers/imap/provider.yaml | 3 +-
providers/imap/pyproject.toml | 8 +-
.../imap/src/airflow/providers/imap/__init__.py | 2 +-
.../imap/src/airflow/providers/imap/hooks/imap.py | 3 +-
providers/imap/tests/unit/imap/hooks/test_imap.py | 2 +-
providers/jdbc/README.rst | 8 +-
providers/jdbc/docs/changelog.rst | 11 +
providers/jdbc/docs/index.rst | 8 +-
providers/jdbc/provider.yaml | 3 +-
providers/jdbc/pyproject.toml | 8 +-
.../jdbc/src/airflow/providers/jdbc/__init__.py | 2 +-
.../jdbc/src/airflow/providers/jdbc/hooks/jdbc.py | 2 +-
providers/jdbc/tests/unit/jdbc/hooks/test_jdbc.py | 2 +-
providers/jenkins/README.rst | 8 +-
providers/jenkins/docs/changelog.rst | 11 +
providers/jenkins/docs/index.rst | 8 +-
providers/jenkins/provider.yaml | 3 +-
providers/jenkins/pyproject.toml | 8 +-
.../src/airflow/providers/jenkins/__init__.py | 2 +-
.../jenkins/operators/jenkins_job_trigger.py | 3 +-
.../airflow/providers/jenkins/sensors/jenkins.py | 3 +-
.../jenkins/operators/test_jenkins_job_trigger.py | 2 +-
.../tests/unit/jenkins/sensors/test_jenkins.py | 2 +-
providers/keycloak/.pre-commit-config.yaml | 2 +-
providers/keycloak/README.rst | 38 +-
.../docs/auth-manager/manage/permissions.rst | 6 +-
providers/keycloak/docs/changelog.rst | 27 +
providers/keycloak/docs/index.rst | 43 +-
providers/keycloak/provider.yaml | 20 +-
providers/keycloak/pyproject.toml | 8 +-
.../src/airflow/providers/keycloak/__init__.py | 2 +-
.../keycloak/auth_manager/cli/commands.py | 227 +-
.../keycloak/auth_manager/cli/definition.py | 32 +-
.../providers/keycloak/auth_manager/cli/utils.py | 81 +
.../providers/keycloak/auth_manager/constants.py | 2 +
.../keycloak/auth_manager/keycloak_auth_manager.py | 70 +-
.../keycloak/auth_manager/routes/login.py | 30 +-
.../providers/keycloak/get_provider_info.py | 15 +
.../keycloak/auth_manager/cli/test_commands.py | 278 +-
.../keycloak/auth_manager/cli/test_definition.py | 85 +-
.../unit/keycloak/auth_manager/cli/test_utils.py | 61 +
.../keycloak/auth_manager/routes/test_login.py | 33 +-
.../auth_manager/test_keycloak_auth_manager.py | 96 +-
providers/microsoft/azure/README.rst | 8 +-
providers/microsoft/azure/docs/changelog.rst | 23 +
providers/microsoft/azure/docs/index.rst | 8 +-
providers/microsoft/azure/docs/logging/index.rst | 2 +-
providers/microsoft/azure/provider.yaml | 3 +-
providers/microsoft/azure/pyproject.toml | 8 +-
.../airflow/providers/microsoft/azure/__init__.py | 2 +-
.../airflow/providers/microsoft/azure/hooks/adx.py | 3 +-
.../providers/microsoft/azure/hooks/base_azure.py | 3 +-
.../providers/microsoft/azure/hooks/batch.py | 3 +-
.../microsoft/azure/hooks/container_instance.py | 2 +-
.../providers/microsoft/azure/hooks/cosmos.py | 4 +-
.../microsoft/azure/hooks/data_factory.py | 3 +-
.../providers/microsoft/azure/hooks/data_lake.py | 3 +-
.../providers/microsoft/azure/hooks/msgraph.py | 169 +-
.../providers/microsoft/azure/hooks/powerbi.py | 2 +-
.../providers/microsoft/azure/hooks/synapse.py | 3 +-
.../providers/microsoft/azure/hooks/wasb.py | 3 +-
.../providers/microsoft/azure/operators/batch.py | 3 +-
.../azure/operators/container_instances.py | 3 +-
.../microsoft/azure/operators/data_factory.py | 9 +-
.../providers/microsoft/azure/operators/msgraph.py | 25 +-
.../providers/microsoft/azure/operators/powerbi.py | 3 +-
.../providers/microsoft/azure/operators/synapse.py | 9 +-
.../providers/microsoft/azure/secrets/key_vault.py | 3 +-
.../microsoft/azure/sensors/data_factory.py | 3 +-
.../providers/microsoft/azure/sensors/msgraph.py | 3 +-
.../providers/microsoft/azure/sensors/wasb.py | 3 +-
.../microsoft/azure/transfers/local_to_adls.py | 3 +-
.../microsoft/azure/transfers/sftp_to_wasb.py | 3 +-
.../providers/microsoft/azure/triggers/msgraph.py | 2 +-
.../tests/unit/microsoft/azure/hooks/test_adx.py | 2 +-
.../unit/microsoft/azure/hooks/test_cosmos.py | 2 +-
.../microsoft/azure/hooks/test_data_factory.py | 2 +-
.../unit/microsoft/azure/hooks/test_msgraph.py | 108 +-
.../unit/microsoft/azure/hooks/test_powerbi.py | 2 +-
.../unit/microsoft/azure/hooks/test_synapse.py | 2 +-
.../tests/unit/microsoft/azure/hooks/test_wasb.py | 2 +-
.../unit/microsoft/azure/operators/test_batch.py | 2 +-
.../azure/operators/test_container_instances.py | 2 +-
.../microsoft/azure/operators/test_data_factory.py | 2 +-
.../unit/microsoft/azure/operators/test_msgraph.py | 4 +-
.../unit/microsoft/azure/operators/test_powerbi.py | 3 +-
.../microsoft/azure/operators/test_powerbi_list.py | 3 +-
.../unit/microsoft/azure/operators/test_synapse.py | 2 +-
.../microsoft/azure/sensors/test_data_factory.py | 2 +-
.../unit/microsoft/azure/sensors/test_wasb.py | 2 +-
.../azure/transfers/test_local_to_adls.py | 2 +-
.../microsoft/azure/transfers/test_sftp_to_wasb.py | 2 +-
.../unit/microsoft/azure/triggers/test_msgraph.py | 2 +-
providers/microsoft/psrp/README.rst | 8 +-
providers/microsoft/psrp/docs/changelog.rst | 11 +
providers/microsoft/psrp/docs/index.rst | 8 +-
providers/microsoft/psrp/provider.yaml | 3 +-
providers/microsoft/psrp/pyproject.toml | 8 +-
.../airflow/providers/microsoft/psrp/__init__.py | 2 +-
.../airflow/providers/microsoft/psrp/hooks/psrp.py | 3 +-
.../providers/microsoft/psrp/operators/psrp.py | 3 +-
.../tests/unit/microsoft/psrp/hooks/test_psrp.py | 2 +-
.../unit/microsoft/psrp/operators/test_psrp.py | 3 +-
providers/microsoft/winrm/README.rst | 8 +-
providers/microsoft/winrm/docs/changelog.rst | 11 +
providers/microsoft/winrm/docs/index.rst | 8 +-
providers/microsoft/winrm/provider.yaml | 3 +-
providers/microsoft/winrm/pyproject.toml | 8 +-
.../airflow/providers/microsoft/winrm/__init__.py | 2 +-
.../providers/microsoft/winrm/hooks/winrm.py | 3 +-
.../providers/microsoft/winrm/operators/winrm.py | 3 +-
.../tests/unit/microsoft/winrm/hooks/test_winrm.py | 2 +-
.../unit/microsoft/winrm/operators/test_winrm.py | 2 +-
.../tests/unit/mysql/transfers/test_s3_to_mysql.py | 8 +-
providers/neo4j/README.rst | 8 +-
providers/neo4j/docs/changelog.rst | 11 +
providers/neo4j/docs/index.rst | 8 +-
providers/neo4j/provider.yaml | 3 +-
providers/neo4j/pyproject.toml | 8 +-
.../neo4j/src/airflow/providers/neo4j/__init__.py | 2 +-
.../src/airflow/providers/neo4j/sensors/neo4j.py | 3 +-
.../neo4j/tests/unit/neo4j/sensors/test_neo4j.py | 3 +-
providers/openai/README.rst | 8 +-
providers/openai/docs/changelog.rst | 11 +
providers/openai/docs/index.rst | 8 +-
providers/openai/provider.yaml | 3 +-
providers/openai/pyproject.toml | 8 +-
.../src/airflow/providers/openai/__init__.py | 2 +-
.../src/airflow/providers/openai/exceptions.py | 2 +-
.../tests/unit/openai/operators/test_openai.py | 3 +-
providers/openfaas/README.rst | 8 +-
providers/openfaas/docs/changelog.rst | 11 +
providers/openfaas/docs/index.rst | 8 +-
providers/openfaas/provider.yaml | 3 +-
providers/openfaas/pyproject.toml | 8 +-
.../src/airflow/providers/openfaas/__init__.py | 2 +-
.../airflow/providers/openfaas/hooks/openfaas.py | 3 +-
.../tests/unit/openfaas/hooks/test_openfaas.py | 2 +-
providers/openlineage/README.rst | 8 +-
providers/openlineage/docs/changelog.rst | 15 +
providers/openlineage/docs/index.rst | 12 +-
providers/openlineage/provider.yaml | 3 +-
providers/openlineage/pyproject.toml | 12 +-
.../src/airflow/providers/openlineage/__init__.py | 2 +-
.../providers/openlineage/plugins/adapter.py | 2 +-
.../providers/openlineage/plugins/listener.py | 12 +-
.../openlineage/utils/selective_enable.py | 4 +-
.../airflow/providers/openlineage/utils/utils.py | 76 +-
.../example_openlineage_defer_simple_dag.py | 10 +-
.../example_openlineage_mapped_simple_dag.py | 4 +-
...ample_openlineage_schedule_asset_or_time_dag.py | 2 +-
.../example_openlineage_trigger_dag_deferrable.py | 93 +
.../expected_events/openlineage_trigger_dag.json | 298 +-
.../openlineage_trigger_dag_deferrable.json | 660 +++
.../tests/system/openlineage/operator.py | 16 +-
.../tests/unit/openlineage/utils/test_spark.py | 31 +-
.../tests/unit/openlineage/utils/test_utils.py | 150 +-
providers/opensearch/README.rst | 8 +-
providers/opensearch/docs/changelog.rst | 11 +
providers/opensearch/docs/index.rst | 8 +-
providers/opensearch/provider.yaml | 3 +-
providers/opensearch/pyproject.toml | 8 +-
.../src/airflow/providers/opensearch/__init__.py | 2 +-
.../providers/opensearch/hooks/opensearch.py | 3 +-
.../providers/opensearch/log/os_task_handler.py | 4 +-
.../providers/opensearch/operators/opensearch.py | 3 +-
.../tests/unit/opensearch/hooks/test_opensearch.py | 2 +-
providers/pagerduty/README.rst | 8 +-
providers/pagerduty/docs/changelog.rst | 11 +
providers/pagerduty/docs/index.rst | 8 +-
providers/pagerduty/provider.yaml | 3 +-
providers/pagerduty/pyproject.toml | 8 +-
.../src/airflow/providers/pagerduty/__init__.py | 2 +-
.../airflow/providers/pagerduty/hooks/pagerduty.py | 3 +-
.../providers/pagerduty/hooks/pagerduty_events.py | 3 +-
.../system/pinecone/example_create_pod_index.py | 13 +-
.../system/pinecone/example_pinecone_cohere.py | 32 +-
.../system/pinecone/example_pinecone_openai.py | 10 +-
providers/postgres/README.rst | 8 +-
providers/postgres/docs/changelog.rst | 11 +
providers/postgres/docs/index.rst | 8 +-
providers/postgres/provider.yaml | 3 +-
providers/postgres/pyproject.toml | 8 +-
.../src/airflow/providers/postgres/__init__.py | 2 +-
.../airflow/providers/postgres/hooks/postgres.py | 7 +-
.../tests/unit/postgres/hooks/test_postgres.py | 3 +-
providers/presto/README.rst | 8 +-
providers/presto/docs/changelog.rst | 11 +
providers/presto/docs/index.rst | 8 +-
providers/presto/provider.yaml | 3 +-
providers/presto/pyproject.toml | 8 +-
.../src/airflow/providers/presto/__init__.py | 2 +-
.../src/airflow/providers/presto/hooks/presto.py | 7 +-
.../presto/tests/unit/presto/hooks/test_presto.py | 2 +-
.../tests/unit/presto/hooks/test_presto_sql.py | 2 +-
providers/redis/README.rst | 8 +-
providers/redis/docs/changelog.rst | 16 +
providers/redis/docs/index.rst | 8 +-
providers/redis/provider.yaml | 3 +-
providers/redis/pyproject.toml | 8 +-
.../redis/src/airflow/providers/redis/__init__.py | 2 +-
.../redis/triggers/redis_await_message.py | 4 +
.../redis/triggers/test_redis_await_message.py | 23 +
providers/salesforce/README.rst | 8 +-
providers/salesforce/docs/changelog.rst | 11 +
providers/salesforce/docs/index.rst | 8 +-
providers/salesforce/provider.yaml | 3 +-
providers/salesforce/pyproject.toml | 8 +-
.../src/airflow/providers/salesforce/__init__.py | 2 +-
.../tests/unit/salesforce/operators/test_bulk.py | 2 +-
providers/samba/README.rst | 8 +-
providers/samba/docs/changelog.rst | 11 +
providers/samba/docs/index.rst | 8 +-
providers/samba/provider.yaml | 3 +-
providers/samba/pyproject.toml | 8 +-
.../samba/src/airflow/providers/samba/__init__.py | 2 +-
.../providers/samba/transfers/gcs_to_samba.py | 3 +-
.../samba/tests/unit/samba/hooks/test_samba.py | 2 +-
.../unit/samba/transfers/test_gcs_to_samba.py | 2 +-
providers/segment/README.rst | 8 +-
providers/segment/docs/changelog.rst | 11 +
providers/segment/docs/index.rst | 8 +-
providers/segment/provider.yaml | 3 +-
providers/segment/pyproject.toml | 8 +-
.../src/airflow/providers/segment/__init__.py | 2 +-
.../src/airflow/providers/segment/hooks/segment.py | 3 +-
.../tests/unit/segment/hooks/test_segment.py | 2 +-
.../segment/operators/test_segment_track_event.py | 2 +-
providers/sftp/README.rst | 8 +-
providers/sftp/docs/changelog.rst | 16 +
providers/sftp/docs/index.rst | 8 +-
providers/sftp/provider.yaml | 3 +-
providers/sftp/pyproject.toml | 8 +-
.../sftp/src/airflow/providers/sftp/__init__.py | 2 +-
.../sftp/src/airflow/providers/sftp/exceptions.py | 2 +-
.../sftp/src/airflow/providers/sftp/hooks/sftp.py | 34 +-
.../src/airflow/providers/sftp/operators/sftp.py | 3 +-
.../src/airflow/providers/sftp/sensors/sftp.py | 3 +-
.../src/airflow/providers/sftp/triggers/sftp.py | 2 +-
providers/sftp/tests/unit/sftp/hooks/test_sftp.py | 63 +-
.../sftp/tests/unit/sftp/operators/test_sftp.py | 2 +-
.../sftp/tests/unit/sftp/sensors/test_sftp.py | 2 +-
.../sftp/tests/unit/sftp/triggers/test_sftp.py | 2 +-
providers/singularity/README.rst | 8 +-
providers/singularity/docs/changelog.rst | 11 +
providers/singularity/docs/index.rst | 8 +-
providers/singularity/provider.yaml | 3 +-
providers/singularity/pyproject.toml | 8 +-
.../src/airflow/providers/singularity/__init__.py | 2 +-
.../providers/singularity/operators/singularity.py | 3 +-
.../unit/singularity/operators/test_singularity.py | 2 +-
providers/slack/README.rst | 8 +-
providers/slack/docs/changelog.rst | 13 +
providers/slack/docs/index.rst | 8 +-
providers/slack/provider.yaml | 3 +-
providers/slack/pyproject.toml | 8 +-
.../slack/src/airflow/providers/slack/__init__.py | 2 +-
.../src/airflow/providers/slack/hooks/slack.py | 3 +-
.../airflow/providers/slack/hooks/slack_webhook.py | 3 +-
.../providers/slack/transfers/base_sql_to_slack.py | 3 +-
.../providers/slack/transfers/sql_to_slack.py | 4 +-
.../slack/transfers/sql_to_slack_webhook.py | 2 +-
.../slack/tests/unit/slack/hooks/test_slack.py | 2 +-
.../tests/unit/slack/hooks/test_slack_webhook.py | 2 +-
.../unit/slack/transfers/test_base_sql_to_slack.py | 2 +-
.../unit/slack/transfers/test_sql_to_slack.py | 2 +-
providers/smtp/README.rst | 8 +-
providers/smtp/docs/changelog.rst | 11 +
providers/smtp/docs/index.rst | 8 +-
providers/smtp/provider.yaml | 3 +-
providers/smtp/pyproject.toml | 8 +-
.../smtp/src/airflow/providers/smtp/__init__.py | 2 +-
.../smtp/src/airflow/providers/smtp/hooks/smtp.py | 3 +-
.../src/airflow/providers/smtp/operators/smtp.py | 3 +-
providers/smtp/tests/unit/smtp/hooks/test_smtp.py | 2 +-
providers/snowflake/README.rst | 8 +-
providers/snowflake/docs/changelog.rst | 17 +
providers/snowflake/docs/connections/snowflake.rst | 1 +
providers/snowflake/docs/index.rst | 8 +-
providers/snowflake/provider.yaml | 3 +-
providers/snowflake/pyproject.toml | 8 +-
.../src/airflow/providers/snowflake/__init__.py | 2 +-
.../airflow/providers/snowflake/hooks/snowflake.py | 12 +-
.../providers/snowflake/hooks/snowflake_sql_api.py | 3 +-
.../providers/snowflake/operators/snowflake.py | 2 +-
.../snowflake/triggers/snowflake_trigger.py | 21 +-
.../providers/snowflake/utils/openlineage.py | 88 +-
.../tests/unit/snowflake/hooks/test_snowflake.py | 110 +-
.../unit/snowflake/hooks/test_snowflake_sql_api.py | 3 +-
.../unit/snowflake/operators/test_snowflake.py | 2 +-
.../tests/unit/snowflake/utils/test_openlineage.py | 59 +-
providers/ssh/README.rst | 8 +-
providers/ssh/docs/changelog.rst | 13 +
providers/ssh/docs/index.rst | 8 +-
providers/ssh/provider.yaml | 3 +-
providers/ssh/pyproject.toml | 8 +-
.../ssh/src/airflow/providers/ssh/__init__.py | 2 +-
.../ssh/src/airflow/providers/ssh/hooks/ssh.py | 3 +-
.../ssh/src/airflow/providers/ssh/operators/ssh.py | 3 +-
providers/ssh/tests/unit/ssh/hooks/test_ssh.py | 2 +-
providers/ssh/tests/unit/ssh/operators/test_ssh.py | 8 +-
providers/standard/README.rst | 8 +-
providers/standard/docs/changelog.rst | 22 +
providers/standard/docs/index.rst | 8 +-
providers/standard/provider.yaml | 3 +-
providers/standard/pyproject.toml | 8 +-
.../src/airflow/providers/standard/__init__.py | 2 +-
.../src/airflow/providers/standard/exceptions.py | 2 +-
.../airflow/providers/standard/operators/bash.py | 7 +-
.../providers/standard/operators/datetime.py | 3 +-
.../airflow/providers/standard/operators/hitl.py | 7 +
.../providers/standard/operators/latest_only.py | 25 +-
.../airflow/providers/standard/operators/python.py | 3 +-
.../providers/standard/operators/trigger_dagrun.py | 24 +-
.../providers/standard/sensors/filesystem.py | 3 +-
.../providers/standard/utils/python_virtualenv.py | 3 +-
.../airflow/providers/standard/utils/skipmixin.py | 2 +-
.../tests/unit/standard/decorators/test_bash.py | 2 +-
.../decorators/test_branch_external_python.py | 2 +-
.../unit/standard/decorators/test_branch_python.py | 2 +-
.../standard/decorators/test_branch_virtualenv.py | 2 +-
.../tests/unit/standard/decorators/test_python.py | 25 +-
.../tests/unit/standard/decorators/test_sensor.py | 2 +-
.../unit/standard/decorators/test_short_circuit.py | 2 +-
.../tests/unit/standard/operators/test_bash.py | 2 +-
.../standard/operators/test_branch_operator.py | 2 +-
.../tests/unit/standard/operators/test_datetime.py | 16 +-
.../tests/unit/standard/operators/test_hitl.py | 38 +-
.../operators/test_latest_only_operator.py | 2 +-
.../tests/unit/standard/operators/test_python.py | 61 +-
.../unit/standard/operators/test_trigger_dagrun.py | 5 +-
.../tests/unit/standard/operators/test_weekday.py | 10 +-
.../tests/unit/standard/sensors/test_bash.py | 2 +-
.../standard/sensors/test_external_task_sensor.py | 14 +-
.../tests/unit/standard/sensors/test_filesystem.py | 2 +-
.../tests/unit/standard/sensors/test_python.py | 2 +-
.../tests/unit/standard/sensors/test_time.py | 2 +-
.../tests/unit/standard/sensors/test_time_delta.py | 3 +-
.../tests/unit/standard/sensors/test_weekday.py | 2 +-
.../tests/unit/standard/triggers/test_hitl.py | 20 +-
.../unit/standard/utils/test_python_virtualenv.py | 48 +-
.../unit/standard/utils/test_sensor_helper.py | 14 +-
.../tests/unit/standard/utils/test_skipmixin.py | 4 +-
providers/tableau/README.rst | 8 +-
providers/tableau/docs/changelog.rst | 11 +
providers/tableau/docs/index.rst | 8 +-
providers/tableau/provider.yaml | 3 +-
providers/tableau/pyproject.toml | 8 +-
.../src/airflow/providers/tableau/__init__.py | 2 +-
.../src/airflow/providers/tableau/hooks/tableau.py | 3 +-
.../airflow/providers/tableau/operators/tableau.py | 3 +-
.../tests/unit/tableau/hooks/test_tableau.py | 2 +-
.../tests/unit/tableau/operators/test_tableau.py | 2 +-
.../tests/unit/tableau/sensors/test_tableau.py | 2 +-
providers/telegram/README.rst | 8 +-
providers/telegram/docs/changelog.rst | 11 +
providers/telegram/docs/index.rst | 8 +-
providers/telegram/provider.yaml | 3 +-
providers/telegram/pyproject.toml | 8 +-
.../src/airflow/providers/telegram/__init__.py | 2 +-
.../airflow/providers/telegram/hooks/telegram.py | 3 +-
.../providers/telegram/operators/telegram.py | 3 +-
providers/teradata/README.rst | 8 +-
providers/teradata/docs/changelog.rst | 11 +
providers/teradata/docs/index.rst | 8 +-
providers/teradata/docs/operators/tpt.rst | 133 +-
providers/teradata/provider.yaml | 3 +-
providers/teradata/pyproject.toml | 8 +-
.../src/airflow/providers/teradata/__init__.py | 2 +-
.../src/airflow/providers/teradata/hooks/bteq.py | 2 +-
.../src/airflow/providers/teradata/hooks/tpt.py | 245 +-
.../src/airflow/providers/teradata/hooks/ttu.py | 3 +-
.../teradata/operators/teradata_compute_cluster.py | 2 +-
.../airflow/providers/teradata/operators/tpt.py | 436 ++
.../teradata/triggers/teradata_compute_cluster.py | 2 +-
.../airflow/providers/teradata/utils/bteq_util.py | 2 +-
.../airflow/providers/teradata/utils/tpt_util.py | 146 +
.../tests/system/teradata/example_remote_tpt.py | 111 +-
.../teradata/tests/system/teradata/example_tpt.py | 108 +-
.../tests/system/teradata/tdload_src_file.txt | 1000 ++++
.../tests/unit/teradata/hooks/test_bteq.py | 2 +-
.../teradata/tests/unit/teradata/hooks/test_tpt.py | 284 ++
.../teradata/tests/unit/teradata/hooks/test_ttu.py | 2 +-
.../tests/unit/teradata/operators/test_teradata.py | 2 +-
.../operators/test_teradata_compute_cluster.py | 2 +-
.../tests/unit/teradata/operators/test_tpt.py | 591 ++-
.../tests/unit/teradata/utils/test_bteq_util.py | 2 +-
.../tests/unit/teradata/utils/test_tpt_util.py | 262 ++
providers/trino/README.rst | 8 +-
providers/trino/docs/changelog.rst | 11 +
providers/trino/docs/index.rst | 8 +-
providers/trino/provider.yaml | 3 +-
providers/trino/pyproject.toml | 8 +-
.../trino/src/airflow/providers/trino/__init__.py | 2 +-
.../src/airflow/providers/trino/hooks/trino.py | 7 +-
.../trino/tests/unit/trino/hooks/test_trino.py | 2 +-
.../tests/unit/vertica/hooks/test_vertica_sql.py | 2 +-
.../system/weaviate/example_weaviate_cohere.py | 9 +
.../example_weaviate_dynamic_mapping_dag.py | 9 +
.../system/weaviate/example_weaviate_openai.py | 8 +
.../system/weaviate/example_weaviate_operator.py | 10 +
.../system/weaviate/example_weaviate_using_hook.py | 9 +
.../weaviate/example_weaviate_vectorizer_dag.py | 9 +
.../example_weaviate_without_vectorizer_dag.py | 9 +
.../airflow/providers/yandex/secrets/lockbox.py | 3 +-
providers/ydb/README.rst | 36 +-
providers/ydb/docs/changelog.rst | 11 +
providers/ydb/docs/index.rst | 36 +-
providers/ydb/provider.yaml | 3 +-
providers/ydb/pyproject.toml | 8 +-
.../ydb/src/airflow/providers/ydb/__init__.py | 2 +-
.../ydb/src/airflow/providers/ydb/hooks/ydb.py | 2 +-
.../ydb/src/airflow/providers/ydb/operators/ydb.py | 2 +-
pyproject.toml | 26 +-
reproducible_build.yaml | 4 +-
.../ci/prek/check_airflowctl_command_coverage.py | 147 +
scripts/ci/prek/check_imports_in_providers.py | 2 +-
scripts/ci/prek/check_lazy_logging.py | 2 +-
.../check_providers_subpackages_all_have_init.py | 22 +-
.../prek/check_shared_distributions_structure.py | 6 +-
.../check_template_context_variable_in_sync.py | 4 +-
scripts/ci/prek/check_tests_in_right_folders.py | 1 +
scripts/ci/prek/check_version_consistency.py | 299 ++
.../ci/prek/prevent_deprecated_sqlalchemy_usage.py | 11 +
scripts/ci/prek/ruff_format.py | 2 +-
scripts/ci/prek/supported_versions.py | 2 +-
scripts/ci/prek/update_airflow_pyproject_toml.py | 8 +-
scripts/ci/prek/upgrade_important_versions.py | 456 +-
scripts/docker/entrypoint_ci.sh | 6 +-
scripts/docker/install_os_dependencies.sh | 13 +-
.../in_container/run_check_imports_in_providers.py | 11 +-
.../run_generate_openapi_spec_providers.py | 2 +-
scripts/in_container/run_migration_reference.py | 2 +-
scripts/in_container/run_schema_defaults_check.py | 5 +
scripts/tools/setup_breeze | 4 +-
shared/README.md | 3 +-
shared/configuration/pyproject.toml | 5 +-
.../src/airflow_shared/configuration/parser.py | 4 +-
.../tests/configuration/test_parser.py | 4 +-
shared/logging/pyproject.toml | 4 -
.../airflow_shared/logging/percent_formatter.py | 10 +
.../tests/logging/test_percent_formatter.py | 13 +-
.../{timezones => module_loading}/pyproject.toml | 10 +-
.../src/airflow_shared/module_loading/__init__.py | 22 +
.../tests/conftest.py | 0
.../tests/module_loading}/__init__.py | 0
.../tests/module_loading}/test_module_loading.py | 20 +-
shared/{timezones => observability}/pyproject.toml | 24 +-
.../src/airflow_shared/observability}/__init__.py | 11 +-
.../src/airflow_shared/observability/exceptions.py | 9 +-
.../observability/metrics}/__init__.py | 0
.../observability}/metrics/base_stats_logger.py | 4 +-
.../observability}/metrics/datadog_logger.py | 49 +-
.../observability}/metrics/otel_logger.py | 47 +-
.../observability}/metrics/protocols.py | 2 +-
.../observability}/metrics/statsd_logger.py | 70 +-
.../observability}/metrics/validators.py | 40 +-
.../observability}/traces/__init__.py | 0
.../observability/traces/base_tracer.py | 108 +-
.../observability}/traces/otel_tracer.py | 53 +-
.../airflow_shared/observability/traces/utils.py | 62 +
.../tests/conftest.py | 0
.../observability/tests/observability}/__init__.py | 0
.../tests/observability/metrics}/__init__.py | 0
.../observability/metrics}/test_otel_logger.py | 9 +-
.../tests/observability}/test_otel_utils.py | 8 +-
.../{timezones => secrets_backend}/pyproject.toml | 9 +-
.../airflow_shared/secrets_backend}/__init__.py | 6 +-
.../src/airflow_shared/secrets_backend/base.py | 83 +-
.../secrets_backend/tests/conftest.py | 29 +-
.../tests/secrets_backend}/__init__.py | 0
.../tests/secrets_backend/test_base.py | 95 +
shared/secrets_masker/pyproject.toml | 4 -
shared/timezones/pyproject.toml | 4 -
.../src/airflow_shared/timezones/timezone.py | 26 +-
task-sdk-integration-tests/dags/test_asset_dag.py | 11 +-
task-sdk-integration-tests/dags/test_dag.py | 11 +-
task-sdk-integration-tests/docker-compose.yaml | 2 +-
.../tests/task_sdk_tests/conftest.py | 1 +
.../task_sdk_tests/test_asset_event_operations.py | 56 +-
.../tests/task_sdk_tests/test_asset_operations.py | 36 +-
.../task_sdk_tests/test_dag_run_operations.py | 4 +-
.../tests/task_sdk_tests/test_xcom_operations.py | 235 +-
task-sdk/.pre-commit-config.yaml | 2 +-
task-sdk/docs/api.rst | 26 +
task-sdk/pyproject.toml | 19 +-
task-sdk/src/airflow/sdk/__init__.py | 34 +
task-sdk/src/airflow/sdk/__init__.pyi | 24 +-
task-sdk/src/airflow/sdk/_shared/module_loading | 1 +
task-sdk/src/airflow/sdk/_shared/observability | 1 +
task-sdk/src/airflow/sdk/_shared/secrets_backend | 1 +
task-sdk/src/airflow/sdk/api/client.py | 13 +-
.../src/airflow/sdk/api/datamodels/_generated.py | 4 +-
task-sdk/src/airflow/sdk/bases/decorator.py | 3 +-
task-sdk/src/airflow/sdk/bases/operator.py | 10 +-
.../src/airflow/sdk/bases/secrets_backend.py | 5 +-
task-sdk/src/airflow/sdk/bases/timetable.py | 58 +
task-sdk/src/airflow/sdk/bases/xcom.py | 4 +-
task-sdk/src/airflow/sdk/configuration.py | 6 +-
.../src/airflow/sdk/definitions/asset/__init__.py | 252 +-
.../airflow/sdk/definitions/asset/decorators.py | 35 +-
task-sdk/src/airflow/sdk/definitions/callback.py | 2 +-
task-sdk/src/airflow/sdk/definitions/connection.py | 2 +-
task-sdk/src/airflow/sdk/definitions/dag.py | 65 +-
task-sdk/src/airflow/sdk/definitions/deadline.py | 2 +-
task-sdk/src/airflow/sdk/definitions/param.py | 46 +-
.../sdk/definitions/timetables}/__init__.py | 0
.../airflow/sdk/definitions/timetables/_cron.py | 31 +-
.../airflow/sdk/definitions/timetables/_delta.py | 30 +-
.../airflow/sdk/definitions/timetables/assets.py | 68 +
.../airflow/sdk/definitions/timetables/events.py | 68 +
.../airflow/sdk/definitions/timetables/interval.py | 54 +
.../airflow/sdk/definitions/timetables/simple.py | 39 +-
.../airflow/sdk/definitions/timetables/trigger.py | 116 +
task-sdk/src/airflow/sdk/exceptions.py | 4 +
task-sdk/src/airflow/sdk/execution_time/comms.py | 102 +-
.../sdk/execution_time/secrets/execution_api.py | 6 +-
.../src/airflow/sdk/execution_time/supervisor.py | 31 +-
.../src/airflow/sdk/execution_time/task_runner.py | 14 +-
task-sdk/src/airflow/sdk/io/fs.py | 4 +-
task-sdk/src/airflow/sdk/io/store.py | 2 +-
task-sdk/src/airflow/sdk/module_loading.py | 71 +-
.../src/airflow/sdk/observability}/__init__.py | 0
.../airflow/sdk/observability/metrics}/__init__.py | 0
.../sdk/observability/metrics/datadog_logger.py | 40 +
.../sdk/observability/metrics/otel_logger.py | 43 +
.../sdk/observability/metrics/statsd_logger.py | 63 +
.../src/airflow/sdk/observability}/stats.py | 12 +-
task-sdk/src/airflow/sdk/observability/trace.py | 122 +
.../airflow/sdk/observability/traces}/__init__.py | 0
.../sdk/observability/traces/otel_tracer.py | 30 +-
.../src/airflow/sdk/serde/__init__.py | 15 +-
.../src/airflow/sdk/serde/serializers}/__init__.py | 0
.../src/airflow/sdk/serde}/serializers/bignum.py | 4 +-
.../src/airflow/sdk/serde}/serializers/builtin.py | 4 +-
.../src/airflow/sdk/serde}/serializers/datetime.py | 8 +-
.../airflow/sdk/serde}/serializers/deltalake.py | 4 +-
.../src/airflow/sdk/serde}/serializers/iceberg.py | 4 +-
.../airflow/sdk/serde}/serializers/kubernetes.py | 4 +-
.../src/airflow/sdk/serde}/serializers/numpy.py | 4 +-
.../src/airflow/sdk/serde}/serializers/pandas.py | 4 +-
.../src/airflow/sdk/serde}/serializers/pydantic.py | 6 +-
.../src/airflow/sdk/serde}/serializers/timezone.py | 6 +-
.../src/airflow/sdk/serde/typing.py | 38 +-
task-sdk/src/airflow/sdk/timezone.py | 9 +
task-sdk/tests/task_sdk/api/test_client.py | 25 +-
task-sdk/tests/task_sdk/bases/test_decorator.py | 65 +
task-sdk/tests/task_sdk/bases/test_operator.py | 31 +
task-sdk/tests/task_sdk/definitions/test_asset.py | 54 +-
.../tests/task_sdk/definitions/test_callback.py | 2 +-
.../tests/task_sdk/definitions/test_connection.py | 2 +-
task-sdk/tests/task_sdk/definitions/test_dag.py | 4 +-
.../task_sdk/definitions/test_module_loading.py | 23 -
task-sdk/tests/task_sdk/definitions/test_param.py | 56 +-
.../tests/task_sdk/docs/test_docs_inventory.py | 3 +-
task-sdk/tests/task_sdk/docs/test_public_api.py | 2 +
.../tests/task_sdk/execution_time/test_context.py | 93 +-
.../tests/task_sdk/execution_time/test_sentry.py | 9 +-
.../task_sdk/execution_time/test_supervisor.py | 114 +-
.../task_sdk/execution_time/test_task_runner.py | 79 +
task-sdk/tests/task_sdk/io/test_path.py | 2 +-
.../tests/task_sdk/serde}/__init__.py | 0
.../tests/task_sdk/serde}/test_serde.py | 51 +-
2145 files changed, 40309 insertions(+), 18184 deletions(-)
create mode 100644 .github/workflows/ui-e2e-tests.yml
create mode 100644 airflow-core/docs/howto/performance.rst
create mode 100644 airflow-core/docs/img/change-theme/exmaple_theme_configuration_dark_mode.png
create mode 100644 airflow-core/docs/img/change-theme/exmaple_theme_configuration_light_mode.png
create mode 100644 airflow-core/newsfragments/58337.feature.rst
create mode 120000 airflow-core/src/airflow/_shared/module_loading
create mode 120000 airflow-core/src/airflow/_shared/observability
create mode 120000 airflow-core/src/airflow/_shared/secrets_backend
copy airflow-core/src/airflow/api_fastapi/{core_api/datamodels/ui/teams.py => auth/managers/simple/middleware.py} (52%)
copy providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/datamodels/login.py => airflow-core/src/airflow/api_fastapi/compat.py (75%)
copy airflow-core/src/airflow/{listeners/spec/asset.py => api_fastapi/execution_api/versions/v2025_12_08.py} (57%)
create mode 100644 airflow-core/src/airflow/migrations/versions/0000_2_6_2_squashed_migrations.py
create mode 100644 airflow-core/src/airflow/migrations/versions/0094_3_2_0_remove_team_id.py
rename airflow-core/src/airflow/{metrics => observability}/__init__.py (100%)
copy airflow-core/src/airflow/{_shared => observability/metrics}/__init__.py (100%)
create mode 100644 airflow-core/src/airflow/observability/metrics/datadog_logger.py
create mode 100644 airflow-core/src/airflow/observability/metrics/otel_logger.py
create mode 100644 airflow-core/src/airflow/observability/metrics/statsd_logger.py
copy airflow-core/src/airflow/{ => observability}/stats.py (87%)
create mode 100644 airflow-core/src/airflow/observability/trace.py
copy airflow-core/src/airflow/{api/common => observability/traces}/__init__.py (100%)
copy airflow-core/src/airflow/{listeners/spec/asset.py => observability/traces/otel_tracer.py} (54%)
create mode 100644 airflow-core/src/airflow/serialization/decoders.py
create mode 100644 airflow-core/src/airflow/serialization/definitions/assets.py
create mode 100644 airflow-core/src/airflow/serialization/encoders.py
create mode 100644 airflow-core/src/airflow/serialization/stringify.py
delete mode 100644 airflow-core/src/airflow/traces/utils.py
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/admin.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/assets.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/browse.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/common.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/components.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/dag.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/dags.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/dashboard.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/hitl.json
create mode 100644 airflow-core/src/airflow/ui/public/i18n/locales/ja/tasks.json
create mode 100644 airflow-core/src/airflow/ui/src/components/BasicTooltip.tsx
rename airflow-core/src/airflow/ui/src/components/{ui/DagWarningsModal.tsx => DAGWarningsModal.tsx} (96%)
delete mode 100644 airflow-core/src/airflow/ui/src/components/HoverTooltip.tsx
create mode 100644 airflow-core/src/airflow/ui/src/components/TriggerDag/TriggerDAGAdvancedOptions.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/Button.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/FileUpload/Dropzone.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/FileUpload/FileInput.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/FileUpload/Item.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/FileUpload/List.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/FileUpload/Root.tsx
delete mode 100644 airflow-core/src/airflow/ui/src/components/ui/InputGroup.tsx
rename airflow-core/src/airflow/ui/src/{components/ui/FileUpload/index.ts => context/ChakraCustomProvider.tsx} (57%)
create mode 100644 airflow-core/src/airflow/ui/src/pages/Dag/Code/CodeDiffViewer.tsx
create mode 100644 airflow-core/src/airflow/ui/src/pages/Dag/Code/VersionCompareSelect.tsx
rename airflow-core/src/airflow/ui/src/{components/ui/CloseButton.tsx => pages/XCom/AddXComButton.tsx} (51%)
create mode 100644 airflow-core/src/airflow/ui/src/pages/XCom/DeleteXComButton.tsx
copy airflow-core/src/airflow/ui/src/{components/DagActions/ParseDag.tsx => pages/XCom/EditXComButton.tsx} (50%)
create mode 100644 airflow-core/src/airflow/ui/src/pages/XCom/XComModal.tsx
rename airflow-core/src/airflow/ui/tests/e2e/specs/{dag-trigger.spec.ts => dags-list.spec.ts} (54%)
create mode 100644 airflow-core/tests/unit/api_fastapi/auth/managers/simple/test_middleware.py
create mode 100644 airflow-core/tests/unit/api_fastapi/common/test_types.py
copy airflow-core/{src/airflow/_shared => tests/unit/api_fastapi/execution_api/versions/v2025_11_07}/__init__.py (100%)
create mode 100644 airflow-core/tests/unit/api_fastapi/execution_api/versions/v2025_11_07/test_dag_runs.py
copy airflow-core/{src/airflow/_shared => tests/unit/observability}/__init__.py (100%)
copy airflow-core/{src/airflow/_shared => tests/unit/observability/metrics}/__init__.py (100%)
rename airflow-core/tests/unit/{core => observability/metrics}/test_stats.py (80%)
copy airflow-core/{src/airflow/_shared => tests/unit/observability/traces}/__init__.py (100%)
rename airflow-core/tests/unit/{core => observability/traces}/test_otel_tracer.py (90%)
copy airflow-core/{src/airflow/api/common => tests/unit/serialization/definitions}/__init__.py (100%)
create mode 100644 airflow-core/tests/unit/serialization/definitions/test_assets.py
create mode 100644 airflow-core/tests/unit/serialization/test_stringify.py
delete mode 100644 airflow-core/tests/unit/utils/test_module_loading.py
create mode 100644 airflow-ctl-tests/tests/airflowctl_tests/fixtures/pools_export.json
create mode 100644 airflow-ctl-tests/tests/airflowctl_tests/fixtures/test_connections.json
create mode 100644 airflow-ctl-tests/tests/airflowctl_tests/fixtures/test_pools.json
create mode 100644 airflow-ctl-tests/tests/airflowctl_tests/fixtures/test_variables.json
create mode 100644 airflow-ctl-tests/tests/airflowctl_tests/fixtures/variables_export.json
create mode 100644 chart/docs/service-account-token-examples.rst
create mode 100644 chart/newsfragments/52357.improvement.rst
delete mode 100644 chart/newsfragments/53350.significant.rst
create mode 100644 chart/newsfragments/59156.significant.rst
create mode 100644 chart/newsfragments/59271.significant.rst
copy chart/templates/{scheduler/scheduler-serviceaccount.yaml => api-server/api-server-hpa.yaml} (57%)
copy chart/templates/{cleanup/cleanup-serviceaccount.yaml => triggerer/triggerer-poddisruptionbudget.yaml} (63%)
copy chart/templates/{cleanup/cleanup-serviceaccount.yaml => workers/worker-poddisruptionbudget.yaml} (64%)
create mode 100644 contributing-docs/testing/airflow_ctl_tests.rst
create mode 100644 dev/breeze/src/airflow_breeze/utils/docker_compose_utils.py
create mode 100644 dev/breeze/tests/test_release_candidate_command.py
create mode 100644 dev/breeze/tests/test_release_command.py
create mode 100755 dev/verify_release_calendar.py
create mode 100644 helm-tests/tests/helm_tests/airflow_core/test_pdb_triggerer.py
create mode 100644 helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py
create mode 100644 helm-tests/tests/helm_tests/apiserver/test_hpa_apiserver.py
copy providers/{edge3/src/airflow/providers/edge3/version_compat.py => common/compat/src/airflow/providers/common/compat/module_loading/__init__.py} (50%)
copy providers/{pinecone/tests/system/pinecone/example_create_pod_index.py => common/sql/tests/system/common/sql/example_sql_value_check.py} (50%)
delete mode 100644 providers/edge3/src/airflow/providers/edge3/openapi/edge_worker_api_v1.yaml
delete mode 100644 providers/edge3/src/airflow/providers/edge3/worker_api/routes/_v2_compat.py
delete mode 100644 providers/edge3/src/airflow/providers/edge3/worker_api/routes/_v2_routes.py
rename providers/edge3/src/airflow/providers/edge3/{openapi => worker_api}/v2-edge-generated.yaml (99%)
rename providers/fab/src/airflow/providers/fab/migrations/versions/{0001_1_4_0_create_ab_tables_if_missing.py => 0000_1_4_0_create_ab_tables_if_missing.py} (100%)
create mode 100644 providers/keycloak/src/airflow/providers/keycloak/auth_manager/cli/utils.py
create mode 100644 providers/keycloak/tests/unit/keycloak/auth_manager/cli/test_utils.py
create mode 100644 providers/openlineage/tests/system/openlineage/example_openlineage_trigger_dag_deferrable.py
create mode 100644 providers/openlineage/tests/system/openlineage/expected_events/openlineage_trigger_dag_deferrable.json
create mode 100644 providers/teradata/tests/system/teradata/tdload_src_file.txt
create mode 100755 scripts/ci/prek/check_airflowctl_command_coverage.py
create mode 100755 scripts/ci/prek/check_version_consistency.py
copy providers/openai/src/airflow/providers/openai/exceptions.py => shared/logging/tests/logging/test_percent_formatter.py (68%)
copy shared/{timezones => module_loading}/pyproject.toml (82%)
rename airflow-core/src/airflow/utils/module_loading.py => shared/module_loading/src/airflow_shared/module_loading/__init__.py (77%)
copy shared/{secrets_masker => module_loading}/tests/conftest.py (100%)
copy {airflow-core/src/airflow/api/common => shared/module_loading/tests/module_loading}/__init__.py (100%)
copy {task-sdk/tests/task_sdk/definitions => shared/module_loading/tests/module_loading}/test_module_loading.py (72%)
copy shared/{timezones => observability}/pyproject.toml (72%)
copy {airflow-core/src/airflow/traces => shared/observability/src/airflow_shared/observability}/__init__.py (85%)
copy airflow-core/src/airflow/traces/__init__.py => shared/observability/src/airflow_shared/observability/exceptions.py (85%)
copy {airflow-core/src/airflow/_shared => shared/observability/src/airflow_shared/observability/metrics}/__init__.py (100%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/base_stats_logger.py (96%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/datadog_logger.py (80%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/otel_logger.py (92%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/protocols.py (98%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/statsd_logger.py (73%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/metrics/validators.py (91%)
copy {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/traces/__init__.py (100%)
rename airflow-core/src/airflow/traces/tracer.py => shared/observability/src/airflow_shared/observability/traces/base_tracer.py (60%)
rename {airflow-core/src/airflow => shared/observability/src/airflow_shared/observability}/traces/otel_tracer.py (92%)
create mode 100644 shared/observability/src/airflow_shared/observability/traces/utils.py
copy shared/{secrets_masker => observability}/tests/conftest.py (100%)
copy {airflow-core/src/airflow/_shared => shared/observability/tests/observability}/__init__.py (100%)
copy {airflow-core/src/airflow/_shared => shared/observability/tests/observability/metrics}/__init__.py (100%)
rename {airflow-core/tests/unit/core => shared/observability/tests/observability/metrics}/test_otel_logger.py (97%)
rename {airflow-core/tests/unit/utils => shared/observability/tests/observability}/test_otel_utils.py (98%)
copy shared/{timezones => secrets_backend}/pyproject.toml (88%)
rename {providers/edge3/src/airflow/providers/edge3/openapi => shared/secrets_backend/src/airflow_shared/secrets_backend}/__init__.py (84%)
copy airflow-core/src/airflow/secrets/base_secrets.py => shared/secrets_backend/src/airflow_shared/secrets_backend/base.py (65%)
copy scripts/ci/prek/ruff_format.py => shared/secrets_backend/tests/conftest.py (64%)
mode change 100755 => 100644
copy {airflow-core/src/airflow/api/common => shared/secrets_backend/tests/secrets_backend}/__init__.py (100%)
create mode 100644 shared/secrets_backend/tests/secrets_backend/test_base.py
create mode 120000 task-sdk/src/airflow/sdk/_shared/module_loading
create mode 120000 task-sdk/src/airflow/sdk/_shared/observability
create mode 120000 task-sdk/src/airflow/sdk/_shared/secrets_backend
rename airflow-core/src/airflow/traces/__init__.py => task-sdk/src/airflow/sdk/bases/secrets_backend.py (87%)
create mode 100644 task-sdk/src/airflow/sdk/bases/timetable.py
copy {airflow-core/src/airflow/_shared => task-sdk/src/airflow/sdk/definitions/timetables}/__init__.py (100%)
copy airflow-core/src/airflow/listeners/spec/asset.py => task-sdk/src/airflow/sdk/definitions/timetables/_cron.py (60%)
copy airflow-core/src/airflow/listeners/spec/asset.py => task-sdk/src/airflow/sdk/definitions/timetables/_delta.py (63%)
create mode 100644 task-sdk/src/airflow/sdk/definitions/timetables/assets.py
create mode 100644 task-sdk/src/airflow/sdk/definitions/timetables/events.py
create mode 100644 task-sdk/src/airflow/sdk/definitions/timetables/interval.py
copy providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/resource_convert/env_variable.py => task-sdk/src/airflow/sdk/definitions/timetables/simple.py (57%)
create mode 100644 task-sdk/src/airflow/sdk/definitions/timetables/trigger.py
copy {airflow-core/src/airflow/_shared => task-sdk/src/airflow/sdk/observability}/__init__.py (100%)
copy {airflow-core/src/airflow/_shared => task-sdk/src/airflow/sdk/observability/metrics}/__init__.py (100%)
create mode 100644 task-sdk/src/airflow/sdk/observability/metrics/datadog_logger.py
create mode 100644 task-sdk/src/airflow/sdk/observability/metrics/otel_logger.py
create mode 100644 task-sdk/src/airflow/sdk/observability/metrics/statsd_logger.py
rename {airflow-core/src/airflow => task-sdk/src/airflow/sdk/observability}/stats.py (85%)
create mode 100644 task-sdk/src/airflow/sdk/observability/trace.py
copy {airflow-core/src/airflow/api/common => task-sdk/src/airflow/sdk/observability/traces}/__init__.py (100%)
copy airflow-core/src/airflow/listeners/spec/asset.py => task-sdk/src/airflow/sdk/observability/traces/otel_tracer.py (54%)
rename airflow-core/src/airflow/serialization/serde.py => task-sdk/src/airflow/sdk/serde/__init__.py (96%)
copy {airflow-core/src/airflow/api/common => task-sdk/src/airflow/sdk/serde/serializers}/__init__.py (100%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/bignum.py (95%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/builtin.py (94%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/datetime.py (93%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/deltalake.py (96%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/iceberg.py (96%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/kubernetes.py (95%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/numpy.py (95%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/pandas.py (95%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/pydantic.py (94%)
copy {airflow-core/src/airflow/serialization => task-sdk/src/airflow/sdk/serde}/serializers/timezone.py (94%)
copy airflow-core/tests/unit/listeners/lifecycle_listener.py => task-sdk/src/airflow/sdk/serde/typing.py (55%)
create mode 100644 task-sdk/tests/task_sdk/bases/test_decorator.py
copy {airflow-core/src/airflow/_shared => task-sdk/tests/task_sdk/serde}/__init__.py (100%)
rename {airflow-core/tests/unit/serialization => task-sdk/tests/task_sdk/serde}/test_serde.py (91%)
(airflow) 01/01: glue pass openlineage params
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch glue-pass-params-v2
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 758753706b88fb891e5a43c6702f65756f1f85d6
Author: Maciej Obuchowski <ma...@datadoghq.com>
AuthorDate: Fri Dec 5 13:30:17 2025 +0100
glue pass openlineage params
Signed-off-by: Maciej Obuchowski <ma...@datadoghq.com>
---
.../airflow/providers/amazon/aws/operators/glue.py | 10 ++
.../providers/amazon/aws/utils/openlineage.py | 74 +++++++++++
.../unit/amazon/aws/utils/test_openlineage.py | 144 +++++++++++++++++++++
.../common/compat/openlineage/utils/spark.py | 6 +
.../airflow/providers/openlineage/utils/spark.py | 54 ++++++--
.../prek/check_contextmanager_class_decorators.py | 0
6 files changed, 280 insertions(+), 8 deletions(-)
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
index 536121de679..d6dbb9ccc8a 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/glue.py
@@ -36,6 +36,9 @@ from airflow.providers.amazon.aws.triggers.glue import (
)
from airflow.providers.amazon.aws.utils import validate_execute_complete_event
from airflow.providers.amazon.aws.utils.mixins import aws_template_fields
+from airflow.providers.amazon.aws.utils.openlineage import (
+ inject_parent_job_information_into_glue_script_args,
+)
from airflow.providers.common.compat.sdk import AirflowException
if TYPE_CHECKING:
@@ -139,10 +142,14 @@ class GlueJobOperator(AwsBaseOperator[GlueJobHook]):
job_poll_interval: int | float = 6,
waiter_delay: int = 60,
waiter_max_attempts: int = 75,
+ openlineage_inject_parent_job_info: bool = conf.getboolean(
+ "openlineage", "spark_inject_parent_job_info", fallback=False
+ ),
**kwargs,
):
super().__init__(**kwargs)
self.job_name = job_name
+ self._openlineage_inject_parent_job_info = openlineage_inject_parent_job_info
self.job_desc = job_desc
self.script_location = script_location
self.concurrent_run_limit = concurrent_run_limit or 1
@@ -217,6 +224,9 @@ class GlueJobOperator(AwsBaseOperator[GlueJobHook]):
:return: the current Glue job ID.
"""
+ if self._openlineage_inject_parent_job_info:
+ self.log.debug("Injecting OpenLineage parent job information into Glue script_args.")
+ self.script_args = inject_parent_job_information_into_glue_script_args(self.script_args, context)
self.log.info(
"Initializing AWS Glue Job: %s. Wait for completion: %s",
self.job_name,
diff --git a/providers/amazon/src/airflow/providers/amazon/aws/utils/openlineage.py b/providers/amazon/src/airflow/providers/amazon/aws/utils/openlineage.py
index be5703e2f6e..65946ef2869 100644
--- a/providers/amazon/src/airflow/providers/amazon/aws/utils/openlineage.py
+++ b/providers/amazon/src/airflow/providers/amazon/aws/utils/openlineage.py
@@ -17,6 +17,7 @@
# under the License.
from __future__ import annotations
+import logging
from typing import TYPE_CHECKING, Any
from airflow.providers.amazon.aws.hooks.redshift_sql import RedshiftSQLHook
@@ -28,9 +29,13 @@ from airflow.providers.common.compat.openlineage.facet import (
SchemaDatasetFacet,
SchemaDatasetFacetFields,
)
+from airflow.providers.common.compat.openlineage.utils.spark import get_parent_job_information
if TYPE_CHECKING:
from airflow.providers.amazon.aws.hooks.redshift_data import RedshiftDataHook
+ from airflow.utils.context import Context
+
+log = logging.getLogger(__name__)
def get_facets_from_redshift_table(
@@ -136,3 +141,72 @@ def get_identity_column_lineage_facet(
}
)
return column_lineage_facet
+
+
+def inject_parent_job_information_into_glue_script_args(
+ script_args: dict[str, Any], context: Context
+) -> dict[str, Any]:
+ """
+ Inject OpenLineage parent job info into Glue script_args via --conf.
+
+ The parent job information is injected as Spark configuration properties via
+ the ``--conf`` argument. This uses the same ``spark.openlineage.*`` properties
+ that the OpenLineage Spark integration recognizes.
+
+ Note: We use ``--conf`` instead of ``--customer-driver-env-vars`` because AWS Glue
+ requires all environment variable keys to have a ``CUSTOMER_`` prefix, which would
+ not be recognized by the OpenLineage Spark integration.
+
+ AWS Glue ``--conf`` format requires multiple properties to be formatted as::
+
+ spark.prop1=val1 --conf spark.prop2=val2 --conf spark.prop3=val3
+
+ See: https://docs.aws.amazon.com/glue/latest/dg/aws-glue-programming-etl-glue-arguments.html
+
+ Behavior:
+ - If OpenLineage provider is not available, returns script_args unchanged
+ - If user already set any ``spark.openlineage.parent*`` properties in ``--conf``,
+ skips injection to preserve user-provided values
+ - Appends to existing ``--conf`` if present (with ``--conf`` prefix)
+ - Returns new dict (does not mutate original)
+
+ Args:
+ script_args: The Glue job's script_args dict.
+ context: Airflow task context.
+
+ Returns:
+ Modified script_args with OpenLineage Spark properties injected in ``--conf``.
+ """
+ info = get_parent_job_information(context)
+ if info is None:
+ return script_args
+
+ existing_conf = script_args.get("--conf", "")
+
+ # Skip if user already configured OpenLineage parent properties
+ if "spark.openlineage.parent" in existing_conf:
+ log.debug(
+ "OpenLineage parent job properties already present in --conf. "
+ "Skipping injection to preserve user-provided values."
+ )
+ return script_args
+
+ ol_spark_properties = [
+ f"spark.openlineage.parentJobNamespace={info.parent_job_namespace}",
+ f"spark.openlineage.parentJobName={info.parent_job_name}",
+ f"spark.openlineage.parentRunId={info.parent_run_id}",
+ f"spark.openlineage.rootParentJobNamespace={info.root_parent_job_namespace}",
+ f"spark.openlineage.rootParentJobName={info.root_parent_job_name}",
+ f"spark.openlineage.rootParentRunId={info.root_parent_run_id}",
+ ]
+
+ # AWS Glue --conf format: "prop1=val1 --conf prop2=val2 --conf prop3=val3"
+ # First property has no --conf prefix, subsequent ones do
+ new_conf = " --conf ".join(ol_spark_properties)
+ if existing_conf:
+ new_conf = f"{existing_conf} --conf {new_conf}"
+
+ new_script_args = {**script_args}
+ new_script_args["--conf"] = new_conf
+
+ return new_script_args
diff --git a/providers/amazon/tests/unit/amazon/aws/utils/test_openlineage.py b/providers/amazon/tests/unit/amazon/aws/utils/test_openlineage.py
index 3790a590238..ac58420bafc 100644
--- a/providers/amazon/tests/unit/amazon/aws/utils/test_openlineage.py
+++ b/providers/amazon/tests/unit/amazon/aws/utils/test_openlineage.py
@@ -17,6 +17,7 @@
from __future__ import annotations
+from datetime import datetime
from unittest import mock
import pytest
@@ -26,6 +27,7 @@ from airflow.providers.amazon.aws.hooks.redshift_sql import RedshiftSQLHook
from airflow.providers.amazon.aws.utils.openlineage import (
get_facets_from_redshift_table,
get_identity_column_lineage_facet,
+ inject_parent_job_information_into_glue_script_args,
)
from airflow.providers.common.compat.openlineage.facet import (
ColumnLineageDatasetFacet,
@@ -34,6 +36,17 @@ from airflow.providers.common.compat.openlineage.facet import (
InputField,
)
+EXAMPLE_CONTEXT = {
+ "ti": mock.MagicMock(
+ dag_id="dag_id",
+ task_id="task_id",
+ try_number=1,
+ map_index=-1,
+ logical_date=datetime(2024, 11, 11),
+ dag_run=mock.MagicMock(logical_date=datetime(2024, 11, 11), clear_number=0),
+ )
+}
+
@mock.patch("airflow.providers.amazon.aws.hooks.redshift_sql.RedshiftSQLHook.get_records")
def test_get_facets_from_redshift_table_sql_hook(mock_get_records):
@@ -168,3 +181,134 @@ def test_get_identity_column_lineage_facet_no_input_datasets():
ValueError, match="When providing `field_names` You must provide at least one `input_dataset`."
):
get_identity_column_lineage_facet(field_names=field_names, input_datasets=input_datasets)
+
+
+class TestInjectParentJobInformationIntoGlueScriptArgs:
+ """Tests for injecting OpenLineage parent job info via --conf Spark properties."""
+
+ @mock.patch(
+ "airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information",
+ return_value=None,
+ )
+ def test_skips_when_openlineage_not_available(self, mock_get_parent_info):
+ """Returns unchanged script_args when OpenLineage is not available."""
+ result = inject_parent_job_information_into_glue_script_args({}, EXAMPLE_CONTEXT)
+ assert result == {}
+
+ @mock.patch("airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information")
+ def test_injects_into_empty_script_args(self, mock_get_parent_info):
+ """Injects --conf with Spark OpenLineage properties into empty script_args."""
+ from airflow.providers.openlineage.utils.spark import ParentJobInformation
+
+ mock_get_parent_info.return_value = ParentJobInformation(
+ parent_job_namespace="default",
+ parent_job_name="dag_id.task_id",
+ parent_run_id="uuid-123",
+ root_parent_job_namespace="default",
+ root_parent_job_name="dag_id",
+ root_parent_run_id="uuid-456",
+ )
+
+ result = inject_parent_job_information_into_glue_script_args({}, EXAMPLE_CONTEXT)
+
+ assert "--conf" in result
+ conf = result["--conf"]
+ # Verify AWS Glue --conf format: "prop1=val1 --conf prop2=val2 --conf ..."
+ assert "spark.openlineage.parentJobNamespace=default" in conf
+ assert "spark.openlineage.parentJobName=dag_id.task_id" in conf
+ assert "spark.openlineage.parentRunId=uuid-123" in conf
+ assert "spark.openlineage.rootParentJobNamespace=default" in conf
+ assert "spark.openlineage.rootParentJobName=dag_id" in conf
+ assert "spark.openlineage.rootParentRunId=uuid-456" in conf
+ # Verify the format uses " --conf " as separator between properties
+ assert " --conf " in conf
+
+ @mock.patch("airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information")
+ def test_appends_to_existing_conf(self, mock_get_parent_info):
+ """Appends OpenLineage properties to existing --conf value with --conf prefix."""
+ from airflow.providers.openlineage.utils.spark import ParentJobInformation
+
+ mock_get_parent_info.return_value = ParentJobInformation(
+ parent_job_namespace="default",
+ parent_job_name="dag.task",
+ parent_run_id="uuid-123",
+ root_parent_job_namespace="default",
+ root_parent_job_name="dag",
+ root_parent_run_id="uuid-456",
+ )
+
+ existing = {"--conf": "spark.executor.memory=4g --conf spark.driver.memory=2g"}
+ result = inject_parent_job_information_into_glue_script_args(existing, EXAMPLE_CONTEXT)
+
+ conf = result["--conf"]
+ # Original conf preserved at the beginning
+ assert conf.startswith("spark.executor.memory=4g --conf spark.driver.memory=2g")
+ # OpenLineage properties added with --conf prefix
+ assert " --conf spark.openlineage.parentJobName=dag.task" in conf
+
+ @mock.patch("airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information")
+ def test_skips_when_ol_properties_already_in_conf(self, mock_get_parent_info):
+ """Skips injection if user already set spark.openlineage.parent* properties."""
+ from airflow.providers.openlineage.utils.spark import ParentJobInformation
+
+ mock_get_parent_info.return_value = ParentJobInformation(
+ parent_job_namespace="default",
+ parent_job_name="dag.task",
+ parent_run_id="uuid-123",
+ root_parent_job_namespace="default",
+ root_parent_job_name="dag",
+ root_parent_run_id="uuid-456",
+ )
+
+ # User already set their own parent job info
+ existing = {"--conf": "spark.openlineage.parentJobName=custom_job"}
+ result = inject_parent_job_information_into_glue_script_args(existing, EXAMPLE_CONTEXT)
+
+ # Should return unchanged - user's values preserved
+ assert result == existing
+
+ @mock.patch("airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information")
+ def test_preserves_other_script_args(self, mock_get_parent_info):
+ """Preserves other script_args while adding --conf."""
+ from airflow.providers.openlineage.utils.spark import ParentJobInformation
+
+ mock_get_parent_info.return_value = ParentJobInformation(
+ parent_job_namespace="default",
+ parent_job_name="dag.task",
+ parent_run_id="uuid-123",
+ root_parent_job_namespace="default",
+ root_parent_job_name="dag",
+ root_parent_run_id="uuid-456",
+ )
+
+ existing = {"--input": "s3://bucket/input", "--output": "s3://bucket/output"}
+ result = inject_parent_job_information_into_glue_script_args(existing, EXAMPLE_CONTEXT)
+
+ assert result["--input"] == "s3://bucket/input"
+ assert result["--output"] == "s3://bucket/output"
+ assert "--conf" in result
+
+ @mock.patch("airflow.providers.amazon.aws.utils.openlineage.get_parent_job_information")
+ def test_does_not_mutate_original_script_args(self, mock_get_parent_info):
+ """Ensures original script_args dict is not mutated."""
+ from airflow.providers.openlineage.utils.spark import ParentJobInformation
+
+ mock_get_parent_info.return_value = ParentJobInformation(
+ parent_job_namespace="default",
+ parent_job_name="dag.task",
+ parent_run_id="uuid-123",
+ root_parent_job_namespace="default",
+ root_parent_job_name="dag",
+ root_parent_run_id="uuid-456",
+ )
+
+ original = {"--input": "s3://bucket/input"}
+ original_copy = dict(original)
+ result = inject_parent_job_information_into_glue_script_args(original, EXAMPLE_CONTEXT)
+
+ # Original unchanged
+ assert original == original_copy
+ # Result is different object
+ assert result is not original
+ assert "--conf" in result
+ assert "--conf" not in original
diff --git a/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py b/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py
index 1028bf3debf..414acce2081 100644
--- a/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py
+++ b/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py
@@ -24,12 +24,14 @@ log = logging.getLogger(__name__)
if TYPE_CHECKING:
from airflow.providers.openlineage.utils.spark import (
+ get_parent_job_information,
inject_parent_job_information_into_spark_properties,
inject_transport_information_into_spark_properties,
)
from airflow.sdk import Context
try:
from airflow.providers.openlineage.utils.spark import (
+ get_parent_job_information,
inject_parent_job_information_into_spark_properties,
inject_transport_information_into_spark_properties,
)
@@ -49,8 +51,12 @@ except ImportError:
)
return properties
+ def get_parent_job_information(context: Context) -> None:
+ return None
+
__all__ = [
"inject_parent_job_information_into_spark_properties",
"inject_transport_information_into_spark_properties",
+ "get_parent_job_information",
]
diff --git a/providers/openlineage/src/airflow/providers/openlineage/utils/spark.py b/providers/openlineage/src/airflow/providers/openlineage/utils/spark.py
index a92ac25eab2..438789e0c9c 100644
--- a/providers/openlineage/src/airflow/providers/openlineage/utils/spark.py
+++ b/providers/openlineage/src/airflow/providers/openlineage/utils/spark.py
@@ -18,13 +18,14 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, NamedTuple
from airflow.providers.openlineage.plugins.listener import get_openlineage_listener
from airflow.providers.openlineage.plugins.macros import (
lineage_job_name,
lineage_job_namespace,
lineage_root_job_name,
+ lineage_root_job_namespace,
lineage_root_run_id,
lineage_run_id,
)
@@ -35,6 +36,43 @@ if TYPE_CHECKING:
log = logging.getLogger(__name__)
+class ParentJobInformation(NamedTuple):
+ """Container for OpenLineage parent job information."""
+
+ parent_job_namespace: str
+ parent_job_name: str
+ parent_run_id: str
+ root_parent_job_namespace: str
+ root_parent_job_name: str
+ root_parent_run_id: str
+
+
+def get_parent_job_information(context: Context) -> ParentJobInformation | None:
+ """
+ Retrieve parent job information from the Airflow context.
+
+ This function extracts OpenLineage parent job details from the task instance,
+ which can be used by various integrations (Spark, Glue, etc.) to propagate
+ lineage information to child jobs.
+
+ Args:
+ context: The Airflow context containing task instance information.
+
+ Returns:
+ ParentJobInformation containing namespace, job name, and run IDs
+ for both parent and root parent.
+ """
+ ti = context["ti"]
+ return ParentJobInformation(
+ parent_job_namespace=lineage_job_namespace(),
+ parent_job_name=lineage_job_name(ti), # type: ignore[arg-type]
+ parent_run_id=lineage_run_id(ti), # type: ignore[arg-type]
+ root_parent_job_namespace=lineage_root_job_namespace(),
+ root_parent_job_name=lineage_root_job_name(ti), # type: ignore[arg-type]
+ root_parent_run_id=lineage_root_run_id(ti), # type: ignore[arg-type]
+ )
+
+
def _get_parent_job_information_as_spark_properties(context: Context) -> dict:
"""
Retrieve parent job information as Spark properties.
@@ -45,14 +83,14 @@ def _get_parent_job_information_as_spark_properties(context: Context) -> dict:
Returns:
Spark properties with the parent job information.
"""
- ti = context["ti"]
+ info = get_parent_job_information(context)
return {
- "spark.openlineage.parentJobNamespace": lineage_job_namespace(),
- "spark.openlineage.parentJobName": lineage_job_name(ti), # type: ignore[arg-type]
- "spark.openlineage.parentRunId": lineage_run_id(ti), # type: ignore[arg-type]
- "spark.openlineage.rootParentRunId": lineage_root_run_id(ti), # type: ignore[arg-type]
- "spark.openlineage.rootParentJobName": lineage_root_job_name(ti), # type: ignore[arg-type]
- "spark.openlineage.rootParentJobNamespace": lineage_job_namespace(),
+ "spark.openlineage.parentJobNamespace": info.parent_job_namespace,
+ "spark.openlineage.parentJobName": info.parent_job_name,
+ "spark.openlineage.parentRunId": info.parent_run_id,
+ "spark.openlineage.rootParentRunId": info.root_parent_run_id,
+ "spark.openlineage.rootParentJobName": info.root_parent_job_name,
+ "spark.openlineage.rootParentJobNamespace": info.root_parent_job_namespace,
}
diff --git a/scripts/ci/prek/check_contextmanager_class_decorators.py b/scripts/ci/prek/check_contextmanager_class_decorators.py
old mode 100644
new mode 100755