You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2024/07/09 13:45:05 UTC
(airflow) branch aip-62/s3 updated (af23b6c0c4 -> af947f3a7b)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch aip-62/s3
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard af23b6c0c4 AIP-62 support for S3Hook and S3 usage of ObjectStore
discard 1293a079bd Add HookLineageCollector that during task execution should register and hold lineage sent from hooks.
add 35faaf8b55 Switch to newer pgbouncer-exporter image as default for Helm Chart (#40318)
add ddee71f50f Remove flaky test of deferred state for served logs (#40257)
add 25132c9180 AIP 64: Add TI try history to Task Instance Details, Logs, and Gantt chart (#40304)
add d9d09638a5 BUGFIX: Make sure XComs work correctly in MSGraphAsyncOperator with paged results and dynamic task mapping (#40301)
add ca7e2a05b5 Fix replace parameter for BigQueryToPostgresOperator (#40278)
add dd69748059 Improving webserver code coverage of utils.py (#40242)
add cfb83c4b16 Resolve deprecations in the tests for `Google Dataflow` operators (#40327)
add 4fbdd07c13 fix: sqa deprecations for airflow providers (#39293)
add 3b6fba9201 Resolve deprecations in the tests for `Google Analytics` operators & hooks (#40328)
add 19fc7d2e73 feat: added OL support for AzureBlobStorageToGCSOperator in google provider package (#40290)
add 8217bec8ac Update uv to 0.2.13 (#40344)
add c3752e29dd Support disabling helm hooks on extraConfigMaps and extraSecrets (#40294)
add 3f8c1e4343 Remove double warning in CLI when config value is deprecated (#40319)
add 7cd8ac32c0 Improve UI error handling (#40350)
add e24b7c1de3 Add `[webserver]update_fab_perms` to deprecated configs (#40317)
add 06eabdd32c Improve Task TryHistory UI (#40357)
add 832099c83f Validate expected types for args for DAG, BaseOperator and TaskGroup (#40269)
add 2f2796f675 Fix `GCSToGCSOperator` behavior difference for moving single object (#40162)
add 9dc521e66d Upgrade `pip` to just-released 24.1 (#40367)
add fbcee8d01b fix: scheduler crashing with OL provider on airflow standalone (#40353)
add ed57711866 Add annotations for Redis StatefulSet (#40281)
add b59e8b7b0e Update backfill tests to use executor loader (#40354)
add e7e62bee01 Fix create a local virtualenv example (#40373)
add 6a1c574337 Resolve deprecations in `CloudComposerEnvironmentSensor` tests (#40368)
add 75d30a6af1 Resolve deprecations in the tests of `Databricks` operators (#40370)
add 806bb80ce4 Fix TriggeredDagRunOperator triggered link (#40336)
add 0019c9b35a Add common.compat provider (#40374)
add 33a14198c1 Updates to Teradata Provider (#40378)
add 6e5ae26382 Prepare docs 2nd wave June 2024 (#40273)
add 9bd81dccb1 Clarifying the breeze helper text for --backend (#40381)
add 3d18b1d43b Adding version while releasing providers using prepare-provider-documentation (#40380)
add 0842919982 Increase timeout for backfill pooled task tests (#40384)
add 83400045ea Adds a codespace button (#40386)
add 054e9fc00f Fixes hatch command in contributor doc (#40385)
add de5c751cff Bug/fix support azure managed identities in Databricks operator (#40332)
add 0779d9f74f Upgrade to latest version of hatchling as build dependency (#40387)
add 68a87f7676 Added chunk_size parameter to LocalFilesystemToGCSOperator (#40379)
add fd441983e5 Resolve sqlite deprecations in tests (#40389)
add 5df3c4efab Add few removed Task properties in AirflowRunFacet (#40371)
add 741332c0a3 Optimize startup time for parallel tests (#40388)
add 248e0a2eca Resolve postgres deprecations in tests (#40392)
add 7e5588238a remove or (#40393)
add 9901a065fc add next_kwargs to StartTriggerArgs (#40376)
add 0320172658 Show TI history for log links, more ti details and mapped tasks (#40375)
add 7904828f05 add delete index teardown to pinecone system tests (#40396)
add e8a8208477 fix typos in example of DAG parameters (#40400)
add a1e6e598ed Revert "fix: scheduler crashing with OL provider on airflow standalone (#40353)" (#40402)
add 3133c33684 docs: fix typo in upgrading.rst (#40399)
add d1a8d3f916 fix: provide stack trace under proper key in OL facet (#40372)
add 906bc96c67 fix: trim leading // character using mysql backend (#40401)
add 67e1ce69e9 Resolve snowflake deprecations in tests (#40398)
add bd18585a85 Update uv to 0.2.15 (#40411)
add 70972e617a Fix startupProbe timing comment (#40412)
add ec09600d18 Add persistent volume claim retention policy (#40271)
add 0d84cc5105 Ability to update provider version dynamically during a release (#40395)
add 19baf9c605 fix(statsd): handle unsupport operand int + str when value of tag is int (job_id) (#40407)
add c310159bc2 chore: Add param support for client_request_mfa_token (#40394)
add 5f2da71bf8 Fix typos and grammatical errors in INSTALL documentation (#40417)
add dcaf82a155 Upgrade to weaviate-client to v4 (#40194)
add 982d269d4b Fix misleading mac menu structure in howto (#40440)
add 6694f47e73 Ensure try_number incremented for empty operator (#40426)
add 5e161d9f9e Resolve mysql deprecations in tests (#40418)
add 375cdddb0e Resolve jdbc deprecations in tests (#40436)
add d5c1fc2830 Remove "experimental" banner for OTel Metrics (#40286)
add 923992d300 Move import to make PythonOperator working on Windows (#40424)
add f97c2977ee ADD prototype version dark mode for Airflow UI (#39355)
add a856ed4fb0 extra being passed to SQLalchemy (#40391)
add dd7fa8aa68 Resolve mssql deprecations in tests (#40433)
add bd64ac635c Resolve trino deprecations in tests (#40437)
add 92cb6aa2fd Update providers metadata 2024-06-27 (#40449)
add e4c125fc6a add timeout to influx db (#40439)
add d0e4b8d959 Refactor DAG.dataset_triggers into the timetable class (#39321)
add 91e6e6055b support auth key from content and from file (#40390)
add 3eea44e9a7 Chart: Add init containers for jobs (#40454)
add a62bd83188 Enable enforcing pydocstyle rule D213 in ruff. (#40448)
add ad988b2a5d Fix some docstrings for the D213 rule (#40464)
add 322130ed82 nit: fix logging level (#40461)
add a956e30bce Upgrade to latest `pip` and `uv` version (#40463)
add 8654cf67e2 Fix failure of "Add connection" introduced by influxdb change (#40469)
add 260953edca Clean up dark mode (#40466)
add 271bfa6121 Support upper bound in provider dependencies (#40465)
add 57fb776408 fix: scheduler crashing with OL provider on airflow standalone (#40459)
add 26768d9408 Update example_cloud_run.py to show job resource limit setting (#40456)
add f18f48492d Validating provider description for urls in provider list view (#40475)
add addafdb811 Adding timeout for generate-providers-metadata to tackle network issues (#40452)
add 3ca86de885 Fix editable setup for having old airflow and `main` providers (#40480)
add 2423238295 Add backward compatibility to CSRF protection of `/logout` method (#40479)
add 6c12744dd8 Adding cluster to ecs trigger event to avoid defer error (#40482)
add bbfeee4aee Reduce memory footprint of s3 key trigger (#40473)
add 8d5dbaae37 Remove todo re bucket_name decorator in s3 hook (#40485)
add 5ec81b6f95 Fix docs build re aws changelog (#40488)
add c8b7dc5ce6 Improve logging behavior of DockerOperator (#40489)
add a84d07768f ADD git-sync container lifecycle hooks (#40369)
add 3c0f9b10c1 Resolve google data pipelines deprecations in tests (#40500)
add a5294e0c25 Resolve gke deprecations in tests (#40493)
add da5fed9c5d uv version bump 0.2.18 (#40501)
add 54dfead4cc Fix latest botocore tests to not conflict with apache-beam provider (#40502)
add 930edec75b Lowercase machine in breeze (#40498)
add 34e7cab949 revamp some confusing log messages (#40334)
add 506e9f661a Add RNHTTR to committers in global_constants.py (#40510)
add dc038892d0 Add more precise description of maskiing sensitive field names (#40512)
add e3d62c3a59 Fix dag (un)pausing won't work on environment where dag files are missing (#40345)
add a37109cba8 Speeding up regex match time for custom warnings (#40513)
add 43eaa60967 test(weaviate): fix system tests (#40517)
add 07e6eb8e36 Add notes about passing secrets via environment variables (#40519)
add acdac24e3f Fix calendar view scroll (#40458)
add 47e7e254ba fix OpenLineage extraction for GCP deferrable operators (#40521)
add 4a60319e45 Update INTHEWILD.md (#40524)
add de60b7bd49 Update to latest pre-commit (#40529)
add dc5b909b22 Resolve secrets manager backend deprecations in tests (#40491)
add 39ddec9b06 code cov of sequential executor to 90 (#40530)
add da8ef6db6d test(weaviate): extract collection_name from system tests and make them unique (#40534)
add 09bbc9c62f Update example_cloud_run system test (#40540)
add 8159f6e247 fix: disable jinja2 rednering for doc_md (#40522)
add db16eeb6ef fix BigQueryInsertJobOperator's return value and openlineage extraction in deferrable mode (#40457)
add 4fb2140f39 Make Databricks operators' json parameter compatible with XComs, Jinja expression values (#40471)
add fb4bcaabf9 Update trove classifiers to the latest version as build dependency (#40542)
add 0f070154e6 Update Breeze docs to handle bad interpreter error (#40544)
add 8bdd0e72ff Add a note and an example clarifying the usage of DAG-level params (#40541)
add 9557cdbece Resolve azure provider deprecations in tests (#40132)
add eb3e199415 Resolve google secret manager deprecations in tests (#40548)
add 2f9b23d568 Recognize quotes when parsing urls in logs (#40508)
add 90a6d7f9ea Further clarify triage process (#40536)
add 0fae73dc45 Update docs for RC2 openlineage provider (#40551)
add b8aab5cf63 Use base aws classes in AWS Glue Crawlers Operators/Sensors/Triggers (#40504)
add 015ac89689 Use base aws classes in AWS Glue Data Catalog Sensors (#40492)
add 8e04ef088f Be able to remove ACL in S3 hook's copy_object (#40518)
add bd7306455e 40559: fix add project_id to submit_batch_job hook (#40560)
add 7fde239187 Add parameter for the last number of queries to the DB in DAG file processing stats (#40323)
add 5b2becbfb4 Update best-practices.rst (#40505)
add 8ced563f26 Added support of Teradata Compute Cluster Provision, Decommission, Suspend and Resume operations (#40509)
add ed6849eb60 Remove logging for "blocked" events (#40446)
add 888b4a0b18 Upgrade to another version of trove-classifier (new CUDA classifiers) (#40564)
add a8c4830d97 Fix bug in `SchedulerJobRunner._process_executor_events` (#40563)
add 7e80dc6afa Add support for query parameters to BigQueryCheckOperator (#40556) (#40558)
add dbef3e6195 Add D1 pydocstyle rules to pyproject.toml (#40569)
add c5c50cc07f Revert "Fix `GCSToGCSOperator` behavior difference for moving single object (#40162)" (#40577)
add 4d97461739 MSSQL-DOC-FIX (#40565)
add 3583329c1b Add notes about reverting the change in GCSToGCSOperator (#40579)
add 3c1120afd9 Resolve google vertex ai deprecations in tests (#40506)
add b7d0bf9800 fix OpenLineage extraction for AthenaOperator (#40545)
add 25ab6e3fc5 Resolve deprecations in `API` tests (#40249)
add ce12553aa7 uv version bump 0.2.21 (#40599)
add 93488d09f9 DataflowStartFlexTemplateOperator. Check for Dataflow job type each check cycle. (#40584)
add a984d973aa Fix import statement for DatasetOrTimetable example (#40601)
add 572b633c9d Extend list of folders which are not forced to be a subpackage to templates (#40592)
add 1225c4b155 Add exit clause suggesting --clean-build if toctree contains nonexisting _api/* reference (#40588)
add 49ded9f533 Update serializers.rst (#40597)
add 9918f2a182 Enhancing changelog generation for provider releases (#40552)
add 9c9706730c Update Google Cloud Generative Model Hooks/Operators to bring parity with Vertex AI API (#40484)
add 4edcac2445 Add tests for prevent changing main GCSToGCSOperator behavior (#40610)
add fa447c2505 Update providers metadata 2024-07-04 (#40611)
add 9d92feda7f Fix Glue system test (#40612)
add 89d2f2992f Warn user about unused replace parameter in insert_rows (OracleHook) (#39408)
add 80943bd177 Add missing usePgbouncer from values.yaml for triggerer (#40614)
add 77a9850556 Updating airflow download links to CDN based links (#40618)
add 1dc582dba3 fix openlineage parsing dag tree with MappedOperator (#40621)
add 3f0979c9f1 FIX: DbApiHook.insert_rows unnecessarily restarting connections (#40615)
add ec5da92440 Skip checking subdagslist if taskinstance state is skipped (#40578)
add 2eda7376f4 Resolve deprecations in `serialization` tests (#40625)
add 9255167bb3 Resolve google vertex ai deprecations in tests (#40628)
add c72920af3b Do not dynamically determine op links for emr serverless (#40627)
add e827bfbd2c Fix classifying of changes for multiple providers (#40634)
add e7dedbe95a Fix class instance vs. class type in validate_database_executor_compatibility() call (#40626)
add 7360c8bbb8 Fix case when release manager does not accept version change (#40636)
add be271827e0 Resolve google deprecations in tests (#40629)
add 0f30bfd27e Add HookLineageCollector that during task execution should register and hold lineage sent from hooks.
add 75ec80130c Remove default `create_dataset` method.
new af947f3a7b AIP-62 support for S3Hook and S3 usage of ObjectStore
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (af23b6c0c4)
\
N -- N -- N refs/heads/aip-62/s3 (af947f3a7b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 1 -
.../airflow_providers_bug_report.yml | 1 +
.pre-commit-config.yaml | 12 +-
Dockerfile | 4 +-
Dockerfile.ci | 18 +-
INSTALL | 20 +-
INTHEWILD.md | 1 +
ISSUE_TRIAGE_PROCESS.rst | 17 +
airflow/api/client/api_client.py | 15 +-
airflow/api/client/json_client.py | 27 +-
airflow/api/common/experimental/get_code.py | 3 +-
.../api/common/experimental/get_dag_run_state.py | 3 +-
airflow/api/common/mark_tasks.py | 17 +-
airflow/api/common/trigger_dag.py | 6 +-
.../endpoints/dag_warning_endpoint.py | 3 +-
.../endpoints/task_instance_endpoint.py | 2 +-
airflow/api_connexion/schemas/dag_run_schema.py | 3 +-
airflow/auth/managers/base_auth_manager.py | 3 +-
airflow/cli/cli_parser.py | 3 +-
airflow/cli/commands/config_command.py | 11 +-
airflow/cli/commands/connection_command.py | 3 +-
airflow/cli/commands/daemon_utils.py | 3 +-
airflow/cli/commands/dag_command.py | 32 +-
airflow/cli/commands/provider_command.py | 3 +-
airflow/cli/commands/scheduler_command.py | 2 +-
airflow/cli/commands/task_command.py | 6 +-
airflow/cli/utils.py | 3 +-
airflow/config_templates/config.yml | 28 -
airflow/configuration.py | 9 +-
airflow/dag_processing/manager.py | 66 +-
airflow/dag_processing/processor.py | 23 +-
airflow/datasets/__init__.py | 40 +-
airflow/decorators/base.py | 9 +-
airflow/decorators/bash.py | 6 +-
airflow/decorators/task_group.py | 9 +-
airflow/example_dags/example_params_ui_tutorial.py | 4 +-
airflow/executors/base_executor.py | 9 +-
airflow/executors/executor_loader.py | 12 +-
airflow/io/path.py | 12 +-
airflow/jobs/backfill_job_runner.py | 7 +-
airflow/jobs/local_task_job_runner.py | 5 +-
airflow/jobs/scheduler_job_runner.py | 13 +-
.../pre_7_4_0_compatibility/pod_generator.py | 6 +-
airflow/lineage/hook.py | 9 +-
airflow/metrics/statsd_logger.py | 2 +-
airflow/migrations/env.py | 6 +-
.../versions/0001_1_5_0_current_schema.py | 3 +-
.../versions/0002_1_5_0_create_is_encrypted.py | 3 +-
.../versions/0003_1_5_0_for_compatibility.py | 3 +-
.../0004_1_5_0_more_logging_into_task_isntance.py | 3 +-
.../versions/0005_1_5_2_job_id_indices.py | 3 +-
.../versions/0006_1_6_0_adding_extra_to_log.py | 3 +-
.../migrations/versions/0007_1_6_0_add_dagrun.py | 3 +-
.../versions/0008_1_6_0_task_duration.py | 3 +-
.../versions/0009_1_6_0_dagrun_config.py | 3 +-
.../0010_1_6_2_add_password_column_to_user.py | 3 +-
.../versions/0011_1_6_2_dagrun_start_end.py | 3 +-
...7_0_add_notification_sent_column_to_sla_miss.py | 3 +-
..._1_7_0_add_a_column_to_track_the_encryption_.py | 3 +-
...4_1_7_0_add_is_encrypted_column_to_variable_.py | 3 +-
.../versions/0015_1_7_1_rename_user_table.py | 3 +-
.../versions/0016_1_7_1_add_ti_state_index.py | 3 +-
.../0017_1_7_1_add_task_fails_journal_table.py | 3 +-
.../versions/0018_1_7_1_add_dag_stats_table.py | 3 +-
...1_7_1_add_fractional_seconds_to_mysql_tables.py | 3 +-
.../versions/0020_1_7_1_xcom_dag_task_indices.py | 3 +-
.../0021_1_7_1_add_pid_field_to_taskinstance.py | 3 +-
..._7_1_add_dag_id_state_index_on_dag_run_table.py | 3 +-
..._1_8_2_add_max_tries_column_to_task_instance.py | 3 +-
..._1_8_2_make_xcom_value_column_a_large_binary.py | 3 +-
.../versions/0025_1_8_2_add_ti_job_id_index.py | 3 +-
.../0026_1_8_2_increase_text_size_for_mysql.py | 3 +-
.../0027_1_10_0_add_time_zone_awareness.py | 3 +-
...1_10_0_add_kubernetes_resource_checkpointing.py | 3 +-
..._1_10_0_add_executor_config_to_task_instance.py | 3 +-
...0_1_10_0_add_kubernetes_scheduler_uniqueness.py | 3 +-
.../migrations/versions/0031_1_10_0_merge_heads.py | 3 +-
.../0032_1_10_0_fix_mysql_not_null_constraint.py | 3 +-
.../versions/0033_1_10_0_fix_sqlite_foreign_key.py | 3 +-
.../versions/0034_1_10_0_index_taskfail.py | 3 +-
.../versions/0035_1_10_2_add_idx_log_dag.py | 3 +-
.../0036_1_10_2_add_index_to_taskinstance.py | 3 +-
.../0037_1_10_2_add_task_reschedule_table.py | 3 +-
.../versions/0038_1_10_2_add_sm_dag_index.py | 3 +-
.../versions/0039_1_10_2_add_superuser_field.py | 3 +-
.../versions/0040_1_10_3_add_fields_to_dag.py | 3 +-
.../0041_1_10_3_add_schedule_interval_to_dag.py | 3 +-
..._1_10_3_task_reschedule_fk_on_cascade_delete.py | 3 +-
...3_1_10_4_make_taskinstance_pool_not_nullable.py | 3 +-
.../0044_1_10_7_add_serialized_dag_table.py | 3 +-
.../versions/0045_1_10_7_add_root_dag_id_to_dag.py | 3 +-
...0_5_change_datetime_to_datetime2_6_on_mssql_.py | 18 +-
.../0047_1_10_4_increase_queue_name_size_limit.py | 3 +-
.../versions/0048_1_10_3_remove_dag_stat_table.py | 3 +-
.../migrations/versions/0049_1_10_7_merge_heads.py | 3 +-
...10_7_increase_length_for_connection_password.py | 3 +-
.../versions/0051_1_10_8_add_dagtags_table.py | 3 +-
..._10_10_add_pool_slots_field_to_task_instance.py | 3 +-
...0_10_add_rendered_task_instance_fields_table.py | 3 +-
.../versions/0054_1_10_10_add_dag_code_table.py | 3 +-
..._11_add_precision_to_execution_date_in_mysql.py | 3 +-
...10_12_add_dag_hash_column_to_serialized_dag_.py | 3 +-
.../versions/0057_1_10_13_add_fab_tables.py | 3 +-
...1_10_13_increase_length_of_fab_ab_view_menu_.py | 3 +-
.../versions/0059_2_0_0_drop_user_and_chart.py | 3 +-
.../0060_2_0_0_remove_id_column_from_xcom.py | 3 +-
.../0061_2_0_0_increase_length_of_pool_name.py | 3 +-
.../0063_2_0_0_set_conn_type_as_non_nullable.py | 3 +-
.../0064_2_0_0_add_unique_constraint_to_conn_id.py | 3 +-
.../0065_2_0_0_update_schema_for_smart_sensor.py | 3 +-
.../0066_2_0_0_add_queued_by_job_id_to_ti.py | 3 +-
.../0067_2_0_0_add_external_executor_id_to_ti.py | 3 +-
.../0068_2_0_0_drop_kuberesourceversion_and_.py | 3 +-
...2_0_0_add_scheduling_decision_to_dagrun_and_.py | 3 +-
...0_fix_mssql_exec_date_rendered_task_instance.py | 3 +-
.../0071_2_0_0_add_job_id_to_dagrun_table.py | 3 +-
...072_2_0_0_add_k8s_yaml_to_rendered_templates.py | 3 +-
.../versions/0073_2_0_0_prefix_dag_permissions.py | 3 +-
.../0074_2_0_0_resource_based_permissions.py | 3 +-
...75_2_0_0_add_description_field_to_connection.py | 3 +-
..._0_0_fix_description_field_in_connection_to_.py | 3 +-
...2_0_0_change_field_in_dagcode_to_mediumtext_.py | 3 +-
..._2_0_1_remove_can_read_permission_on_config_.py | 3 +-
...0_2_increase_size_of_connection_extra_field_.py | 3 +-
.../0080_2_0_2_change_default_pool_slots_to_1.py | 3 +-
.../0081_2_0_2_rename_last_scheduler_run_column.py | 3 +-
..._1_0_increase_pool_name_size_in_taskinstance.py | 3 +-
...0083_2_1_0_add_description_field_to_variable.py | 3 +-
..._1_0_resource_based_permissions_for_default_.py | 3 +-
...5_2_1_3_add_queued_at_column_to_dagrun_table.py | 3 +-
..._1_4_add_max_active_runs_column_to_dagmodel_.py | 3 +-
..._2_1_4_add_index_on_state_dag_id_for_queued_.py | 3 +-
.../0088_2_2_0_improve_mssql_compatibility.py | 3 +-
...89_2_2_0_make_xcom_pkey_columns_non_nullable.py | 3 +-
..._2_0_rename_concurrency_column_in_dag_table_.py | 3 +-
.../0091_2_2_0_add_trigger_table_and_task_info.py | 3 +-
...ta_interval_start_end_to_dagmodel_and_dagrun.py | 3 +-
.../0093_2_2_0_taskinstance_keyed_to_dagrun.py | 3 +-
...2_3_add_has_import_errors_column_to_dagmodel.py | 3 +-
.../versions/0095_2_2_4_add_session_table_to_db.py | 3 +-
.../0096_2_2_4_adding_index_for_dag_id_in_job.py | 3 +-
..._2_3_0_increase_length_of_email_and_username.py | 3 +-
...098_2_3_0_added_timetable_description_column.py | 3 +-
...9_2_3_0_add_task_log_filename_template_model.py | 3 +-
...2_3_0_add_taskmap_and_map_id_on_taskinstance.py | 3 +-
..._2_3_0_add_data_compressed_to_serialized_dag.py | 3 +-
.../0102_2_3_0_switch_xcom_table_to_use_run_id.py | 9 +-
.../0103_2_3_0_add_callback_request_table.py | 3 +-
...3_0_migrate_rtif_to_use_run_id_and_map_index.py | 3 +-
.../0105_2_3_0_add_map_index_to_taskfail.py | 3 +-
...on_for_fab_tables_to_add_missing_constraints.py | 3 +-
.../versions/0107_2_3_0_add_map_index_to_log.py | 3 +-
.../versions/0108_2_3_0_default_dag_view_grid.py | 3 +-
.../0109_2_3_1_add_index_for_event_in_log.py | 3 +-
...0110_2_3_2_add_cascade_to_dag_tag_foreignkey.py | 3 +-
.../0111_2_3_3_add_indexes_for_cascade_deletes.py | 3 +-
.../versions/0112_2_4_0_add_dagwarning_model.py | 3 +-
.../0113_2_4_0_compare_types_between_orm_and_db.py | 3 +-
.../versions/0114_2_4_0_add_dataset_model.py | 3 +-
.../versions/0115_2_4_0_remove_smart_sensors.py | 3 +-
.../0116_2_4_0_add_dag_owner_attributes_table.py | 3 +-
..._2_4_0_add_processor_subdir_to_dagmodel_and_.py | 3 +-
.../versions/0118_2_4_2_add_missing_autoinc_fab.py | 6 +-
...e_insensitive_unique_constraint_for_username.py | 3 +-
.../0120_2_5_0_add_updated_at_to_dagrun_and_ti.py | 3 +-
...21_2_5_0_add_dagrunnote_and_taskinstancenote.py | 3 +-
.../0122_2_5_0_add_is_orphaned_to_datasetmodel.py | 3 +-
.../0123_2_6_0_add_dttm_index_on_log_table.py | 3 +-
...0_increase_length_of_user_identifier_columns.py | 3 +-
.../0125_2_6_2_add_onupdate_cascade_to_taskmap.py | 3 +-
.../0126_2_7_0_add_index_to_task_instance_table.py | 3 +-
.../0127_2_7_0_add_custom_operator_name_column.py | 3 +-
...28_2_7_0_add_include_deferred_column_to_pool.py | 3 +-
.../0129_2_8_0_add_clear_number_to_dag_run.py | 3 +-
..._0_add_owner_display_name_to_audit_log_table.py | 3 +-
...31_2_8_0_make_connection_login_password_text.py | 3 +-
...0132_2_8_0_add_processor_subdir_import_error.py | 3 +-
.../0133_2_8_1_refactor_dag_run_indexes.py | 3 +-
...2_9_0_add_rendered_map_index_to_taskinstance.py | 3 +-
...audit_log_table_and_change_event_name_length.py | 3 +-
...136_2_9_0_add_dataset_expression_in_dagmodel.py | 3 +-
...2_9_0_adding_adding_max_failure_runs_column_.py | 3 +-
..._2_9_0_make_xcom_value_to_longblob_for_mysql.py | 3 +-
...139_2_9_0_add_display_name_for_dag_and_task_.py | 3 +-
.../0140_2_9_0_update_trigger_kwargs_type.py | 3 +-
...1_2_9_2_remove_idx_last_scheduling_decision_.py | 3 +-
...nconsistency_between_ORM_and_migration_files.py | 3 +-
...dexes_on_dag_id_column_in_referencing_tables.py | 3 +-
.../0144_2_10_0_add_new_executor_field_to_db.py | 3 +-
...2_10_0_added_dagpriorityparsingrequest_table.py | 3 +-
.../0146_2_10_0_add_task_instance_history.py | 3 +-
airflow/models/abstractoperator.py | 33 +-
airflow/models/baseoperator.py | 66 +-
airflow/models/baseoperatorlink.py | 3 +-
airflow/models/connection.py | 3 +-
airflow/models/dag.py | 143 ++--
airflow/models/dagbag.py | 3 +-
airflow/models/dagcode.py | 24 +-
airflow/models/dagrun.py | 16 +-
airflow/models/expandinput.py | 15 +-
airflow/models/mappedoperator.py | 15 +-
airflow/models/param.py | 3 +-
airflow/models/renderedtifields.py | 3 +-
airflow/models/serialized_dag.py | 12 +-
airflow/models/taskinstance.py | 30 +-
airflow/models/taskinstancekey.py | 3 +-
airflow/models/tasklog.py | 3 +-
airflow/models/taskmap.py | 6 +-
airflow/models/taskmixin.py | 3 +-
airflow/models/trigger.py | 6 +-
airflow/models/variable.py | 15 +-
airflow/models/xcom.py | 48 +-
airflow/models/xcom_arg.py | 30 +-
airflow/notifications/basenotifier.py | 3 +-
airflow/operators/bash.py | 3 +-
airflow/operators/datetime.py | 3 +-
airflow/operators/python.py | 9 +-
airflow/operators/subdag.py | 3 +-
airflow/operators/trigger_dagrun.py | 12 +-
airflow/operators/weekday.py | 3 +-
airflow/policies.py | 3 +-
airflow/providers/amazon/CHANGELOG.rst | 40 +
airflow/providers/amazon/__init__.py | 2 +-
airflow/providers/amazon/aws/hooks/athena.py | 27 +-
airflow/providers/amazon/aws/hooks/athena_sql.py | 3 +-
airflow/providers/amazon/aws/hooks/base_aws.py | 27 +-
airflow/providers/amazon/aws/hooks/chime.py | 3 +-
airflow/providers/amazon/aws/hooks/datasync.py | 9 +-
airflow/providers/amazon/aws/hooks/ecr.py | 3 +-
airflow/providers/amazon/aws/hooks/ecs.py | 18 +-
airflow/providers/amazon/aws/hooks/glacier.py | 12 +-
airflow/providers/amazon/aws/hooks/kinesis.py | 3 +-
airflow/providers/amazon/aws/hooks/logs.py | 6 +-
.../providers/amazon/aws/hooks/redshift_cluster.py | 36 +-
.../providers/amazon/aws/hooks/redshift_data.py | 6 +-
airflow/providers/amazon/aws/hooks/redshift_sql.py | 9 +-
airflow/providers/amazon/aws/hooks/s3.py | 123 +--
airflow/providers/amazon/aws/hooks/sagemaker.py | 123 ++-
.../providers/amazon/aws/hooks/secrets_manager.py | 9 +-
airflow/providers/amazon/aws/hooks/sts.py | 3 +-
airflow/providers/amazon/aws/operators/athena.py | 29 +-
airflow/providers/amazon/aws/operators/batch.py | 18 +-
airflow/providers/amazon/aws/operators/datasync.py | 3 +-
airflow/providers/amazon/aws/operators/ecs.py | 1 +
airflow/providers/amazon/aws/operators/emr.py | 92 +--
airflow/providers/amazon/aws/operators/glue.py | 6 +-
.../providers/amazon/aws/operators/glue_crawler.py | 41 +-
airflow/providers/amazon/aws/operators/neptune.py | 3 +-
.../amazon/aws/operators/redshift_cluster.py | 3 +-
.../providers/amazon/aws/operators/sagemaker.py | 3 +-
airflow/providers/amazon/aws/sensors/base_aws.py | 3 +-
.../amazon/aws/sensors/glue_catalog_partition.py | 42 +-
.../providers/amazon/aws/sensors/glue_crawler.py | 28 +-
.../providers/amazon/aws/transfers/mongo_to_s3.py | 9 +-
.../amazon/aws/transfers/s3_to_dynamodb.py | 3 +-
.../providers/amazon/aws/transfers/s3_to_sql.py | 3 +-
airflow/providers/amazon/aws/triggers/ecs.py | 4 +-
airflow/providers/amazon/aws/triggers/glue.py | 18 +-
.../providers/amazon/aws/triggers/glue_crawler.py | 9 +-
.../amazon/aws/utils/connection_wrapper.py | 15 +-
airflow/providers/amazon/aws/utils/mixins.py | 3 +-
airflow/providers/amazon/aws/utils/redshift.py | 3 +-
airflow/providers/amazon/provider.yaml | 3 +-
airflow/providers/apache/drill/CHANGELOG.rst | 8 +
airflow/providers/apache/drill/__init__.py | 2 +-
airflow/providers/apache/drill/provider.yaml | 3 +-
.../apache/druid/transfers/hive_to_druid.py | 3 +-
airflow/providers/apache/flink/CHANGELOG.rst | 8 +
airflow/providers/apache/flink/__init__.py | 2 +-
airflow/providers/apache/flink/provider.yaml | 3 +-
airflow/providers/apache/hdfs/CHANGELOG.rst | 9 +
airflow/providers/apache/hdfs/__init__.py | 2 +-
airflow/providers/apache/hdfs/hooks/webhdfs.py | 3 +-
airflow/providers/apache/hdfs/provider.yaml | 3 +-
airflow/providers/apache/hive/CHANGELOG.rst | 9 +
airflow/providers/apache/hive/__init__.py | 2 +-
airflow/providers/apache/hive/hooks/hive.py | 6 +-
.../providers/apache/hive/operators/hive_stats.py | 3 +-
airflow/providers/apache/hive/provider.yaml | 3 +-
.../apache/hive/transfers/vertica_to_hive.py | 3 +-
.../apache/iceberg/.latest-doc-only-change.txt | 1 +
airflow/providers/apache/kafka/CHANGELOG.rst | 18 +
airflow/providers/apache/kafka/__init__.py | 2 +-
airflow/providers/apache/kafka/hooks/client.py | 3 +-
.../providers/apache/kafka/operators/consume.py | 3 +-
.../providers/apache/kafka/operators/produce.py | 3 +-
airflow/providers/apache/kafka/provider.yaml | 3 +-
airflow/providers/apache/kafka/sensors/kafka.py | 3 +-
.../apache/kafka/triggers/await_message.py | 3 +-
airflow/providers/apache/kylin/CHANGELOG.rst | 8 +
airflow/providers/apache/kylin/__init__.py | 2 +-
.../providers/apache/kylin/operators/kylin_cube.py | 3 +-
airflow/providers/apache/kylin/provider.yaml | 3 +-
airflow/providers/apache/livy/triggers/livy.py | 3 +-
airflow/providers/apache/pig/hooks/pig.py | 3 +-
airflow/providers/apache/spark/CHANGELOG.rst | 8 +
airflow/providers/apache/spark/__init__.py | 2 +-
.../providers/apache/spark/hooks/spark_submit.py | 6 +-
airflow/providers/apache/spark/provider.yaml | 3 +-
airflow/providers/atlassian/jira/operators/jira.py | 3 +-
.../providers/celery/executors/celery_executor.py | 3 +-
airflow/providers/cloudant/CHANGELOG.rst | 8 +
airflow/providers/cloudant/__init__.py | 2 +-
airflow/providers/cloudant/provider.yaml | 3 +-
airflow/providers/cncf/kubernetes/CHANGELOG.rst | 13 +
airflow/providers/cncf/kubernetes/__init__.py | 2 +-
airflow/providers/cncf/kubernetes/callbacks.py | 24 +-
.../cncf/kubernetes/decorators/kubernetes.py | 3 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 30 +-
.../cncf/kubernetes/kubernetes_helper_functions.py | 9 +-
airflow/providers/cncf/kubernetes/provider.yaml | 3 +-
airflow/providers/cohere/operators/embedding.py | 3 +-
.../{io => compat}/.latest-doc-only-change.txt | 0
.../providers/{ydb => common/compat}/CHANGELOG.rst | 12 +-
.../{apache/flink => common/compat}/__init__.py | 4 +-
.../common/compat/provider.yaml} | 46 +-
airflow/providers/common/io/xcom/backend.py | 15 +-
airflow/providers/common/sql/CHANGELOG.rst | 10 +
airflow/providers/common/sql/__init__.py | 2 +-
airflow/providers/common/sql/hooks/sql.py | 32 +-
airflow/providers/common/sql/hooks/sql.pyi | 5 +-
airflow/providers/common/sql/operators/sql.py | 8 +-
airflow/providers/common/sql/provider.yaml | 3 +-
airflow/providers/common/sql/sensors/sql.py | 3 +-
airflow/providers/databricks/CHANGELOG.rst | 21 +
airflow/providers/databricks/__init__.py | 2 +-
airflow/providers/databricks/hooks/databricks.py | 9 +-
.../providers/databricks/hooks/databricks_base.py | 104 +--
.../providers/databricks/hooks/databricks_sql.py | 3 +-
.../providers/databricks/operators/databricks.py | 187 ++---
airflow/providers/databricks/provider.yaml | 8 +-
airflow/providers/databricks/utils/databricks.py | 4 +-
airflow/providers/dbt/cloud/CHANGELOG.rst | 13 +
airflow/providers/dbt/cloud/__init__.py | 2 +-
airflow/providers/dbt/cloud/provider.yaml | 3 +-
airflow/providers/dbt/cloud/sensors/dbt.py | 6 +-
airflow/providers/dbt/cloud/triggers/dbt.py | 3 +-
airflow/providers/dingding/hooks/dingding.py | 6 +-
airflow/providers/docker/CHANGELOG.rst | 8 +
airflow/providers/docker/__init__.py | 2 +-
airflow/providers/docker/operators/docker.py | 18 +-
airflow/providers/docker/operators/docker_swarm.py | 3 +-
airflow/providers/docker/provider.yaml | 3 +-
.../providers/elasticsearch/log/es_task_handler.py | 3 +-
airflow/providers/exasol/CHANGELOG.rst | 13 +
airflow/providers/exasol/__init__.py | 2 +-
airflow/providers/exasol/hooks/exasol.py | 27 +-
airflow/providers/exasol/provider.yaml | 3 +-
airflow/providers/fab/CHANGELOG.rst | 16 +
airflow/providers/fab/__init__.py | 2 +-
.../fab/auth_manager/security_manager/override.py | 34 +-
airflow/providers/fab/provider.yaml | 9 +-
airflow/providers/facebook/CHANGELOG.rst | 8 +
airflow/providers/facebook/__init__.py | 2 +-
airflow/providers/facebook/ads/hooks/ads.py | 12 +-
airflow/providers/facebook/provider.yaml | 3 +-
airflow/providers/ftp/CHANGELOG.rst | 8 +
airflow/providers/ftp/__init__.py | 2 +-
airflow/providers/ftp/provider.yaml | 3 +-
airflow/providers/github/CHANGELOG.rst | 8 +
airflow/providers/github/__init__.py | 2 +-
airflow/providers/github/operators/github.py | 3 +-
airflow/providers/github/provider.yaml | 3 +-
airflow/providers/google/CHANGELOG.rst | 68 ++
airflow/providers/google/__init__.py | 2 +-
airflow/providers/google/ads/hooks/ads.py | 24 +-
.../providers/google/ads/transfers/ads_to_gcs.py | 3 +-
.../_internal_client/secret_manager_client.py | 9 +-
airflow/providers/google/cloud/hooks/bigquery.py | 237 ++++--
airflow/providers/google/cloud/hooks/cloud_sql.py | 18 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 51 +-
airflow/providers/google/cloud/hooks/dataflow.py | 56 +-
airflow/providers/google/cloud/hooks/dataform.py | 3 +-
airflow/providers/google/cloud/hooks/datafusion.py | 6 +-
airflow/providers/google/cloud/hooks/dataproc.py | 153 ++--
airflow/providers/google/cloud/hooks/functions.py | 30 +-
.../google/cloud/hooks/kubernetes_engine.py | 33 +-
airflow/providers/google/cloud/hooks/os_login.py | 3 +-
.../providers/google/cloud/hooks/secret_manager.py | 27 +-
airflow/providers/google/cloud/hooks/translate.py | 3 +-
.../cloud/hooks/vertex_ai/batch_prediction_job.py | 3 +-
.../cloud/hooks/vertex_ai/generative_model.py | 141 ++++
.../google/cloud/hooks/vertex_ai/pipeline_job.py | 3 +-
airflow/providers/google/cloud/links/base.py | 3 +-
airflow/providers/google/cloud/links/datafusion.py | 3 +-
.../google/cloud/log/stackdriver_task_handler.py | 6 +-
.../providers/google/cloud/openlineage/mixins.py | 10 +
.../providers/google/cloud/openlineage/utils.py | 6 +-
.../providers/google/cloud/operators/bigquery.py | 76 +-
.../google/cloud/operators/cloud_batch.py | 4 +-
.../providers/google/cloud/operators/cloud_sql.py | 33 +-
.../providers/google/cloud/operators/dataform.py | 3 +-
.../providers/google/cloud/operators/dataproc.py | 102 ++-
.../google/cloud/operators/dataproc_metastore.py | 36 +-
airflow/providers/google/cloud/operators/gcs.py | 3 +-
airflow/providers/google/cloud/operators/pubsub.py | 15 +-
.../google/cloud/operators/vertex_ai/auto_ml.py | 6 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 21 +-
.../cloud/operators/vertex_ai/generative_model.py | 243 ++++++
.../cloud/operators/vertex_ai/pipeline_job.py | 3 +-
airflow/providers/google/cloud/operators/vision.py | 54 +-
airflow/providers/google/cloud/sensors/gcs.py | 13 +-
airflow/providers/google/cloud/sensors/pubsub.py | 3 +-
.../google/cloud/transfers/azure_blob_to_gcs.py | 13 +
.../google/cloud/transfers/bigquery_to_gcs.py | 33 +-
.../google/cloud/transfers/bigquery_to_postgres.py | 59 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 3 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 22 +-
.../google/cloud/transfers/local_to_gcs.py | 6 +-
.../google/cloud/transfers/mysql_to_gcs.py | 3 +-
.../google/cloud/transfers/oracle_to_gcs.py | 3 +-
.../google/cloud/transfers/presto_to_gcs.py | 3 +-
.../providers/google/cloud/transfers/s3_to_gcs.py | 3 +-
.../google/cloud/transfers/trino_to_gcs.py | 3 +-
.../providers/google/cloud/triggers/cloud_batch.py | 3 +-
.../providers/google/cloud/triggers/cloud_run.py | 3 +-
.../providers/google/cloud/triggers/dataflow.py | 3 +-
.../providers/google/cloud/triggers/vertex_ai.py | 3 +-
.../google/cloud/utils/external_token_supplier.py | 6 +-
.../google/cloud/utils/field_sanitizer.py | 6 +-
.../google/cloud/utils/field_validator.py | 9 +-
airflow/providers/google/cloud/utils/helpers.py | 3 +-
.../providers/google/common/hooks/base_google.py | 3 +-
.../google/common/utils/id_token_credentials.py | 3 +-
airflow/providers/google/go_module_utils.py | 6 +-
.../marketing_platform/hooks/analytics_admin.py | 18 +-
.../marketing_platform/links/analytics_admin.py | 3 +-
airflow/providers/google/provider.yaml | 5 +-
.../google/suite/transfers/local_to_drive.py | 3 +-
airflow/providers/grpc/CHANGELOG.rst | 8 +
airflow/providers/grpc/__init__.py | 2 +-
airflow/providers/grpc/provider.yaml | 3 +-
airflow/providers/http/CHANGELOG.rst | 8 +
airflow/providers/http/__init__.py | 2 +-
airflow/providers/http/hooks/http.py | 27 +-
airflow/providers/http/operators/http.py | 6 +-
airflow/providers/http/provider.yaml | 3 +-
airflow/providers/influxdb/hooks/influxdb.py | 43 +-
airflow/providers/jdbc/hooks/jdbc.py | 9 +-
.../jenkins/operators/jenkins_job_trigger.py | 12 +-
airflow/providers/jenkins/sensors/jenkins.py | 3 +-
airflow/providers/microsoft/azure/CHANGELOG.rst | 18 +
airflow/providers/microsoft/azure/__init__.py | 2 +-
airflow/providers/microsoft/azure/hooks/asb.py | 15 +-
.../microsoft/azure/hooks/data_factory.py | 3 +-
.../providers/microsoft/azure/hooks/data_lake.py | 54 +-
airflow/providers/microsoft/azure/operators/asb.py | 30 +-
.../providers/microsoft/azure/operators/cosmos.py | 3 +-
.../providers/microsoft/azure/operators/msgraph.py | 57 +-
airflow/providers/microsoft/azure/provider.yaml | 3 +-
.../microsoft/azure/sensors/data_factory.py | 3 +-
airflow/providers/microsoft/azure/sensors/wasb.py | 6 +-
.../microsoft/azure/triggers/data_factory.py | 6 +-
airflow/providers/microsoft/azure/utils.py | 12 +-
airflow/providers/microsoft/mssql/CHANGELOG.rst | 9 +
airflow/providers/microsoft/mssql/__init__.py | 2 +-
airflow/providers/microsoft/mssql/provider.yaml | 3 +-
airflow/providers/microsoft/psrp/operators/psrp.py | 3 +-
airflow/providers/mongo/CHANGELOG.rst | 8 +
airflow/providers/mongo/__init__.py | 2 +-
airflow/providers/mongo/provider.yaml | 3 +-
airflow/providers/mysql/CHANGELOG.rst | 9 +
airflow/providers/mysql/__init__.py | 2 +-
airflow/providers/mysql/provider.yaml | 3 +-
airflow/providers/odbc/CHANGELOG.rst | 8 +
airflow/providers/odbc/__init__.py | 2 +-
airflow/providers/odbc/hooks/odbc.py | 6 +-
airflow/providers/odbc/provider.yaml | 3 +-
airflow/providers/openai/CHANGELOG.rst | 8 +
airflow/providers/openai/__init__.py | 2 +-
airflow/providers/openai/provider.yaml | 3 +-
airflow/providers/openlineage/CHANGELOG.rst | 38 +-
airflow/providers/openlineage/__init__.py | 2 +-
airflow/providers/openlineage/extractors/base.py | 9 +-
airflow/providers/openlineage/plugins/adapter.py | 15 +-
airflow/providers/openlineage/plugins/facets.py | 3 +-
airflow/providers/openlineage/plugins/listener.py | 22 +-
airflow/providers/openlineage/provider.yaml | 3 +-
airflow/providers/openlineage/sqlparser.py | 6 +-
.../openlineage/utils/selective_enable.py | 9 +-
airflow/providers/openlineage/utils/sql.py | 8 +-
airflow/providers/openlineage/utils/utils.py | 22 +-
airflow/providers/opensearch/CHANGELOG.rst | 8 +
airflow/providers/opensearch/__init__.py | 2 +-
airflow/providers/opensearch/provider.yaml | 3 +-
airflow/providers/oracle/CHANGELOG.rst | 8 +
airflow/providers/oracle/__init__.py | 2 +-
airflow/providers/oracle/hooks/oracle.py | 17 +-
airflow/providers/oracle/provider.yaml | 3 +-
airflow/providers/pagerduty/CHANGELOG.rst | 8 +
airflow/providers/pagerduty/__init__.py | 2 +-
airflow/providers/pagerduty/provider.yaml | 3 +-
airflow/providers/papermill/CHANGELOG.rst | 9 +
airflow/providers/papermill/__init__.py | 2 +-
airflow/providers/papermill/provider.yaml | 3 +-
airflow/providers/pinecone/CHANGELOG.rst | 8 +
airflow/providers/pinecone/__init__.py | 2 +-
airflow/providers/pinecone/provider.yaml | 3 +-
airflow/providers/postgres/CHANGELOG.rst | 8 +
airflow/providers/postgres/__init__.py | 2 +-
airflow/providers/postgres/hooks/postgres.py | 21 +-
airflow/providers/postgres/provider.yaml | 3 +-
airflow/providers/presto/CHANGELOG.rst | 9 +
airflow/providers/presto/__init__.py | 2 +-
airflow/providers/presto/provider.yaml | 3 +-
airflow/providers/salesforce/CHANGELOG.rst | 9 +
airflow/providers/salesforce/__init__.py | 2 +-
airflow/providers/salesforce/provider.yaml | 3 +-
airflow/providers/samba/hooks/samba.py | 3 +-
airflow/providers/sftp/CHANGELOG.rst | 14 +
airflow/providers/sftp/__init__.py | 2 +-
airflow/providers/sftp/hooks/sftp.py | 54 +-
airflow/providers/sftp/provider.yaml | 3 +-
airflow/providers/slack/hooks/slack.py | 3 +-
airflow/providers/slack/operators/slack.py | 12 +-
airflow/providers/slack/utils/__init__.py | 9 +-
airflow/providers/smtp/hooks/smtp.py | 3 +-
airflow/providers/snowflake/CHANGELOG.rst | 11 +
airflow/providers/snowflake/__init__.py | 2 +-
airflow/providers/snowflake/hooks/snowflake.py | 22 +-
airflow/providers/snowflake/provider.yaml | 3 +-
.../snowflake/transfers/copy_into_snowflake.py | 3 +-
airflow/providers/snowflake/utils/openlineage.py | 3 +-
airflow/providers/ssh/CHANGELOG.rst | 8 +
airflow/providers/ssh/__init__.py | 2 +-
airflow/providers/ssh/hooks/ssh.py | 12 +-
airflow/providers/ssh/provider.yaml | 3 +-
airflow/providers/tableau/CHANGELOG.rst | 8 +
airflow/providers/tableau/__init__.py | 2 +-
airflow/providers/tableau/hooks/tableau.py | 6 +-
airflow/providers/tableau/provider.yaml | 3 +-
airflow/providers/telegram/CHANGELOG.rst | 9 +
airflow/providers/telegram/__init__.py | 2 +-
airflow/providers/telegram/provider.yaml | 3 +-
airflow/providers/teradata/CHANGELOG.rst | 8 +
airflow/providers/teradata/__init__.py | 2 +-
airflow/providers/teradata/hooks/teradata.py | 9 +-
airflow/providers/teradata/operators/teradata.py | 2 +-
.../teradata/operators/teradata_compute_cluster.py | 513 +++++++++++++
airflow/providers/teradata/provider.yaml | 10 +-
.../teradata/transfers/azure_blob_to_teradata.py | 46 +-
.../providers/teradata/transfers/s3_to_teradata.py | 28 +-
.../teradata/triggers}/__init__.py | 0
.../teradata/triggers/teradata_compute_cluster.py | 155 ++++
.../teradata/utils}/__init__.py | 0
airflow/providers/teradata/utils/constants.py | 46 ++
airflow/providers/trino/CHANGELOG.rst | 9 +
airflow/providers/trino/__init__.py | 2 +-
airflow/providers/trino/provider.yaml | 3 +-
airflow/providers/vertica/CHANGELOG.rst | 8 +
airflow/providers/vertica/__init__.py | 2 +-
airflow/providers/vertica/provider.yaml | 3 +-
airflow/providers/weaviate/CHANGELOG.rst | 27 +
airflow/providers/weaviate/__init__.py | 2 +-
airflow/providers/weaviate/hooks/weaviate.py | 831 ++++++++-------------
airflow/providers/weaviate/operators/weaviate.py | 37 +-
airflow/providers/weaviate/provider.yaml | 5 +-
airflow/providers/yandex/CHANGELOG.rst | 18 +
airflow/providers/yandex/__init__.py | 2 +-
airflow/providers/yandex/hooks/yq.py | 18 +-
airflow/providers/yandex/operators/dataproc.py | 21 +-
airflow/providers/yandex/provider.yaml | 3 +-
airflow/providers/ydb/CHANGELOG.rst | 2 +-
airflow/providers/ydb/provider.yaml | 2 +-
airflow/providers/ydb/utils/credentials.py | 15 +-
airflow/providers_manager.py | 16 +-
airflow/security/permissions.py | 3 +-
airflow/security/utils.py | 3 +-
airflow/serialization/helpers.py | 3 +-
airflow/serialization/schema.json | 4 -
airflow/serialization/serde.py | 12 +-
airflow/serialization/serialized_objects.py | 75 +-
airflow/serialization/serializers/timezone.py | 3 +-
airflow/settings.py | 7 +-
airflow/task/priority_strategy.py | 12 +-
airflow/task/task_runner/standard_task_runner.py | 13 +-
airflow/template/templater.py | 3 +-
airflow/ti_deps/deps/base_ti_dep.py | 3 +-
airflow/ti_deps/deps/prev_dagrun_dep.py | 12 +-
airflow/ti_deps/deps/trigger_rule_dep.py | 21 +-
airflow/timetables/_cron.py | 12 +-
airflow/timetables/base.py | 98 ++-
airflow/timetables/datasets.py | 20 +-
airflow/timetables/interval.py | 24 +-
airflow/timetables/simple.py | 31 +-
airflow/timetables/trigger.py | 3 +-
airflow/triggers/base.py | 2 +
airflow/utils/cli_action_loggers.py | 12 +-
airflow/utils/context.py | 30 +-
airflow/utils/dag_cycle_tester.py | 3 +-
airflow/utils/dates.py | 12 +-
airflow/utils/db.py | 24 +-
airflow/utils/email.py | 3 +-
airflow/utils/entry_points.py | 3 +-
airflow/utils/file.py | 9 +-
airflow/utils/helpers.py | 17 +-
airflow/utils/json.py | 3 +-
airflow/utils/log/file_task_handler.py | 3 +-
airflow/utils/log/logging_mixin.py | 6 +-
airflow/utils/log/secrets_masker.py | 12 +-
airflow/utils/log/timezone_aware.py | 6 +-
airflow/utils/mixins.py | 6 +-
airflow/utils/operator_helpers.py | 6 +-
airflow/utils/process_utils.py | 3 +-
airflow/utils/setup_teardown.py | 3 +-
airflow/utils/sqlalchemy.py | 6 +-
airflow/utils/state.py | 6 +-
airflow/utils/task_group.py | 28 +-
airflow/utils/timezone.py | 12 +-
airflow/utils/types.py | 3 +-
airflow/utils/yaml.py | 3 +-
airflow/www/extensions/init_appbuilder.py | 61 +-
airflow/www/extensions/init_manifest_files.py | 6 +-
airflow/www/extensions/init_views.py | 9 +-
airflow/www/forms.py | 9 +-
airflow/www/security.py | 3 +-
airflow/www/security_appless.py | 3 +-
airflow/www/security_manager.py | 6 +-
airflow/www/static/css/bootstrap-theme.css | 21 +
airflow/www/static/js/api/index.ts | 2 +
airflow/www/static/js/api/useTIHistory.ts | 64 ++
airflow/www/static/js/components/ErrorAlert.tsx | 36 +
airflow/www/static/js/dag/StatusBox.tsx | 2 +-
airflow/www/static/js/dag/details/dag/Calendar.tsx | 9 +-
.../www/static/js/dag/details/dagCode/index.tsx | 13 +-
.../static/js/dag/details/gantt/GanttTooltip.tsx | 25 +-
.../static/js/dag/details/gantt/InstanceBar.tsx | 142 ++++
airflow/www/static/js/dag/details/gantt/Row.tsx | 119 +--
airflow/www/static/js/dag/details/gantt/index.tsx | 3 +-
airflow/www/static/js/dag/details/index.tsx | 2 +-
.../static/js/dag/details/taskInstance/Details.tsx | 120 +--
.../dag/details/taskInstance/Logs/LogLink.test.tsx | 2 +-
.../js/dag/details/taskInstance/Logs/LogLink.tsx | 12 +-
.../dag/details/taskInstance/Logs/index.test.tsx | 33 +
.../js/dag/details/taskInstance/Logs/index.tsx | 129 +---
.../dag/details/taskInstance/Logs/utils.test.tsx | 25 +-
.../js/dag/details/taskInstance/Logs/utils.ts | 2 +-
.../dag/details/taskInstance/TaskDocumentation.tsx | 10 +-
.../details/taskInstance/TaskFailedDependency.tsx | 22 +-
.../js/dag/details/taskInstance/TrySelector.tsx | 137 ++++
.../js/dag/details/taskInstance/Xcom/XcomEntry.tsx | 9 +-
.../js/dag/details/taskInstance/Xcom/index.tsx | 14 +-
airflow/www/static/js/toggle_theme.js | 46 ++
airflow/www/static/js/types/index.ts | 1 +
...{useErrorToast.test.tsx => handleError.test.ts} | 16 +-
.../js/utils/{useErrorToast.ts => handleError.ts} | 41 +-
airflow/www/static/js/utils/useErrorToast.ts | 25 +-
airflow/www/templates/airflow/dag.html | 1 +
airflow/www/templates/airflow/dags.html | 6 -
airflow/www/templates/airflow/main.html | 2 +-
airflow/www/templates/appbuilder/navbar.html | 10 +
airflow/www/utils.py | 9 +-
airflow/www/validators.py | 9 +-
airflow/www/views.py | 35 +-
chart/newsfragments/40318.misc.rst | 1 +
chart/templates/_helpers.yaml | 3 +
chart/templates/configmaps/extra-configmaps.yaml | 16 +-
chart/templates/jobs/create-user-job.yaml | 4 +
chart/templates/jobs/migrate-database-job.yaml | 4 +
chart/templates/redis/redis-statefulset.yaml | 3 +
.../templates/scheduler/scheduler-deployment.yaml | 3 +
chart/templates/secrets/extra-secrets.yaml | 16 +-
.../secrets/metadata-connection-secret.yaml | 4 +-
.../templates/triggerer/triggerer-deployment.yaml | 5 +-
chart/templates/workers/worker-deployment.yaml | 3 +
chart/values.schema.json | 70 +-
chart/values.yaml | 33 +-
clients/python/pyproject.toml | 2 +-
contributing-docs/03_contributors_quick_start.rst | 3 +-
.../12_airflow_dependencies_and_extras.rst | 14 +-
contributing-docs/16_contribution_workflow.rst | 2 +-
.../contributors_quick_start_codespaces.rst | 10 +-
contributing-docs/testing/k8s_tests.rst | 2 +-
contributing-docs/testing/unit_tests.rst | 4 +-
dev/breeze/doc/04_troubleshooting.rst | 8 +
dev/breeze/doc/images/output-commands.svg | 134 ++--
dev/breeze/doc/images/output_build-docs.svg | 42 +-
dev/breeze/doc/images/output_build-docs.txt | 2 +-
...tput_release-management_add-back-references.svg | 26 +-
...tput_release-management_add-back-references.txt | 2 +-
...management_generate-issue-content-providers.svg | 18 +-
...management_generate-issue-content-providers.txt | 2 +-
...e-management_prepare-provider-documentation.svg | 32 +-
...e-management_prepare-provider-documentation.txt | 2 +-
...elease-management_prepare-provider-packages.svg | 34 +-
...elease-management_prepare-provider-packages.txt | 2 +-
.../output_release-management_publish-docs.svg | 42 +-
.../output_release-management_publish-docs.txt | 2 +-
...output_sbom_generate-providers-requirements.svg | 70 +-
...output_sbom_generate-providers-requirements.txt | 2 +-
dev/breeze/doc/images/output_setup_config.svg | 6 +-
dev/breeze/doc/images/output_setup_config.txt | 2 +-
dev/breeze/doc/images/output_shell.svg | 268 +++----
dev/breeze/doc/images/output_shell.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 192 ++---
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
dev/breeze/doc/images/output_testing_db-tests.svg | 176 ++---
dev/breeze/doc/images/output_testing_db-tests.txt | 2 +-
.../images/output_testing_integration-tests.svg | 76 +-
.../images/output_testing_integration-tests.txt | 2 +-
dev/breeze/doc/images/output_testing_tests.svg | 190 ++---
dev/breeze/doc/images/output_testing_tests.txt | 2 +-
.../src/airflow_breeze/commands/common_options.py | 5 +-
.../airflow_breeze/commands/testing_commands.py | 22 +-
dev/breeze/src/airflow_breeze/global_constants.py | 3 +-
.../src/airflow_breeze/params/shell_params.py | 6 +-
.../prepare_providers/provider_documentation.py | 185 ++++-
dev/breeze/src/airflow_breeze/utils/github.py | 25 +-
dev/breeze/src/airflow_breeze/utils/packages.py | 11 +
dev/breeze/tests/test_provider_documentation.py | 48 +-
docker_tests/requirements.txt | 2 +-
docs/apache-airflow-providers-amazon/commits.rst | 19 +-
docs/apache-airflow-providers-amazon/index.rst | 6 +-
.../operators/emr/emr_serverless.rst | 3 +-
.../operators/glue.rst | 19 +
.../commits.rst | 14 +-
.../index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 8 +-
.../commits.rst | 15 +-
.../apache-airflow-providers-apache-hdfs/index.rst | 13 +-
.../commits.rst | 15 +-
.../apache-airflow-providers-apache-hive/index.rst | 18 +-
.../commits.rst | 16 +-
.../index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 4 +-
.../commits.rst | 14 +-
.../index.rst | 8 +-
docs/apache-airflow-providers-cloudant/commits.rst | 14 +-
docs/apache-airflow-providers-cloudant/index.rst | 4 +-
.../commits.rst | 13 +
.../index.rst | 6 +-
.../operators.rst | 9 +
.../changelog.rst | 20 +-
.../commits.rst | 23 +-
.../index.rst | 24 +-
.../installing-providers-from-sources.rst | 0
.../security.rst | 0
.../commits.rst | 16 +-
docs/apache-airflow-providers-common-sql/index.rst | 6 +-
.../commits.rst | 19 +-
docs/apache-airflow-providers-databricks/index.rst | 16 +-
.../apache-airflow-providers-dbt-cloud/commits.rst | 15 +-
docs/apache-airflow-providers-dbt-cloud/index.rst | 6 +-
docs/apache-airflow-providers-docker/commits.rst | 14 +-
docs/apache-airflow-providers-docker/index.rst | 11 +-
docs/apache-airflow-providers-exasol/commits.rst | 15 +-
docs/apache-airflow-providers-exasol/index.rst | 15 +-
docs/apache-airflow-providers-fab/commits.rst | 18 +-
docs/apache-airflow-providers-fab/index.rst | 6 +-
docs/apache-airflow-providers-facebook/commits.rst | 14 +-
docs/apache-airflow-providers-facebook/index.rst | 4 +-
docs/apache-airflow-providers-ftp/commits.rst | 14 +-
docs/apache-airflow-providers-ftp/index.rst | 6 +-
docs/apache-airflow-providers-github/commits.rst | 14 +-
docs/apache-airflow-providers-github/index.rst | 4 +-
docs/apache-airflow-providers-google/commits.rst | 40 +-
docs/apache-airflow-providers-google/index.rst | 41 +-
.../operators/cloud/cloud_run.rst | 15 +-
.../operators/cloud/vertex_ai.rst | 42 +-
.../operators/transfer/bigquery_to_postgres.rst | 12 +
.../operators/transfer/local_to_gcs.rst | 2 +-
docs/apache-airflow-providers-grpc/commits.rst | 14 +-
docs/apache-airflow-providers-grpc/index.rst | 4 +-
docs/apache-airflow-providers-http/commits.rst | 14 +-
docs/apache-airflow-providers-http/index.rst | 2 +-
.../connections/influxdb.rst | 9 +-
.../commits.rst | 16 +
.../index.rst | 12 +-
.../commits.rst | 15 +-
.../index.rst | 8 +-
docs/apache-airflow-providers-mongo/commits.rst | 14 +-
docs/apache-airflow-providers-mongo/index.rst | 4 +-
docs/apache-airflow-providers-mysql/commits.rst | 15 +-
docs/apache-airflow-providers-mysql/index.rst | 8 +-
docs/apache-airflow-providers-odbc/commits.rst | 14 +-
docs/apache-airflow-providers-odbc/index.rst | 8 +-
docs/apache-airflow-providers-openai/commits.rst | 14 +-
docs/apache-airflow-providers-openai/index.rst | 4 +-
.../commits.rst | 30 +-
.../apache-airflow-providers-openlineage/index.rst | 10 +-
.../commits.rst | 14 +-
docs/apache-airflow-providers-opensearch/index.rst | 2 +-
docs/apache-airflow-providers-oracle/commits.rst | 14 +-
docs/apache-airflow-providers-oracle/index.rst | 8 +-
.../apache-airflow-providers-pagerduty/commits.rst | 14 +-
docs/apache-airflow-providers-pagerduty/index.rst | 4 +-
.../apache-airflow-providers-papermill/commits.rst | 15 +-
docs/apache-airflow-providers-papermill/index.rst | 10 +-
docs/apache-airflow-providers-pinecone/commits.rst | 14 +-
docs/apache-airflow-providers-pinecone/index.rst | 4 +-
docs/apache-airflow-providers-postgres/commits.rst | 14 +-
docs/apache-airflow-providers-postgres/index.rst | 8 +-
docs/apache-airflow-providers-presto/commits.rst | 15 +-
docs/apache-airflow-providers-presto/index.rst | 15 +-
.../commits.rst | 15 +-
docs/apache-airflow-providers-salesforce/index.rst | 11 +-
docs/apache-airflow-providers-sftp/commits.rst | 15 +-
docs/apache-airflow-providers-sftp/index.rst | 8 +-
.../apache-airflow-providers-snowflake/commits.rst | 17 +-
docs/apache-airflow-providers-snowflake/index.rst | 19 +-
docs/apache-airflow-providers-ssh/commits.rst | 14 +-
docs/apache-airflow-providers-ssh/index.rst | 4 +-
docs/apache-airflow-providers-tableau/commits.rst | 14 +-
docs/apache-airflow-providers-tableau/index.rst | 4 +-
docs/apache-airflow-providers-telegram/commits.rst | 15 +-
docs/apache-airflow-providers-telegram/index.rst | 2 +-
docs/apache-airflow-providers-teradata/commits.rst | 18 +-
docs/apache-airflow-providers-teradata/index.rst | 6 +-
.../operators/azure_blob_to_teradata.rst | 67 +-
.../operators/compute_cluster.rst | 107 +++
.../operators/s3_to_teradata.rst | 10 +-
docs/apache-airflow-providers-trino/commits.rst | 15 +-
docs/apache-airflow-providers-trino/index.rst | 15 +-
docs/apache-airflow-providers-vertica/commits.rst | 14 +-
docs/apache-airflow-providers-vertica/index.rst | 8 +-
docs/apache-airflow-providers-weaviate/commits.rst | 17 +-
.../connections.rst | 20 +-
docs/apache-airflow-providers-weaviate/index.rst | 14 +-
docs/apache-airflow-providers-yandex/commits.rst | 17 +-
docs/apache-airflow-providers-yandex/index.rst | 10 +-
docs/apache-airflow-providers-ydb/commits.rst | 21 +-
docs/apache-airflow-providers-ydb/index.rst | 63 ++
.../administration-and-deployment/lineage.rst | 36 +
.../authoring-and-scheduling/deferring.rst | 5 +-
.../authoring-and-scheduling/serializers.rst | 2 +-
.../authoring-and-scheduling/timetable.rst | 2 +-
docs/apache-airflow/best-practices.rst | 2 +-
docs/apache-airflow/core-concepts/params.rst | 32 +-
docs/apache-airflow/core-concepts/tasks.rst | 7 +-
docs/apache-airflow/extra-packages-ref.rst | 2 +
docs/apache-airflow/faq.rst | 2 +-
docs/apache-airflow/howto/docker-compose/index.rst | 2 +-
docs/apache-airflow/howto/set-up-database.rst | 1 +
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/index.rst | 1 +
docs/apache-airflow/installation/index.rst | 2 +-
.../installation/installing-from-sources.rst | 2 +-
docs/apache-airflow/installation/upgrading.rst | 2 +-
docs/apache-airflow/project.rst | 1 +
.../security/secrets/mask-sensitive-values.rst | 14 +-
docs/apache-airflow/troubleshooting.rst | 48 ++
docs/build_docs.py | 9 +
.../includes/installing-providers-from-sources.rst | 2 +-
docs/helm-chart/index.rst | 17 +
.../installing-helm-chart-from-sources.rst | 2 +-
docs/spelling_wordlist.txt | 4 +
generated/provider_dependencies.json | 16 +-
generated/provider_metadata.json | 368 ++++++---
helm_tests/airflow_aux/test_annotations.py | 26 +
helm_tests/airflow_aux/test_create_user_job.py | 17 +
.../airflow_aux/test_migrate_database_job.py | 17 +
helm_tests/airflow_core/test_scheduler.py | 18 +
helm_tests/airflow_core/test_triggerer.py | 32 +
helm_tests/airflow_core/test_worker.py | 20 +-
helm_tests/other/test_git_sync_worker.py | 34 +
helm_tests/other/test_keda.py | 4 +
.../security/test_extra_configmaps_secrets.py | 29 +
newsfragments/40286.significant.rst | 1 +
newsfragments/40379.improvement.rst | 1 +
pyproject.toml | 31 +-
.../check_providers_subpackages_all_have_init.py | 3 +-
scripts/docker/entrypoint_ci.sh | 10 +-
.../in_container/install_airflow_and_providers.py | 11 +
tests/always/test_project_structure.py | 9 +-
.../endpoints/test_connection_endpoint.py | 8 +-
.../endpoints/test_dag_run_endpoint.py | 20 -
.../endpoints/test_extra_link_endpoint.py | 52 +-
.../schemas/test_connection_schema.py | 4 +-
tests/cli/commands/test_config_command.py | 17 +-
tests/cli/commands/test_dag_command.py | 15 +-
tests/cli/commands/test_plugins_command.py | 4 +
tests/cli/commands/test_scheduler_command.py | 48 +-
tests/cli/test_cli_parser.py | 2 +
tests/core/test_stats.py | 4 +-
tests/dag_processing/test_job_runner.py | 8 +-
tests/dag_processing/test_processor.py | 14 +
.../dags/test_dag_for_db_queries_counter.py | 37 +-
tests/datasets/test_dataset.py | 40 +-
tests/deprecations_ignore.yml | 126 ----
tests/executors/test_sequential_executor.py | 32 +
tests/jobs/test_backfill_job.py | 369 +++++----
tests/jobs/test_scheduler_job.py | 6 +-
tests/lineage/test_hook.py | 12 +-
tests/models/test_baseoperator.py | 17 +
tests/models/test_dag.py | 31 +-
tests/models/test_dagrun.py | 24 +
tests/operators/test_trigger_dagrun.py | 4 +-
tests/providers/amazon/aws/hooks/test_s3.py | 159 ++--
.../providers/amazon/aws/operators/test_athena.py | 23 +-
tests/providers/amazon/aws/operators/test_ecs.py | 4 +-
.../amazon/aws/operators/test_emr_serverless.py | 54 --
.../amazon/aws/operators/test_glue_crawler.py | 105 ++-
.../amazon/aws/secrets/test_secrets_manager.py | 33 +-
.../aws/sensors/test_glue_catalog_partition.py | 25 +
.../amazon/aws/sensors/test_glue_crawler.py | 25 +
tests/providers/amazon/aws/triggers/test_ecs.py | 1 +
tests/providers/amazon/aws/triggers/test_glue.py | 22 +
.../amazon/aws/triggers/test_glue_crawler.py | 24 +-
.../cncf/kubernetes/utils/test_pod_manager.py | 2 +-
.../providers/common/compat}/__init__.py | 0
.../providers/common/compat/test_empty.py | 23 +-
tests/providers/common/sql/hooks/test_dbapi.py | 2 +-
tests/providers/common/sql/hooks/test_sql.py | 19 +
tests/providers/common/sql/operators/test_sql.py | 11 +-
tests/providers/common/sql/test_utils.py | 2 +
.../providers/databricks/hooks/test_databricks.py | 135 ++--
.../databricks/operators/test_databricks.py | 668 ++++++++++++-----
.../providers/databricks/utils/test_databricks.py | 4 +-
.../providers/google/cloud/hooks/test_dataflow.py | 72 ++
.../google/cloud/hooks/test_kubernetes_engine.py | 31 +-
tests/providers/google/cloud/hooks/test_vision.py | 1 -
.../cloud/hooks/vertex_ai/test_generative_model.py | 105 ++-
.../google/cloud/operators/test_bigquery.py | 57 +-
.../google/cloud/operators/test_cloud_batch.py | 4 +-
.../google/cloud/operators/test_dataflow.py | 69 +-
.../google/cloud/operators/test_datapipeline.py | 14 +-
.../google/cloud/operators/test_vertex_ai.py | 66 +-
.../operators/vertex_ai/test_generative_model.py | 307 ++++++--
.../google/cloud/secrets/test_secret_manager.py | 6 +-
.../google/cloud/sensors/test_cloud_composer.py | 54 +-
.../cloud/transfers/test_azure_blob_to_gcs.py | 29 +
.../google/cloud/transfers/test_bigquery_to_gcs.py | 20 +
.../cloud/transfers/test_bigquery_to_postgres.py | 45 +-
.../google/cloud/transfers/test_gcs_to_bigquery.py | 22 +
.../google/cloud/transfers/test_gcs_to_gcs.py | 18 +
.../google/cloud/transfers/test_local_to_gcs.py | 11 +-
.../marketing_platform/hooks/test_analytics.py | 25 +-
.../marketing_platform/operators/test_analytics.py | 113 +--
tests/providers/influxdb/hooks/test_influxdb.py | 11 +-
tests/providers/jdbc/operators/test_jdbc.py | 8 +-
tests/providers/microsoft/azure/hooks/test_adx.py | 11 +-
tests/providers/microsoft/azure/hooks/test_wasb.py | 30 +
tests/providers/microsoft/conftest.py | 4 +-
.../microsoft/mssql/operators/test_mssql.py | 8 +-
tests/providers/mysql/operators/test_mysql.py | 27 +-
.../providers/openlineage/plugins/test_adapter.py | 4 +
tests/providers/openlineage/plugins/test_utils.py | 26 +
tests/providers/openlineage/utils/test_utils.py | 37 +-
tests/providers/postgres/hooks/test_postgres.py | 7 +-
.../providers/postgres/operators/test_postgres.py | 49 +-
tests/providers/snowflake/hooks/test_snowflake.py | 4 +
.../snowflake/operators/test_snowflake.py | 24 +-
.../snowflake/operators/test_snowflake_sql.py | 26 +-
tests/providers/sqlite/operators/test_sqlite.py | 18 +-
.../operators/test_teradata_compute_cluster.py | 713 ++++++++++++++++++
.../providers/teradata/triggers}/__init__.py | 0
.../triggers/test_teradata_compute_cluster.py | 174 +++++
.../providers/teradata/utils}/__init__.py | 0
tests/providers/teradata/utils/test_constants.py | 110 +++
tests/providers/trino/operators/test_trino.py | 27 +-
tests/providers/weaviate/hooks/test_weaviate.py | 620 ++++++---------
.../providers/weaviate/operators/test_weaviate.py | 34 +-
tests/providers/ydb/utils/test_credentials.py | 40 +-
tests/serialization/test_dag_serialization.py | 16 +-
tests/system/providers/amazon/aws/example_glue.py | 16 +-
.../cloud/bigquery/example_bigquery_to_postgres.py | 57 +-
.../google/cloud/cloud_run/example_cloud_run.py | 95 ++-
.../google/cloud/gcs/example_gcs_to_gcs.py | 32 +-
.../example_vertex_ai_generative_model.py | 53 +-
.../providers/pinecone/example_create_pod_index.py | 13 +-
.../pinecone/example_create_serverless_index.py | 13 +-
.../example_azure_blob_to_teradata_transfer.py | 80 +-
.../teradata/example_s3_to_teradata_transfer.py | 79 +-
.../teradata/example_teradata_compute_cluster.py | 158 ++++
.../providers/weaviate/example_weaviate_cohere.py | 29 +-
.../example_weaviate_dynamic_mapping_dag.py | 31 +-
.../providers/weaviate/example_weaviate_openai.py | 36 +-
.../weaviate/example_weaviate_operator.py | 146 ++--
.../weaviate/example_weaviate_using_hook.py | 80 +-
.../weaviate/example_weaviate_vectorizer_dag.py | 37 +-
.../example_weaviate_without_vectorizer_dag.py | 32 +-
tests/test_utils/asserts.py | 29 +-
tests/test_utils/mock_executor.py | 2 +
tests/test_utils/mock_operators.py | 2 +
tests/timetables/test_datasets_timetable.py | 12 +-
tests/utils/test_cli_util.py | 2 +-
tests/utils/test_helpers.py | 34 +
tests/utils/test_log_handlers.py | 7 +-
tests/utils/test_task_group.py | 20 +-
tests/www/test_utils.py | 181 ++++-
tests/www/views/test_views.py | 33 +
983 files changed, 14390 insertions(+), 6134 deletions(-)
create mode 100644 airflow/providers/apache/iceberg/.latest-doc-only-change.txt
copy airflow/providers/common/{io => compat}/.latest-doc-only-change.txt (100%)
copy airflow/providers/{ydb => common/compat}/CHANGELOG.rst (69%)
copy airflow/providers/{apache/flink => common/compat}/__init__.py (91%)
copy airflow/{migrations/versions/0006_1_6_0_adding_extra_to_log.py => providers/common/compat/provider.yaml} (58%)
create mode 100644 airflow/providers/teradata/operators/teradata_compute_cluster.py
copy airflow/{api_connexion => providers/teradata/triggers}/__init__.py (100%)
create mode 100644 airflow/providers/teradata/triggers/teradata_compute_cluster.py
copy airflow/{api_connexion => providers/teradata/utils}/__init__.py (100%)
create mode 100644 airflow/providers/teradata/utils/constants.py
create mode 100644 airflow/www/static/js/api/useTIHistory.ts
create mode 100644 airflow/www/static/js/components/ErrorAlert.tsx
create mode 100644 airflow/www/static/js/dag/details/gantt/InstanceBar.tsx
create mode 100644 airflow/www/static/js/dag/details/taskInstance/TrySelector.tsx
create mode 100644 airflow/www/static/js/toggle_theme.js
rename airflow/www/static/js/utils/{useErrorToast.test.tsx => handleError.test.ts} (82%)
copy airflow/www/static/js/utils/{useErrorToast.ts => handleError.ts} (51%)
create mode 100644 chart/newsfragments/40318.misc.rst
copy airflow/providers/ydb/CHANGELOG.rst => docs/apache-airflow-providers-common-compat/changelog.rst (66%)
copy docs/{apache-airflow-providers-ydb => apache-airflow-providers-common-compat}/commits.rst (50%)
copy docs/{apache-airflow-providers-mongo => apache-airflow-providers-common-compat}/index.rst (81%)
copy docs/{apache-airflow-providers-airbyte => apache-airflow-providers-common-compat}/installing-providers-from-sources.rst (100%)
copy docs/{apache-airflow-providers-airbyte => apache-airflow-providers-common-compat}/security.rst (100%)
create mode 100644 docs/apache-airflow-providers-teradata/operators/compute_cluster.rst
create mode 100644 docs/apache-airflow/troubleshooting.rst
create mode 100644 newsfragments/40286.significant.rst
create mode 100644 newsfragments/40379.improvement.rst
copy airflow/migrations/versions/0036_1_10_2_add_index_to_taskinstance.py => tests/dags/test_dag_for_db_queries_counter.py (56%)
copy {airflow/api_connexion => tests/providers/common/compat}/__init__.py (100%)
copy airflow/migrations/versions/0031_1_10_0_merge_heads.py => tests/providers/common/compat/test_empty.py (68%)
create mode 100644 tests/providers/teradata/operators/test_teradata_compute_cluster.py
copy {airflow/api/auth => tests/providers/teradata/triggers}/__init__.py (100%)
create mode 100644 tests/providers/teradata/triggers/test_teradata_compute_cluster.py
copy {airflow/api/auth => tests/providers/teradata/utils}/__init__.py (100%)
create mode 100644 tests/providers/teradata/utils/test_constants.py
create mode 100644 tests/system/providers/teradata/example_teradata_compute_cluster.py
(airflow) 01/01: AIP-62 support for S3Hook and S3 usage of ObjectStore
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch aip-62/s3
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit af947f3a7b5ec5f032f45490ec91926bedbc0ad8
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Tue Jul 9 14:39:44 2024 +0200
AIP-62 support for S3Hook and S3 usage of ObjectStore
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/providers/amazon/aws/hooks/s3.py | 19 ++++++++++++++++++-
airflow/providers/common/io/datasets/__init__.py | 16 ++++++++++++++++
airflow/providers/common/io/datasets/file.py | 24 ++++++++++++++++++++++++
airflow/providers/common/io/provider.yaml | 4 ++++
4 files changed, 62 insertions(+), 1 deletion(-)
diff --git a/airflow/providers/amazon/aws/hooks/s3.py b/airflow/providers/amazon/aws/hooks/s3.py
index 8ca93766e2..a30959247e 100644
--- a/airflow/providers/amazon/aws/hooks/s3.py
+++ b/airflow/providers/amazon/aws/hooks/s3.py
@@ -41,6 +41,8 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Callable
from urllib.parse import urlsplit
from uuid import uuid4
+from airflow.lineage.hook import get_hook_lineage_collector
+
if TYPE_CHECKING:
from mypy_boto3_s3.service_resource import Bucket as S3Bucket, Object as S3ResourceObject
@@ -1111,6 +1113,9 @@ class S3Hook(AwsBaseHook):
client = self.get_conn()
client.upload_file(filename, bucket_name, key, ExtraArgs=extra_args, Config=self.transfer_config)
+ get_hook_lineage_collector().add_output_dataset(
+ {"scheme": "s3", "bucket_name": bucket_name, "key": key}, self
+ )
@unify_bucket_name_and_key
@provide_bucket_name
@@ -1251,6 +1256,9 @@ class S3Hook(AwsBaseHook):
ExtraArgs=extra_args,
Config=self.transfer_config,
)
+ get_hook_lineage_collector().add_output_dataset(
+ {"scheme": "s3", "bucket_name": bucket_name, "key": key}, self
+ )
def copy_object(
self,
@@ -1306,6 +1314,12 @@ class S3Hook(AwsBaseHook):
response = self.get_conn().copy_object(
Bucket=dest_bucket_name, Key=dest_bucket_key, CopySource=copy_source, **kwargs
)
+ get_hook_lineage_collector().add_input_dataset(
+ {"scheme": "s3", "bucket_name": source_bucket_name, "key": source_bucket_key}, self
+ )
+ get_hook_lineage_collector().add_output_dataset(
+ {"scheme": "s3", "bucket_name": dest_bucket_name, "key": dest_bucket_key}, self
+ )
return response
@provide_bucket_name
@@ -1425,6 +1439,7 @@ class S3Hook(AwsBaseHook):
file_path.parent.mkdir(exist_ok=True, parents=True)
+ get_hook_lineage_collector().add_output_dataset({"scheme": "file", "path": ""}, self)
file = open(file_path, "wb")
else:
file = NamedTemporaryFile(dir=local_path, prefix="airflow_tmp_", delete=False) # type: ignore
@@ -1435,7 +1450,9 @@ class S3Hook(AwsBaseHook):
ExtraArgs=self.extra_args,
Config=self.transfer_config,
)
-
+ get_hook_lineage_collector().add_input_dataset(
+ {"scheme": "s3", "bucket_name": bucket_name, "key": key}, self
+ )
return file.name
def generate_presigned_url(
diff --git a/airflow/providers/common/io/datasets/__init__.py b/airflow/providers/common/io/datasets/__init__.py
new file mode 100644
index 0000000000..13a83393a9
--- /dev/null
+++ b/airflow/providers/common/io/datasets/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/common/io/datasets/file.py b/airflow/providers/common/io/datasets/file.py
new file mode 100644
index 0000000000..46c7499037
--- /dev/null
+++ b/airflow/providers/common/io/datasets/file.py
@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.datasets import Dataset
+
+
+def create_dataset(*, path: str) -> Dataset:
+ # We assume that we get absolute path starting with /
+ return Dataset(uri=f"file://{path}")
diff --git a/airflow/providers/common/io/provider.yaml b/airflow/providers/common/io/provider.yaml
index aa3c2ac901..0270bfaa53 100644
--- a/airflow/providers/common/io/provider.yaml
+++ b/airflow/providers/common/io/provider.yaml
@@ -51,6 +51,10 @@ operators:
xcom:
- airflow.providers.common.io.xcom.backend
+dataset-uris:
+ - schemes: [file]
+ factory: airflow.providers.common.io.datasets.file.create_dataset
+
config:
common.io:
description: Common IO configuration section