You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/09/14 06:55:21 UTC
[airflow] branch main updated: Apply PEP-563 (Postponed Evaluation of Annotations) to core airflow (#26290)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new d67ac5932d Apply PEP-563 (Postponed Evaluation of Annotations) to core airflow (#26290)
d67ac5932d is described below
commit d67ac5932dabbf06ae733fc57b48491a8029b8c2
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed Sep 14 08:55:11 2022 +0200
Apply PEP-563 (Postponed Evaluation of Annotations) to core airflow (#26290)
This PR applies PEP-563 to all core airflow Python files - i.e.
those that we want to cherry-pick to v2-* branches in order
to make cherry-picking for the upcoming 2.4* releases easier. There was
a separate PR (#26289) with non-core changes that are not going to be
cherry-picked..
This PR is a result of combining some of the 47 PRs reviewed
and approved separately (otherwise it would have been unreviewable)
The history of those PRs can be changed in:
https://github.com/apache/airflow/pulls?q=is%3Apr+label%3Afuture-annotations+is%3Aopen
Relevant discussion: https://lists.apache.org/thread/81fr042s5d3v17v83bpo24tnrr2pp0fp
Lazy consensus call: https://lists.apache.org/thread/l74nvjh8tgbtojllhwkcn7f8mfnlz4jq
---
.pre-commit-config.yaml | 6 +-
Dockerfile.ci | 7 +-
airflow/__init__.py | 5 +-
airflow/__main__.py | 3 +-
airflow/api/__init__.py | 2 +
airflow/api/auth/backend/basic_auth.py | 8 +-
airflow/api/auth/backend/default.py | 6 +-
airflow/api/auth/backend/deny_all.py | 6 +-
airflow/api/auth/backend/kerberos_auth.py | 5 +-
airflow/api/auth/backend/session.py | 6 +-
airflow/api/client/__init__.py | 2 +
airflow/api/client/api_client.py | 2 +
airflow/api/client/json_client.py | 1 +
airflow/api/client/local_client.py | 1 +
airflow/api/common/delete_dag.py | 2 +
airflow/api/common/experimental/__init__.py | 5 +-
airflow/api/common/experimental/delete_dag.py | 2 +
airflow/api/common/experimental/get_code.py | 2 +
.../api/common/experimental/get_dag_run_state.py | 5 +-
airflow/api/common/experimental/get_dag_runs.py | 6 +-
airflow/api/common/experimental/get_lineage.py | 8 +-
airflow/api/common/experimental/get_task.py | 2 +
.../api/common/experimental/get_task_instance.py | 3 +-
airflow/api/common/experimental/mark_tasks.py | 3 +
airflow/api/common/experimental/pool.py | 2 +
airflow/api/common/experimental/trigger_dag.py | 1 +
airflow/api/common/mark_tasks.py | 55 ++--
airflow/api/common/trigger_dag.py | 19 +-
airflow/api_connexion/endpoints/config_endpoint.py | 1 +
.../api_connexion/endpoints/connection_endpoint.py | 1 +
airflow/api_connexion/endpoints/dag_endpoint.py | 7 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 45 +--
.../api_connexion/endpoints/dag_source_endpoint.py | 1 +
.../endpoints/dag_warning_endpoint.py | 9 +-
.../api_connexion/endpoints/dataset_endpoint.py | 15 +-
.../api_connexion/endpoints/event_log_endpoint.py | 5 +-
.../api_connexion/endpoints/extra_link_endpoint.py | 1 +
airflow/api_connexion/endpoints/health_endpoint.py | 1 +
.../endpoints/import_error_endpoint.py | 5 +-
airflow/api_connexion/endpoints/log_endpoint.py | 6 +-
airflow/api_connexion/endpoints/plugin_endpoint.py | 2 +
airflow/api_connexion/endpoints/pool_endpoint.py | 4 +-
.../api_connexion/endpoints/provider_endpoint.py | 1 +
airflow/api_connexion/endpoints/request_dict.py | 1 +
.../endpoints/role_and_permission_endpoint.py | 8 +-
airflow/api_connexion/endpoints/task_endpoint.py | 2 +
.../endpoints/task_instance_endpoint.py | 68 ++---
airflow/api_connexion/endpoints/user_endpoint.py | 7 +-
.../api_connexion/endpoints/variable_endpoint.py | 7 +-
.../api_connexion/endpoints/version_endpoint.py | 5 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 7 +-
airflow/api_connexion/exceptions.py | 28 +-
airflow/api_connexion/parameters.py | 10 +-
airflow/api_connexion/schemas/common_schema.py | 1 +
airflow/api_connexion/schemas/config_schema.py | 7 +-
airflow/api_connexion/schemas/connection_schema.py | 6 +-
airflow/api_connexion/schemas/dag_run_schema.py | 6 +-
airflow/api_connexion/schemas/dag_schema.py | 5 +-
airflow/api_connexion/schemas/dag_source_schema.py | 1 +
.../api_connexion/schemas/dag_warning_schema.py | 6 +-
airflow/api_connexion/schemas/dataset_schema.py | 7 +-
airflow/api_connexion/schemas/enum_schemas.py | 1 +
airflow/api_connexion/schemas/error_schema.py | 6 +-
airflow/api_connexion/schemas/event_log_schema.py | 5 +-
airflow/api_connexion/schemas/health_schema.py | 1 +
airflow/api_connexion/schemas/log_schema.py | 6 +-
airflow/api_connexion/schemas/plugin_schema.py | 5 +-
airflow/api_connexion/schemas/pool_schema.py | 5 +-
airflow/api_connexion/schemas/provider_schema.py | 5 +-
.../schemas/role_and_permission_schema.py | 7 +-
airflow/api_connexion/schemas/sla_miss_schema.py | 1 +
.../api_connexion/schemas/task_instance_schema.py | 7 +-
airflow/api_connexion/schemas/task_schema.py | 5 +-
airflow/api_connexion/schemas/user_schema.py | 6 +-
airflow/api_connexion/schemas/variable_schema.py | 1 +
airflow/api_connexion/schemas/version_schema.py | 1 +
airflow/api_connexion/schemas/xcom_schema.py | 6 +-
airflow/api_connexion/security.py | 5 +-
airflow/api_connexion/types.py | 1 +
airflow/callbacks/base_callback_sink.py | 2 +-
airflow/callbacks/callback_requests.py | 25 +-
airflow/callbacks/database_callback_sink.py | 2 +-
airflow/callbacks/pipe_callback_sink.py | 3 +-
airflow/cli/cli_parser.py | 19 +-
airflow/cli/commands/celery_command.py | 4 +-
airflow/cli/commands/cheat_sheet_command.py | 14 +-
airflow/cli/commands/config_command.py | 2 +
airflow/cli/commands/connection_command.py | 8 +-
airflow/cli/commands/dag_command.py | 6 +-
airflow/cli/commands/dag_processor_command.py | 3 +-
airflow/cli/commands/db_command.py | 2 +
airflow/cli/commands/info_command.py | 9 +-
airflow/cli/commands/jobs_command.py | 5 +-
airflow/cli/commands/kerberos_command.py | 3 +-
airflow/cli/commands/kubernetes_command.py | 2 +
airflow/cli/commands/legacy_commands.py | 1 +
airflow/cli/commands/plugins_command.py | 8 +-
airflow/cli/commands/pool_command.py | 2 +
airflow/cli/commands/provider_command.py | 2 +
airflow/cli/commands/role_command.py | 3 +-
airflow/cli/commands/rotate_fernet_key_command.py | 2 +
airflow/cli/commands/scheduler_command.py | 8 +-
airflow/cli/commands/standalone_command.py | 4 +-
airflow/cli/commands/sync_perm_command.py | 2 +
airflow/cli/commands/task_command.py | 22 +-
airflow/cli/commands/triggerer_command.py | 3 +-
airflow/cli/commands/user_command.py | 6 +-
airflow/cli/commands/variable_command.py | 2 +
airflow/cli/commands/version_command.py | 2 +
airflow/cli/commands/webserver_command.py | 9 +-
airflow/cli/simple_table.py | 20 +-
airflow/compat/functools.py | 2 +
airflow/compat/sqlalchemy.py | 1 +
airflow/config_templates/airflow_local_settings.py | 25 +-
airflow/config_templates/default_airflow.cfg | 1 -
airflow/config_templates/default_celery.py | 2 +
.../config_templates/default_webserver_config.py | 2 +
airflow/configuration.py | 103 ++++---
airflow/contrib/hooks/__init__.py | 3 +-
airflow/contrib/operators/__init__.py | 3 +-
airflow/contrib/secrets/__init__.py | 2 +
airflow/contrib/sensors/__init__.py | 1 +
airflow/contrib/task_runner/__init__.py | 2 +
airflow/contrib/utils/__init__.py | 1 +
airflow/contrib/utils/log/__init__.py | 1 +
airflow/dag_processing/manager.py | 38 +--
airflow/dag_processing/processor.py | 57 ++--
airflow/datasets/__init__.py | 6 +-
airflow/datasets/manager.py | 6 +-
airflow/decorators/__init__.py | 1 +
airflow/decorators/base.py | 47 ++-
airflow/decorators/branch_python.py | 5 +-
airflow/decorators/external_python.py | 9 +-
airflow/decorators/python.py | 7 +-
airflow/decorators/python_virtualenv.py | 7 +-
airflow/decorators/short_circuit.py | 7 +-
airflow/decorators/task_group.py | 24 +-
airflow/example_dags/example_bash_operator.py | 2 +-
.../example_branch_datetime_operator.py | 3 +-
.../example_branch_day_of_week_operator.py | 3 +-
airflow/example_dags/example_branch_labels.py | 3 +-
airflow/example_dags/example_branch_operator.py | 2 +-
.../example_branch_operator_decorator.py | 2 -
.../example_branch_python_dop_operator_3.py | 3 +-
airflow/example_dags/example_complex.py | 3 +-
airflow/example_dags/example_dag_decorator.py | 6 +-
airflow/example_dags/example_datasets.py | 2 +
.../example_external_task_marker_dag.py | 2 +-
.../example_dags/example_kubernetes_executor.py | 2 +
airflow/example_dags/example_latest_only.py | 2 +-
.../example_latest_only_with_trigger.py | 1 +
.../example_local_kubernetes_executor.py | 2 +
airflow/example_dags/example_nested_branch_dag.py | 3 +-
.../example_passing_params_via_test_command.py | 2 +-
airflow/example_dags/example_python_operator.py | 3 +-
.../example_short_circuit_decorator.py | 3 +-
.../example_dags/example_short_circuit_operator.py | 3 +-
airflow/example_dags/example_skip_dag.py | 2 +-
airflow/example_dags/example_sla_dag.py | 4 +-
airflow/example_dags/example_subdag_operator.py | 2 +-
airflow/example_dags/example_task_group.py | 3 +-
.../example_dags/example_task_group_decorator.py | 2 +-
.../example_time_delta_sensor_async.py | 2 +-
.../example_dags/example_trigger_controller_dag.py | 3 +-
airflow/example_dags/example_trigger_target_dag.py | 3 +-
airflow/example_dags/example_xcom.py | 3 +-
airflow/example_dags/example_xcomargs.py | 3 +-
airflow/example_dags/libs/helper.py | 1 +
airflow/example_dags/plugins/workday.py | 7 +-
airflow/example_dags/subdags/subdag.py | 2 +-
airflow/example_dags/tutorial.py | 3 +-
airflow/example_dags/tutorial_dag.py | 4 +-
airflow/example_dags/tutorial_taskflow_api.py | 2 +-
.../tutorial_taskflow_api_virtualenv.py | 2 +-
airflow/exceptions.py | 19 +-
airflow/executors/base_executor.py | 40 +--
airflow/executors/celery_executor.py | 30 +-
airflow/executors/celery_kubernetes_executor.py | 30 +-
airflow/executors/dask_executor.py | 12 +-
airflow/executors/debug_executor.py | 13 +-
airflow/executors/executor_constants.py | 1 +
airflow/executors/executor_loader.py | 16 +-
airflow/executors/kubernetes_executor.py | 49 ++--
airflow/executors/local_executor.py | 44 +--
airflow/executors/local_kubernetes_executor.py | 30 +-
airflow/executors/sequential_executor.py | 8 +-
airflow/hooks/__init__.py | 4 +-
airflow/hooks/base.py | 18 +-
airflow/hooks/dbapi.py | 2 +
airflow/hooks/filesystem.py | 2 +-
airflow/hooks/subprocess.py | 11 +-
airflow/jobs/__init__.py | 1 -
airflow/jobs/backfill_job.py | 28 +-
airflow/jobs/base_job.py | 5 +-
airflow/jobs/local_task_job.py | 11 +-
airflow/jobs/scheduler_job.py | 59 ++--
airflow/jobs/triggerer_job.py | 17 +-
airflow/kubernetes/k8s_model.py | 5 +-
airflow/kubernetes/kube_client.py | 7 +-
airflow/kubernetes/kube_config.py | 1 +
airflow/kubernetes/kubernetes_helper_functions.py | 4 +-
airflow/kubernetes/pod.py | 4 +-
airflow/kubernetes/pod_generator.py | 31 +-
airflow/kubernetes/pod_generator_deprecated.py | 55 ++--
airflow/kubernetes/pod_launcher.py | 2 +
airflow/kubernetes/pod_launcher_deprecated.py | 13 +-
airflow/kubernetes/pod_runtime_info_env.py | 2 +
airflow/kubernetes/secret.py | 5 +-
airflow/kubernetes/volume.py | 2 +
airflow/kubernetes/volume_mount.py | 2 +
airflow/lineage/__init__.py | 10 +-
airflow/lineage/backend.py | 12 +-
airflow/lineage/entities.py | 29 +-
airflow/listeners/__init__.py | 2 +
airflow/listeners/events.py | 2 +
airflow/listeners/listener.py | 4 +-
airflow/listeners/spec.py | 10 +-
airflow/logging_config.py | 3 +-
airflow/macros/__init__.py | 6 +-
airflow/macros/hive.py | 1 +
airflow/migrations/db_types.py | 3 +-
airflow/migrations/env.py | 1 +
airflow/migrations/utils.py | 1 +
.../versions/0001_1_5_0_current_schema.py | 2 +-
.../versions/0002_1_5_0_create_is_encrypted.py | 3 +-
.../versions/0003_1_5_0_for_compatibility.py | 2 +-
.../0004_1_5_0_more_logging_into_task_isntance.py | 3 +-
.../versions/0005_1_5_2_job_id_indices.py | 3 +-
.../versions/0006_1_6_0_adding_extra_to_log.py | 3 +-
.../migrations/versions/0007_1_6_0_add_dagrun.py | 2 +-
.../versions/0008_1_6_0_task_duration.py | 2 +-
.../versions/0009_1_6_0_dagrun_config.py | 1 -
.../0010_1_6_2_add_password_column_to_user.py | 3 +-
.../versions/0011_1_6_2_dagrun_start_end.py | 2 +-
...7_0_add_notification_sent_column_to_sla_miss.py | 3 +-
..._1_7_0_add_a_column_to_track_the_encryption_.py | 2 +-
...4_1_7_0_add_is_encrypted_column_to_variable_.py | 3 +-
.../versions/0015_1_7_1_rename_user_table.py | 3 +-
.../versions/0016_1_7_1_add_ti_state_index.py | 3 +-
.../0017_1_7_1_add_task_fails_journal_table.py | 3 +-
.../versions/0018_1_7_1_add_dag_stats_table.py | 3 +-
...1_7_1_add_fractional_seconds_to_mysql_tables.py | 2 +-
.../versions/0020_1_7_1_xcom_dag_task_indices.py | 2 +-
.../0021_1_7_1_add_pid_field_to_taskinstance.py | 2 +-
..._7_1_add_dag_id_state_index_on_dag_run_table.py | 3 +-
..._1_8_2_add_max_tries_column_to_task_instance.py | 2 +-
..._1_8_2_make_xcom_value_column_a_large_binary.py | 3 +-
.../versions/0025_1_8_2_add_ti_job_id_index.py | 3 +-
.../0026_1_8_2_increase_text_size_for_mysql.py | 3 +-
.../0027_1_10_0_add_time_zone_awareness.py | 2 +-
...1_10_0_add_kubernetes_resource_checkpointing.py | 4 +-
..._1_10_0_add_executor_config_to_task_instance.py | 3 +-
...0_1_10_0_add_kubernetes_scheduler_uniqueness.py | 4 +-
.../migrations/versions/0031_1_10_0_merge_heads.py | 2 +-
.../0032_1_10_0_fix_mysql_not_null_constraint.py | 3 +-
.../versions/0033_1_10_0_fix_sqlite_foreign_key.py | 2 +-
.../versions/0034_1_10_0_index_taskfail.py | 3 +-
.../versions/0035_1_10_2_add_idx_log_dag.py | 2 +
.../0036_1_10_2_add_index_to_taskinstance.py | 2 +-
.../0037_1_10_2_add_task_reschedule_table.py | 3 +-
.../versions/0038_1_10_2_add_sm_dag_index.py | 2 +-
.../versions/0039_1_10_2_add_superuser_field.py | 2 +-
.../versions/0040_1_10_3_add_fields_to_dag.py | 2 +-
.../0041_1_10_3_add_schedule_interval_to_dag.py | 2 +-
..._1_10_3_task_reschedule_fk_on_cascade_delete.py | 2 +-
...3_1_10_4_make_taskinstance_pool_not_nullable.py | 2 +-
.../0044_1_10_7_add_serialized_dag_table.py | 3 +-
.../versions/0045_1_10_7_add_root_dag_id_to_dag.py | 2 +-
...0_5_change_datetime_to_datetime2_6_on_mssql_.py | 2 +-
.../0047_1_10_4_increase_queue_name_size_limit.py | 2 +-
.../versions/0048_1_10_3_remove_dag_stat_table.py | 2 +-
.../migrations/versions/0049_1_10_7_merge_heads.py | 2 +-
...10_7_increase_length_for_connection_password.py | 2 +-
.../versions/0051_1_10_8_add_dagtags_table.py | 2 +-
..._10_10_add_pool_slots_field_to_task_instance.py | 2 +-
...0_10_add_rendered_task_instance_fields_table.py | 2 +-
.../versions/0054_1_10_10_add_dag_code_table.py | 2 +-
..._11_add_precision_to_execution_date_in_mysql.py | 2 +-
...10_12_add_dag_hash_column_to_serialized_dag_.py | 2 +-
.../versions/0057_1_10_13_add_fab_tables.py | 2 +-
...1_10_13_increase_length_of_fab_ab_view_menu_.py | 2 +-
.../versions/0059_2_0_0_drop_user_and_chart.py | 3 +-
.../0060_2_0_0_remove_id_column_from_xcom.py | 2 +-
.../0061_2_0_0_increase_length_of_pool_name.py | 2 +-
.../versions/0062_2_0_0_add_dagrun_run_type.py | 2 +-
.../0063_2_0_0_set_conn_type_as_non_nullable.py | 2 +-
.../0064_2_0_0_add_unique_constraint_to_conn_id.py | 2 +-
.../0065_2_0_0_update_schema_for_smart_sensor.py | 3 +-
.../0066_2_0_0_add_queued_by_job_id_to_ti.py | 2 +-
.../0067_2_0_0_add_external_executor_id_to_ti.py | 2 +-
.../0068_2_0_0_drop_kuberesourceversion_and_.py | 2 +-
...2_0_0_add_scheduling_decision_to_dagrun_and_.py | 2 +-
...0_fix_mssql_exec_date_rendered_task_instance.py | 3 +-
.../0071_2_0_0_add_job_id_to_dagrun_table.py | 2 +-
...072_2_0_0_add_k8s_yaml_to_rendered_templates.py | 2 +-
.../versions/0073_2_0_0_prefix_dag_permissions.py | 2 +-
.../0074_2_0_0_resource_based_permissions.py | 3 +-
...75_2_0_0_add_description_field_to_connection.py | 2 +-
..._0_0_fix_description_field_in_connection_to_.py | 2 +-
...2_0_0_change_field_in_dagcode_to_mediumtext_.py | 3 +-
..._2_0_1_remove_can_read_permission_on_config_.py | 3 +-
...0_2_increase_size_of_connection_extra_field_.py | 2 +-
.../0080_2_0_2_change_default_pool_slots_to_1.py | 2 +-
.../0081_2_0_2_rename_last_scheduler_run_column.py | 2 +-
..._1_0_increase_pool_name_size_in_taskinstance.py | 2 +-
...0083_2_1_0_add_description_field_to_variable.py | 2 +-
..._1_0_resource_based_permissions_for_default_.py | 3 +-
...5_2_1_3_add_queued_at_column_to_dagrun_table.py | 2 +-
..._1_4_add_max_active_runs_column_to_dagmodel_.py | 2 +-
..._2_1_4_add_index_on_state_dag_id_for_queued_.py | 2 +-
.../0088_2_2_0_improve_mssql_compatibility.py | 2 +-
...89_2_2_0_make_xcom_pkey_columns_non_nullable.py | 3 +-
..._2_0_rename_concurrency_column_in_dag_table_.py | 2 +-
.../0091_2_2_0_add_trigger_table_and_task_info.py | 2 +-
...ta_interval_start_end_to_dagmodel_and_dagrun.py | 2 +-
.../0093_2_2_0_taskinstance_keyed_to_dagrun.py | 2 +-
...2_3_add_has_import_errors_column_to_dagmodel.py | 2 +-
.../versions/0095_2_2_4_add_session_table_to_db.py | 2 +-
.../0096_2_2_4_adding_index_for_dag_id_in_job.py | 2 +-
..._2_3_0_increase_length_of_email_and_username.py | 2 +-
...098_2_3_0_added_timetable_description_column.py | 2 +-
...9_2_3_0_add_task_log_filename_template_model.py | 2 +-
...2_3_0_add_taskmap_and_map_id_on_taskinstance.py | 2 +-
..._2_3_0_add_data_compressed_to_serialized_dag.py | 2 +-
.../0102_2_3_0_switch_xcom_table_to_use_run_id.py | 3 +-
.../0103_2_3_0_add_callback_request_table.py | 2 +-
...3_0_migrate_rtif_to_use_run_id_and_map_index.py | 2 +-
.../0105_2_3_0_add_map_index_to_taskfail.py | 6 +-
...on_for_fab_tables_to_add_missing_constraints.py | 3 +-
.../versions/0107_2_3_0_add_map_index_to_log.py | 3 +-
.../versions/0108_2_3_0_default_dag_view_grid.py | 2 +-
.../0109_2_3_1_add_index_for_event_in_log.py | 2 +-
...0110_2_3_2_add_cascade_to_dag_tag_foreignkey.py | 2 +-
.../0111_2_3_3_add_indexes_for_cascade_deletes.py | 2 +-
.../versions/0112_2_4_0_add_dagwarning_model.py | 2 +-
.../0113_2_4_0_compare_types_between_orm_and_db.py | 2 +-
.../versions/0114_2_4_0_add_dataset_model.py | 3 +-
.../versions/0115_2_4_0_remove_smart_sensors.py | 2 +-
.../0116_2_4_0_add_dag_owner_attributes_table.py | 2 +-
..._2_4_0_add_processor_subdir_to_dagmodel_and_.py | 2 +-
airflow/models/__init__.py | 1 +
airflow/models/abstractoperator.py | 81 +++--
airflow/models/base.py | 1 +
airflow/models/baseoperator.py | 243 ++++++++-------
airflow/models/connection.py | 36 +--
airflow/models/crypto.py | 2 +
airflow/models/dag.py | 326 ++++++++++-----------
airflow/models/dagbag.py | 23 +-
airflow/models/dagcode.py | 8 +-
airflow/models/dagparam.py | 2 +-
airflow/models/dagpickle.py | 1 +
airflow/models/dagrun.py | 152 +++++-----
airflow/models/dagwarning.py | 2 +
airflow/models/db_callback_request.py | 1 +
airflow/models/errors.py | 1 +
airflow/models/expandinput.py | 1 -
airflow/models/log.py | 1 +
airflow/models/mappedoperator.py | 140 ++++-----
airflow/models/operator.py | 1 +
airflow/models/param.py | 22 +-
airflow/models/pool.py | 11 +-
airflow/models/renderedtifields.py | 7 +-
airflow/models/serialized_dag.py | 28 +-
airflow/models/skipmixin.py | 23 +-
airflow/models/slamiss.py | 1 +
airflow/models/taskfail.py | 1 +
airflow/models/taskinstance.py | 199 ++++++-------
airflow/models/tasklog.py | 1 +
airflow/models/taskmap.py | 8 +-
airflow/models/taskmixin.py | 69 ++---
airflow/models/taskreschedule.py | 3 +-
airflow/models/trigger.py | 10 +-
airflow/models/variable.py | 7 +-
airflow/models/xcom.py | 111 +++----
airflow/models/xcom_arg.py | 107 +++----
airflow/mypy/plugin/decorators.py | 6 +-
airflow/mypy/plugin/outputs.py | 1 -
airflow/operators/__init__.py | 3 +-
airflow/operators/bash.py | 8 +-
airflow/operators/branch.py | 5 +-
airflow/operators/datetime.py | 17 +-
airflow/operators/email.py | 16 +-
airflow/operators/empty.py | 1 +
airflow/operators/generic_transfer.py | 8 +-
airflow/operators/latest_only.py | 10 +-
airflow/operators/python.py | 58 ++--
airflow/operators/smooth.py | 2 +
airflow/operators/subdag.py | 10 +-
airflow/operators/trigger_dagrun.py | 15 +-
airflow/operators/weekday.py | 12 +-
airflow/plugins_manager.py | 70 ++---
.../google/config_templates/default_config.cfg | 1 -
airflow/providers_manager.py | 90 +++---
airflow/secrets/__init__.py | 2 +
airflow/secrets/base_secrets.py | 18 +-
airflow/secrets/environment_variables.py | 8 +-
airflow/secrets/local_filesystem.py | 40 +--
airflow/secrets/metastore.py | 8 +-
airflow/security/kerberos.py | 13 +-
airflow/security/permissions.py | 1 +
airflow/security/utils.py | 4 +-
airflow/sensors/__init__.py | 3 +-
airflow/sensors/base.py | 11 +-
airflow/sensors/bash.py | 1 +
airflow/sensors/date_time.py | 5 +-
airflow/sensors/external_task.py | 23 +-
airflow/sensors/filesystem.py | 2 +-
airflow/sensors/python.py | 10 +-
airflow/sensors/time_delta.py | 1 +
airflow/sensors/time_sensor.py | 2 +
airflow/sensors/weekday.py | 6 +-
airflow/sentry.py | 3 +-
airflow/serialization/__init__.py | 1 -
airflow/serialization/enums.py | 2 +-
airflow/serialization/helpers.py | 7 +-
airflow/serialization/json_schema.py | 4 +-
airflow/serialization/serialized_objects.py | 96 +++---
airflow/settings.py | 16 +-
airflow/stats.py | 13 +-
airflow/task/task_runner/__init__.py | 2 +-
airflow/task/task_runner/base_task_runner.py | 5 +-
airflow/task/task_runner/cgroup_task_runner.py | 5 +-
airflow/task/task_runner/standard_task_runner.py | 5 +-
airflow/templates.py | 1 -
airflow/ti_deps/dep_context.py | 7 +-
airflow/ti_deps/dependencies_deps.py | 1 +
airflow/ti_deps/dependencies_states.py | 1 +
airflow/ti_deps/deps/base_ti_dep.py | 1 +
airflow/ti_deps/deps/dag_ti_slots_available_dep.py | 1 +
airflow/ti_deps/deps/dag_unpaused_dep.py | 1 +
airflow/ti_deps/deps/dagrun_backfill_dep.py | 2 +-
airflow/ti_deps/deps/dagrun_exists_dep.py | 1 +
.../ti_deps/deps/exec_date_after_start_date_dep.py | 1 +
airflow/ti_deps/deps/mapped_task_expanded.py | 2 +
airflow/ti_deps/deps/not_in_retry_period_dep.py | 1 +
airflow/ti_deps/deps/not_previously_skipped_dep.py | 2 +
airflow/ti_deps/deps/pool_slots_available_dep.py | 2 +-
airflow/ti_deps/deps/prev_dagrun_dep.py | 2 +
airflow/ti_deps/deps/ready_to_reschedule.py | 1 +
airflow/ti_deps/deps/runnable_exec_date_dep.py | 1 +
airflow/ti_deps/deps/task_concurrency_dep.py | 1 +
airflow/ti_deps/deps/task_not_running_dep.py | 1 +
airflow/ti_deps/deps/trigger_rule_dep.py | 7 +-
airflow/ti_deps/deps/valid_state_dep.py | 1 +
airflow/timetables/_cron.py | 1 -
airflow/timetables/base.py | 27 +-
airflow/timetables/events.py | 10 +-
airflow/timetables/interval.py | 21 +-
airflow/timetables/simple.py | 31 +-
airflow/timetables/trigger.py | 1 -
airflow/triggers/base.py | 5 +-
airflow/triggers/temporal.py | 5 +-
airflow/triggers/testing.py | 7 +-
airflow/typing_compat.py | 2 +-
airflow/utils/airflow_flask_app.py | 5 +-
airflow/utils/cli.py | 13 +-
airflow/utils/cli_action_loggers.py | 2 +-
airflow/utils/code_utils.py | 5 +-
airflow/utils/compression.py | 1 +
airflow/utils/configuration.py | 1 +
airflow/utils/context.py | 31 +-
airflow/utils/dag_cycle_tester.py | 10 +-
airflow/utils/dag_edges.py | 7 +-
airflow/utils/dag_parsing_context.py | 10 +-
airflow/utils/dates.py | 14 +-
airflow/utils/db.py | 28 +-
airflow/utils/db_cleanup.py | 24 +-
airflow/utils/decorators.py | 2 +-
airflow/utils/deprecation_tools.py | 6 +-
airflow/utils/docs.py | 5 +-
airflow/utils/dot_renderer.py | 16 +-
airflow/utils/edgemodifier.py | 18 +-
airflow/utils/email.py | 47 +--
airflow/utils/entry_points.py | 1 -
airflow/utils/event_scheduler.py | 1 +
airflow/utils/file.py | 42 +--
airflow/utils/helpers.py | 43 +--
airflow/utils/json.py | 1 +
airflow/utils/jwt_signer.py | 8 +-
airflow/utils/log/__init__.py | 2 +
airflow/utils/log/colored_log.py | 6 +-
airflow/utils/log/file_processor_handler.py | 1 +
airflow/utils/log/file_task_handler.py | 12 +-
airflow/utils/log/json_formatter.py | 2 +-
airflow/utils/log/log_reader.py | 11 +-
airflow/utils/log/logging_mixin.py | 6 +-
airflow/utils/log/non_caching_file_handler.py | 1 +
airflow/utils/log/secrets_masker.py | 20 +-
.../log/task_handler_with_custom_formatter.py | 7 +-
airflow/utils/log/timezone_aware.py | 2 +
airflow/utils/mixins.py | 1 +
airflow/utils/module_loading.py | 1 +
airflow/utils/net.py | 3 +-
airflow/utils/operator_helpers.py | 9 +-
airflow/utils/operator_resources.py | 4 +-
airflow/utils/orm_event_handlers.py | 1 +
airflow/utils/platform.py | 3 +-
airflow/utils/process_utils.py | 19 +-
airflow/utils/python_virtualenv.py | 20 +-
airflow/utils/retries.py | 5 +-
airflow/utils/scheduler_health.py | 1 +
airflow/utils/serve_logs.py | 3 +-
airflow/utils/session.py | 2 +
airflow/utils/sqlalchemy.py | 14 +-
airflow/utils/state.py | 19 +-
airflow/utils/strings.py | 4 +-
airflow/utils/task_group.py | 58 ++--
airflow/utils/timeout.py | 7 +-
airflow/utils/timezone.py | 25 +-
airflow/utils/trigger_rule.py | 5 +-
airflow/utils/types.py | 10 +-
airflow/utils/weekday.py | 10 +-
airflow/utils/weight_rule.py | 5 +-
airflow/utils/yaml.py | 7 +-
airflow/version.py | 2 +-
airflow/www/api/__init__.py | 1 -
airflow/www/api/experimental/__init__.py | 1 -
airflow/www/api/experimental/endpoints.py | 2 +
airflow/www/app.py | 6 +-
airflow/www/auth.py | 5 +-
airflow/www/blueprints.py | 3 +-
airflow/www/decorators.py | 1 +
airflow/www/extensions/init_appbuilder.py | 14 +-
airflow/www/extensions/init_appbuilder_links.py | 1 +
airflow/www/extensions/init_dagbag.py | 1 +
airflow/www/extensions/init_jinja_globals.py | 1 +
airflow/www/extensions/init_manifest_files.py | 1 +
airflow/www/extensions/init_robots.py | 2 +
airflow/www/extensions/init_security.py | 2 +
airflow/www/extensions/init_session.py | 1 +
airflow/www/extensions/init_views.py | 1 +
airflow/www/extensions/init_wsgi_middlewares.py | 1 +
airflow/www/fab_security/manager.py | 54 ++--
airflow/www/fab_security/sqla/manager.py | 16 +-
airflow/www/fab_security/sqla/models.py | 5 +-
airflow/www/fab_security/views.py | 2 +-
airflow/www/forms.py | 1 +
airflow/www/gunicorn_config.py | 1 +
airflow/www/security.py | 22 +-
airflow/www/session.py | 1 +
airflow/www/utils.py | 14 +-
airflow/www/validators.py | 1 +
airflow/www/views.py | 79 ++---
airflow/www/widgets.py | 2 +
dev/assign_cherry_picked_prs_with_milestone.py | 23 +-
dev/breeze/README.md | 2 +-
dev/breeze/setup.py | 2 +
dev/breeze/src/airflow_breeze/__init__.py | 2 +
dev/breeze/src/airflow_breeze/branch_defaults.py | 1 +
dev/breeze/src/airflow_breeze/breeze.py | 5 +-
.../src/airflow_breeze/commands/ci_commands.py | 20 +-
.../airflow_breeze/commands/ci_commands_config.py | 6 +-
.../airflow_breeze/commands/ci_image_commands.py | 23 +-
.../commands/ci_image_commands_config.py | 6 +-
.../airflow_breeze/commands/developer_commands.py | 39 +--
.../commands/developer_commands_config.py | 6 +-
.../airflow_breeze/commands/kubernetes_commands.py | 77 ++---
.../commands/kubernetes_commands_config.py | 10 +-
.../src/airflow_breeze/commands/main_command.py | 5 +-
.../commands/production_image_commands.py | 21 +-
.../commands/production_image_commands_config.py | 6 +-
.../commands/release_management_commands.py | 30 +-
.../commands/release_management_commands_config.py | 6 +-
.../src/airflow_breeze/commands/setup_commands.py | 20 +-
.../commands/setup_commands_config.py | 6 +-
.../airflow_breeze/commands/testing_commands.py | 23 +-
.../commands/testing_commands_config.py | 6 +-
.../src/airflow_breeze/configure_rich_click.py | 2 +
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
.../src/airflow_breeze/params/build_ci_params.py | 9 +-
.../src/airflow_breeze/params/build_prod_params.py | 10 +-
.../airflow_breeze/params/common_build_params.py | 18 +-
.../src/airflow_breeze/params/doc_build_params.py | 6 +-
.../src/airflow_breeze/params/shell_params.py | 12 +-
dev/breeze/src/airflow_breeze/utils/cache.py | 11 +-
dev/breeze/src/airflow_breeze/utils/ci_group.py | 6 +-
dev/breeze/src/airflow_breeze/utils/click_utils.py | 1 +
.../src/airflow_breeze/utils/common_options.py | 3 +-
dev/breeze/src/airflow_breeze/utils/confirm.py | 11 +-
dev/breeze/src/airflow_breeze/utils/console.py | 8 +-
.../src/airflow_breeze/utils/custom_param_types.py | 5 +-
.../airflow_breeze/utils/docker_command_utils.py | 33 +--
.../utils/find_newer_dependencies.py | 10 +-
.../src/airflow_breeze/utils/github_actions.py | 1 +
.../src/airflow_breeze/utils/host_info_utils.py | 7 +-
dev/breeze/src/airflow_breeze/utils/image.py | 29 +-
.../src/airflow_breeze/utils/kubernetes_utils.py | 33 ++-
.../utils/mark_image_as_refreshed.py | 1 +
.../src/airflow_breeze/utils/md5_build_check.py | 5 +-
dev/breeze/src/airflow_breeze/utils/parallel.py | 58 ++--
dev/breeze/src/airflow_breeze/utils/path_utils.py | 7 +-
dev/breeze/src/airflow_breeze/utils/platforms.py | 1 +
.../src/airflow_breeze/utils/python_versions.py | 4 +-
dev/breeze/src/airflow_breeze/utils/recording.py | 7 +-
dev/breeze/src/airflow_breeze/utils/registry.py | 7 +-
dev/breeze/src/airflow_breeze/utils/reinstall.py | 1 +
dev/breeze/src/airflow_breeze/utils/run_tests.py | 13 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 22 +-
.../src/airflow_breeze/utils/selective_checks.py | 1 -
dev/breeze/src/airflow_breeze/utils/visuals.py | 1 +
dev/breeze/tests/conftest.py | 2 +
dev/breeze/tests/test_cache.py | 1 +
dev/breeze/tests/test_commands.py | 2 +
dev/breeze/tests/test_docker_command_utils.py | 1 +
dev/breeze/tests/test_find_airflow_directory.py | 1 +
dev/breeze/tests/test_global_constants.py | 1 +
dev/breeze/tests/test_host_info_utils.py | 1 +
dev/breeze/tests/test_pr_info.py | 1 +
dev/breeze/tests/test_run_utils.py | 1 +
dev/breeze/tests/test_selective_checks.py | 31 +-
dev/chart/build_changelog_annotations.py | 16 +-
dev/check_files.py | 10 +-
dev/deprecations/generate_deprecated_dicts.py | 9 +-
dev/example_dags/update_example_dags_paths.py | 5 +-
dev/prepare_bulk_issues.py | 14 +-
dev/prepare_release_issue.py | 33 ++-
dev/provider_packages/prepare_provider_packages.py | 102 +++----
dev/provider_packages/remove_old_releases.py | 7 +-
dev/retag_docker_images.py | 6 +-
dev/send_email.py | 10 +-
...calculate_statistics_provider_testing_issues.py | 20 +-
dev/stats/get_important_pr_candidates.py | 8 +-
dev/system_tests/update_issue_status.py | 19 +-
dev/validate_version_added_fields_in_config.py | 20 +-
docker_tests/command_utils.py | 4 +-
docker_tests/constants.py | 1 +
docker_tests/docker_tests_utils.py | 1 +
docker_tests/test_ci_image.py | 1 +
docker_tests/test_docker_compose_quick_start.py | 4 +-
.../test_examples_of_prod_image_building.py | 1 +
docker_tests/test_prod_image.py | 1 +
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/build_docs.py | 40 +--
docs/conf.py | 20 +-
.../extending/embedding-dags/test_dag.py | 4 +-
docs/exts/airflow_intersphinx.py | 7 +-
docs/exts/docroles.py | 3 +-
docs/exts/docs_build/code_utils.py | 2 +
docs/exts/docs_build/dev_index_generator.py | 1 +
docs/exts/docs_build/docs_builder.py | 7 +-
docs/exts/docs_build/errors.py | 12 +-
docs/exts/docs_build/fetch_inventories.py | 9 +-
docs/exts/docs_build/github_action_utils.py | 1 +
docs/exts/docs_build/helm_chart_utils.py | 1 +
docs/exts/docs_build/lint_checks.py | 33 +--
docs/exts/docs_build/package_filter.py | 4 +-
docs/exts/docs_build/spelling_checks.py | 17 +-
docs/exts/docs_build/third_party_inventories.py | 1 +
docs/exts/exampleinclude.py | 2 +-
docs/exts/extra_files_with_substitutions.py | 1 +
docs/exts/operators_and_hooks_ref.py | 27 +-
docs/exts/provider_init_hack.py | 1 +
docs/exts/provider_yaml_utils.py | 7 +-
docs/exts/providers_packages_ref.py | 1 +
docs/exts/redirects.py | 2 +
docs/exts/removemarktransform.py | 3 +-
docs/exts/sphinx_script_update.py | 5 +-
docs/exts/substitution_extensions.py | 7 +-
docs/publish_docs.py | 2 +-
docs/rtd-deprecation/conf.py | 1 +
kubernetes_tests/test_base.py | 1 +
kubernetes_tests/test_kubernetes_executor.py | 1 +
kubernetes_tests/test_kubernetes_pod_operator.py | 5 +-
.../test_kubernetes_pod_operator_backcompat.py | 1 +
kubernetes_tests/test_other_executors.py | 1 +
metastore_browser/hive_metastore.py | 2 +-
scripts/ci/pre_commit/common_precommit_utils.py | 7 +-
.../pre_commit_base_operator_partial_arguments.py | 1 -
scripts/ci/pre_commit/pre_commit_boring_cyborg.py | 1 +
.../ci/pre_commit/pre_commit_breeze_cmd_line.py | 2 +
.../pre_commit_build_providers_dependencies.py | 32 +-
.../pre_commit/pre_commit_changelog_duplicates.py | 1 +
scripts/ci/pre_commit/pre_commit_chart_schema.py | 1 +
.../pre_commit_check_2_2_compatibility.py | 5 +-
.../pre_commit/pre_commit_check_init_in_tests.py | 5 +-
.../ci/pre_commit/pre_commit_check_lazy_logging.py | 1 -
scripts/ci/pre_commit/pre_commit_check_license.py | 2 +
.../pre_commit_check_order_dockerfile_extras.py | 11 +-
.../ci/pre_commit/pre_commit_check_order_setup.py | 5 +-
.../pre_commit_check_pre_commit_hooks.py | 12 +-
.../pre_commit_check_provider_yaml_files.py | 37 +--
.../pre_commit/pre_commit_check_providers_init.py | 2 +
...it_check_providers_subpackages_all_have_init.py | 4 +-
.../pre_commit_check_setup_extra_packages_ref.py | 14 +-
.../ci/pre_commit/pre_commit_check_system_tests.py | 5 +-
...re_commit_check_system_tests_hidden_in_index.py | 2 +
.../pre_commit_checkout_no_credentials.py | 2 +
.../ci/pre_commit/pre_commit_compile_www_assets.py | 2 +
.../pre_commit_compile_www_assets_dev.py | 2 +
...it_decorator_operator_implements_custom_name.py | 1 -
.../pre_commit/pre_commit_docstring_param_type.py | 2 +
scripts/ci/pre_commit/pre_commit_flake8.py | 2 +
scripts/ci/pre_commit/pre_commit_helm_lint.py | 2 +
.../pre_commit_inline_scripts_in_docker.py | 7 +-
scripts/ci/pre_commit/pre_commit_insert_extras.py | 2 +
scripts/ci/pre_commit/pre_commit_json_schema.py | 8 +-
.../ci/pre_commit/pre_commit_lint_dockerfile.py | 2 +
.../ci/pre_commit/pre_commit_local_yml_mounts.py | 2 +
.../pre_commit/pre_commit_migration_reference.py | 2 +
scripts/ci/pre_commit/pre_commit_mypy.py | 2 +
scripts/ci/pre_commit/pre_commit_newsfragments.py | 1 +
.../pre_commit_replace_bad_characters.py | 5 +-
scripts/ci/pre_commit/pre_commit_setup_cfg_file.py | 2 +
.../ci/pre_commit/pre_commit_sort_in_the_wild.py | 2 +
.../pre_commit_sort_spelling_wordlist.py | 2 +
.../ci/pre_commit/pre_commit_supported_versions.py | 2 +
.../pre_commit_sync_dag_init_decorator.py | 1 -
.../pre_commit_update_breeze_config_hash.py | 2 +
.../ci/pre_commit/pre_commit_update_er_diagram.py | 2 +
.../pre_commit_update_example_dags_paths.py | 7 +-
.../ci/pre_commit/pre_commit_update_versions.py | 2 +
.../pre_commit_vendor_k8s_json_schema.py | 1 +
.../ci/pre_commit/pre_commit_version_heads_map.py | 1 +
scripts/ci/pre_commit/pre_commit_www_lint.py | 2 +
scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py | 2 +-
scripts/ci/testing/summarize_junit_failures.py | 1 +
scripts/docker/entrypoint_ci.sh | 7 +-
scripts/in_container/check_junitxml_result.py | 1 +
scripts/in_container/run_migration_reference.py | 10 +-
scripts/in_container/run_prepare_er_diagram.py | 2 +
scripts/in_container/run_resource_check.py | 4 +-
.../in_container/update_quarantined_test_status.py | 25 +-
scripts/in_container/verify_providers.py | 92 +++---
scripts/tools/check_if_limited_dependencies.py | 2 +
scripts/tools/generate-integrations-json.py | 2 +
scripts/tools/initialize_virtualenv.py | 1 +
scripts/tools/list-integrations.py | 2 +
setup.py | 48 +--
tests/always/test_connection.py | 2 +
tests/always/test_example_dags.py | 1 +
tests/always/test_project_structure.py | 19 +-
tests/always/test_secrets.py | 3 +-
tests/always/test_secrets_backends.py | 3 +-
tests/always/test_secrets_local_filesystem.py | 1 +
tests/api/auth/backend/test_basic_auth.py | 1 +
tests/api/auth/backend/test_kerberos_auth.py | 1 +
tests/api/auth/test_client.py | 1 +
tests/api/client/test_local_client.py | 1 +
tests/api/common/experimental/test_pool.py | 1 +
tests/api/common/test_delete_dag.py | 1 +
tests/api/common/test_mark_tasks.py | 1 +
tests/api/common/test_trigger_dag.py | 1 +
tests/api/conftest.py | 1 +
tests/api_connexion/conftest.py | 1 +
.../endpoints/test_config_endpoint.py | 1 +
.../endpoints/test_connection_endpoint.py | 1 +
tests/api_connexion/endpoints/test_dag_endpoint.py | 2 +
.../endpoints/test_dag_run_endpoint.py | 2 +
.../endpoints/test_dag_source_endpoint.py | 5 +-
.../endpoints/test_dag_warning_endpoint.py | 2 +
.../endpoints/test_dataset_endpoint.py | 1 +
.../endpoints/test_event_log_endpoint.py | 1 +
.../endpoints/test_extra_link_endpoint.py | 1 +
.../endpoints/test_health_endpoint.py | 2 +
.../endpoints/test_import_error_endpoint.py | 2 +
tests/api_connexion/endpoints/test_log_endpoint.py | 2 +
.../test_mapped_task_instance_endpoint.py | 2 +
.../endpoints/test_plugin_endpoint.py | 2 +-
.../api_connexion/endpoints/test_pool_endpoint.py | 1 +
.../endpoints/test_provider_endpoint.py | 1 +
.../endpoints/test_role_and_permission_endpoint.py | 1 +
.../api_connexion/endpoints/test_task_endpoint.py | 2 +
.../endpoints/test_task_instance_endpoint.py | 2 +
.../api_connexion/endpoints/test_user_endpoint.py | 2 +
.../endpoints/test_variable_endpoint.py | 2 +
.../endpoints/test_version_endpoint.py | 2 +
.../api_connexion/endpoints/test_xcom_endpoint.py | 2 +
tests/api_connexion/schemas/test_common_schema.py | 1 +
tests/api_connexion/schemas/test_config_schema.py | 1 +
.../schemas/test_connection_schema.py | 2 +
tests/api_connexion/schemas/test_dag_run_schema.py | 2 +
tests/api_connexion/schemas/test_dag_schema.py | 1 +
tests/api_connexion/schemas/test_dataset_schema.py | 1 +
tests/api_connexion/schemas/test_error_schema.py | 2 +
.../api_connexion/schemas/test_event_log_schema.py | 1 +
tests/api_connexion/schemas/test_health_schema.py | 2 +
tests/api_connexion/schemas/test_plugin_schema.py | 1 +
tests/api_connexion/schemas/test_pool_schemas.py | 1 +
.../schemas/test_role_and_permission_schema.py | 1 +
.../schemas/test_task_instance_schema.py | 1 +
tests/api_connexion/schemas/test_task_schema.py | 1 +
tests/api_connexion/schemas/test_user_schema.py | 1 +
tests/api_connexion/schemas/test_version_schema.py | 1 +
tests/api_connexion/schemas/test_xcom_schema.py | 2 +
tests/api_connexion/test_auth.py | 1 +
tests/api_connexion/test_cors.py | 1 +
tests/api_connexion/test_error_handling.py | 3 +
tests/api_connexion/test_parameters.py | 1 +
tests/api_connexion/test_security.py | 1 +
tests/callbacks/test_callback_requests.py | 1 +
tests/cli/commands/__init__.py | 1 -
tests/cli/commands/test_celery_command.py | 2 +
tests/cli/commands/test_cheat_sheet_command.py | 4 +-
tests/cli/commands/test_config_command.py | 2 +
tests/cli/commands/test_connection_command.py | 1 +
tests/cli/commands/test_dag_command.py | 2 +
tests/cli/commands/test_dag_processor_command.py | 2 +
tests/cli/commands/test_db_command.py | 1 +
tests/cli/commands/test_info_command.py | 1 +
tests/cli/commands/test_jobs_command.py | 2 +
tests/cli/commands/test_kerberos_command.py | 1 +
tests/cli/commands/test_kubernetes_command.py | 1 +
tests/cli/commands/test_legacy_commands.py | 2 +
tests/cli/commands/test_plugins_command.py | 1 +
tests/cli/commands/test_pool_command.py | 3 +-
tests/cli/commands/test_role_command.py | 3 +-
.../cli/commands/test_rotate_fernet_key_command.py | 1 +
tests/cli/commands/test_scheduler_command.py | 2 +
tests/cli/commands/test_sync_perm_command.py | 3 +-
tests/cli/commands/test_task_command.py | 1 +
tests/cli/commands/test_triggerer_command.py | 2 +
tests/cli/commands/test_user_command.py | 1 +
tests/cli/commands/test_variable_command.py | 3 +-
tests/cli/commands/test_version_command.py | 1 +
tests/cli/commands/test_webserver_command.py | 1 +
tests/cli/conftest.py | 3 +-
tests/cli/test_cli_parser.py | 1 +
tests/cluster_policies/__init__.py | 6 +-
tests/conftest.py | 2 +
tests/core/test_config_templates.py | 2 +
tests/core/test_configuration.py | 2 +
tests/core/test_core.py | 2 +
tests/core/test_example_dags_system.py | 2 +
tests/core/test_impersonation_tests.py | 1 +
tests/core/test_logging_config.py | 2 +
tests/core/test_providers_manager.py | 5 +-
tests/core/test_sentry.py | 1 +
tests/core/test_settings.py | 3 +-
tests/core/test_sqlalchemy_config.py | 1 +
tests/core/test_stats.py | 2 +
tests/core/test_templates.py | 1 +
tests/dag_processing/test_manager.py | 1 +
tests/dag_processing/test_processor.py | 3 +-
tests/dags/no_dags.py | 2 +-
tests/dags/subdir1/test_ignore_this.py | 1 +
tests/dags/subdir2/subdir3/test_nested_dag.py | 1 +
tests/dags/subdir2/test_dont_ignore_this.py | 2 +
tests/dags/test_clear_subdag.py | 5 +-
tests/dags/test_cli_triggered_dags.py | 2 +-
tests/dags/test_dag_with_no_tags.py | 1 +
tests/dags/test_dagrun_fast_follow.py | 1 +
tests/dags/test_datasets.py | 1 +
tests/dags/test_default_impersonation.py | 1 +
tests/dags/test_default_views.py | 1 +
tests/dags/test_double_trigger.py | 2 +
tests/dags/test_example_bash_operator.py | 2 +
tests/dags/test_heartbeat_failed_fast.py | 2 +
tests/dags/test_impersonation.py | 1 +
tests/dags/test_impersonation_subdag.py | 2 +
tests/dags/test_invalid_cron.py | 1 +
tests/dags/test_invalid_param.py | 1 +
tests/dags/test_issue_1225.py | 4 +-
tests/dags/test_latest_runs.py | 2 +-
tests/dags/test_logging_in_dag.py | 2 +
tests/dags/test_mapped_classic.py | 1 +
tests/dags/test_mapped_taskflow.py | 1 +
tests/dags/test_mark_state.py | 2 +
tests/dags/test_miscellaneous.py | 2 +-
tests/dags/test_missing_owner.py | 1 +
tests/dags/test_multiple_dags.py | 2 +
tests/dags/test_no_impersonation.py | 1 +
tests/dags/test_on_failure_callback.py | 1 +
tests/dags/test_on_kill.py | 2 +
tests/dags/test_only_empty_tasks.py | 2 +
tests/dags/test_parsing_context.py | 2 +
tests/dags/test_prev_dagrun_dep.py | 1 +
tests/dags/test_retry_handling_job.py | 1 +
tests/dags/test_scheduler_dags.py | 1 +
tests/dags/test_sensor.py | 1 +
tests/dags/test_subdag.py | 3 +-
tests/dags/test_task_view_type_check.py | 4 +-
tests/dags/test_with_non_default_owner.py | 1 +
tests/dags_corrupted/test_impersonation_custom.py | 1 +
tests/dags_corrupted/test_nonstring_owner.py | 1 +
tests/dags_with_system_exit/a_system_exit.py | 8 +-
.../dags_with_system_exit/b_test_scheduler_dags.py | 1 +
tests/dags_with_system_exit/c_system_exit.py | 8 +-
tests/datasets/test_manager.py | 1 +
tests/decorators/test_branch_python.py | 2 +-
tests/decorators/test_external_python.py | 2 +
tests/decorators/test_mapped.py | 1 +
tests/decorators/test_python_virtualenv.py | 2 +
tests/decorators/test_short_circuit.py | 2 +
tests/decorators/test_task_group.py | 2 +
tests/executors/test_base_executor.py | 2 +
tests/executors/test_celery_executor.py | 2 +
tests/executors/test_celery_kubernetes_executor.py | 2 +
tests/executors/test_dask_executor.py | 2 +
tests/executors/test_debug_executor.py | 1 +
tests/executors/test_executor_loader.py | 1 +
tests/executors/test_kubernetes_executor.py | 3 +-
tests/executors/test_local_executor.py | 2 +
tests/executors/test_local_kubernetes_executor.py | 2 +
tests/executors/test_sequential_executor.py | 1 +
tests/hooks/test_subprocess.py | 1 +
tests/jobs/__init__.py | 1 -
tests/jobs/test_backfill_job.py | 2 +-
tests/jobs/test_base_job.py | 2 +-
tests/jobs/test_local_task_job.py | 6 +-
tests/jobs/test_scheduler_job.py | 8 +-
tests/jobs/test_triggerer_job.py | 1 +
tests/kubernetes/models/test_secret.py | 2 +
tests/kubernetes/test_client.py | 1 +
tests/kubernetes/test_pod_generator.py | 2 +
tests/lineage/test_lineage.py | 2 +
tests/listeners/empty_listener.py | 1 +
tests/listeners/full_listener.py | 1 +
tests/listeners/partial_listener.py | 1 +
tests/listeners/test_listeners.py | 2 +
tests/listeners/throwing_listener.py | 1 +
tests/macros/__init__.py | 1 -
tests/macros/test_hive.py | 1 +
tests/macros/test_macros.py | 3 +-
tests/models/__init__.py | 1 +
tests/models/test_base.py | 1 +
tests/models/test_baseoperator.py | 2 +
tests/models/test_cleartasks.py | 1 +
tests/models/test_dag.py | 10 +-
tests/models/test_dagbag.py | 2 +
tests/models/test_dagcode.py | 2 +
tests/models/test_dagrun.py | 7 +-
tests/models/test_dataset.py | 1 +
tests/models/test_mappedoperator.py | 2 +
tests/models/test_param.py | 1 +
tests/models/test_pool.py | 1 +
tests/models/test_renderedtifields.py | 2 +-
tests/models/test_serialized_dag.py | 2 +-
tests/models/test_skipmixin.py | 1 +
tests/models/test_taskinstance.py | 15 +-
tests/models/test_timestamp.py | 1 +
tests/models/test_trigger.py | 1 +
tests/models/test_variable.py | 1 +
tests/models/test_xcom.py | 2 +
tests/models/test_xcom_arg.py | 2 +
tests/models/test_xcom_arg_map.py | 3 +-
tests/operators/test_bash.py | 1 +
tests/operators/test_branch_operator.py | 1 +
tests/operators/test_datetime.py | 1 +
tests/operators/test_email.py | 1 +
tests/operators/test_generic_transfer.py | 1 +
tests/operators/test_latest_only_operator.py | 1 +
tests/operators/test_python.py | 5 +-
tests/operators/test_subdag_operator.py | 1 +
tests/operators/test_trigger_dagrun.py | 1 +
tests/operators/test_weekday.py | 2 +
tests/plugins/test_plugin.py | 1 +
tests/plugins/test_plugin_ignore.py | 2 +-
tests/plugins/test_plugins_manager.py | 2 +
tests/security/test_kerberos.py | 2 +
tests/sensors/test_base.py | 2 +
tests/sensors/test_bash.py | 2 +-
tests/sensors/test_date_time.py | 1 +
tests/sensors/test_external_task_sensor.py | 2 +
tests/sensors/test_filesystem.py | 2 +-
tests/sensors/test_python.py | 2 +-
tests/sensors/test_time_delta.py | 2 +
tests/sensors/test_time_sensor.py | 1 +
tests/sensors/test_timeout_sensor.py | 2 +
tests/sensors/test_weekday_sensor.py | 2 +-
tests/serialization/test_dag_serialization.py | 5 +-
tests/system/conftest.py | 1 +
tests/system/utils/__init__.py | 2 +
tests/system/utils/watcher.py | 2 +
tests/task/task_runner/test_base_task_runner.py | 2 +
tests/task/task_runner/test_cgroup_task_runner.py | 2 +
.../task/task_runner/test_standard_task_runner.py | 2 +
tests/task/task_runner/test_task_runner.py | 1 +
tests/test_utils/__init__.py | 2 +
tests/test_utils/amazon_system_helpers.py | 6 +-
tests/test_utils/api_connexion_utils.py | 2 +
tests/test_utils/asserts.py | 5 +-
tests/test_utils/azure_system_helpers.py | 1 +
tests/test_utils/config.py | 1 +
tests/test_utils/db.py | 2 +
tests/test_utils/decorators.py | 1 +
tests/test_utils/fake_datetime.py | 1 +
tests/test_utils/gcp_system_helpers.py | 14 +-
tests/test_utils/get_all_tests.py | 2 +
tests/test_utils/hdfs_utils.py | 1 +
tests/test_utils/logging_command_executor.py | 2 +
tests/test_utils/mapping.py | 3 +-
tests/test_utils/mock_executor.py | 1 +
tests/test_utils/mock_operators.py | 2 +
tests/test_utils/mock_plugins.py | 1 +
tests/test_utils/mock_security_manager.py | 1 +
tests/test_utils/perf/dags/elastic_dag.py | 1 +
tests/test_utils/perf/dags/perf_dag_1.py | 2 +
tests/test_utils/perf/dags/perf_dag_2.py | 2 +
tests/test_utils/perf/dags/sql_perf_dag.py | 1 +
tests/test_utils/perf/perf_kit/memory.py | 2 +
tests/test_utils/perf/perf_kit/python.py | 2 +
tests/test_utils/perf/perf_kit/repeat_and_time.py | 2 +
tests/test_utils/perf/perf_kit/sqlalchemy.py | 2 +
.../perf/scheduler_dag_execution_timing.py | 2 +
tests/test_utils/perf/scheduler_ops_metrics.py | 1 +
tests/test_utils/perf/sql_queries.py | 9 +-
tests/test_utils/remote_user_api_auth_backend.py | 6 +-
tests/test_utils/reset_warning_registry.py | 1 +
tests/test_utils/salesforce_system_helpers.py | 1 +
tests/test_utils/sftp_system_helpers.py | 1 +
tests/test_utils/system_tests_class.py | 5 +-
tests/test_utils/terraform.py | 1 +
tests/test_utils/timetables.py | 1 +
tests/test_utils/www.py | 2 +
tests/ti_deps/deps/fake_models.py | 1 +
.../deps/test_dag_ti_slots_available_dep.py | 2 +-
tests/ti_deps/deps/test_dag_unpaused_dep.py | 2 +-
tests/ti_deps/deps/test_dagrun_exists_dep.py | 2 +-
tests/ti_deps/deps/test_dagrun_id_dep.py | 2 +-
tests/ti_deps/deps/test_not_in_retry_period_dep.py | 2 +-
.../deps/test_not_previously_skipped_dep.py | 1 +
.../ti_deps/deps/test_pool_slots_available_dep.py | 2 +-
tests/ti_deps/deps/test_prev_dagrun_dep.py | 2 +-
tests/ti_deps/deps/test_ready_to_reschedule_dep.py | 2 +-
tests/ti_deps/deps/test_runnable_exec_date_dep.py | 2 +-
tests/ti_deps/deps/test_task_concurrency.py | 2 +-
tests/ti_deps/deps/test_task_not_running_dep.py | 2 +-
tests/ti_deps/deps/test_trigger_rule_dep.py | 2 +
tests/ti_deps/deps/test_valid_state_dep.py | 2 +-
tests/timetables/test_events_timetable.py | 2 +
tests/timetables/test_interval_timetable.py | 9 +-
tests/timetables/test_trigger_timetable.py | 1 -
tests/timetables/test_workday_timetable.py | 2 +
tests/triggers/test_temporal.py | 1 +
tests/utils/log/test_file_processor_handler.py | 1 +
tests/utils/log/test_json_formatter.py | 3 +-
tests/utils/log/test_log_reader.py | 1 +
tests/utils/log/test_secrets_masker.py | 2 +
tests/utils/test_cli_util.py | 3 +-
tests/utils/test_compression.py | 1 +
tests/utils/test_config.py | 8 +-
tests/utils/test_dag_cycle.py | 1 +
tests/utils/test_dates.py | 1 +
tests/utils/test_db.py | 1 +
tests/utils/test_db_cleanup.py | 2 +
tests/utils/test_docs.py | 1 +
tests/utils/test_dot_renderer.py | 2 +
tests/utils/test_edgemodifier.py | 1 +
tests/utils/test_email.py | 2 +
tests/utils/test_entry_points.py | 3 +-
tests/utils/test_event_scheduler.py | 1 +
tests/utils/test_file.py | 1 +
tests/utils/test_helpers.py | 2 +
tests/utils/test_json.py | 1 +
tests/utils/test_log_handlers.py | 1 +
tests/utils/test_logging_mixin.py | 1 +
tests/utils/test_module_loading.py | 1 +
tests/utils/test_net.py | 2 +
tests/utils/test_operator_helpers.py | 1 +
tests/utils/test_operator_resources.py | 1 +
...test_preexisting_python_virtualenv_decorator.py | 3 +-
tests/utils/test_process_utils.py | 2 +-
tests/utils/test_python_virtualenv.py | 3 +-
tests/utils/test_retries.py | 1 +
tests/utils/test_serve_logs.py | 24 +-
tests/utils/test_session.py | 3 +-
tests/utils/test_sqlalchemy.py | 3 +-
tests/utils/test_state.py | 1 +
tests/utils/test_task_group.py | 1 +
.../test_task_handler_with_custom_formatter.py | 2 +
tests/utils/test_timezone.py | 1 +
tests/utils/test_trigger_rule.py | 1 +
tests/utils/test_types.py | 1 +
tests/utils/test_weekday.py | 2 +
tests/utils/test_weight_rule.py | 1 +
tests/www/api/experimental/conftest.py | 2 +
.../www/api/experimental/test_dag_runs_endpoint.py | 2 +
tests/www/api/experimental/test_endpoints.py | 2 +
tests/www/test_app.py | 2 +
tests/www/test_init_views.py | 1 +
tests/www/test_security.py | 2 +
tests/www/test_utils.py | 1 +
tests/www/test_validators.py | 1 +
tests/www/views/conftest.py | 8 +-
tests/www/views/test_session.py | 1 +
tests/www/views/test_views.py | 2 +
tests/www/views/test_views_acl.py | 2 +
tests/www/views/test_views_base.py | 2 +
tests/www/views/test_views_blocked.py | 2 +
tests/www/views/test_views_configuration.py | 1 +
tests/www/views/test_views_connection.py | 2 +
tests/www/views/test_views_custom_user_views.py | 1 +
tests/www/views/test_views_dagrun.py | 2 +
tests/www/views/test_views_decorators.py | 5 +-
tests/www/views/test_views_extra_links.py | 2 +
tests/www/views/test_views_graph_gantt.py | 2 +
tests/www/views/test_views_grid.py | 5 +-
tests/www/views/test_views_home.py | 1 +
tests/www/views/test_views_log.py | 2 +
tests/www/views/test_views_mount.py | 2 +
tests/www/views/test_views_pool.py | 2 +
tests/www/views/test_views_rendered.py | 2 +
tests/www/views/test_views_robots.py | 2 +
tests/www/views/test_views_task_norun.py | 2 +
tests/www/views/test_views_tasks.py | 2 +
tests/www/views/test_views_trigger_dag.py | 2 +
tests/www/views/test_views_variable.py | 2 +
1097 files changed, 4444 insertions(+), 3462 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index eb4a592e5c..eda184522d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -215,7 +215,11 @@ repos:
args: ["--py37-plus"]
# We need to exclude gcs hook from pyupgrade because it has public "list" command which clashes
# with `list` that is used as type
- exclude: ^airflow/_vendor/|^airflow/providers/google/cloud/hooks/gcs.py$
+ # Test Python tests if different kinds of typing including one that does not have
+ # __future__ annotations, so it should be left without automated upgrade
+ # BaseOperator is disabled because replacing ClassVar[List[Type with corresponding list/type causes the attr to fail
+ # see https://github.com/apache/airflow/pull/26290#issuecomment-1246014807
+ exclude: ^airflow/_vendor/|^airflow/providers/google/cloud/hooks/gcs.py$|^test/decorators/test_python.py$|^airflow/models/baseoperator.py$
- repo: https://github.com/pre-commit/pygrep-hooks
rev: v1.9.0
hooks:
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 5498a16f7c..40911fe285 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -908,7 +908,12 @@ else
SELECTED_TESTS=()
for provider in ${BASH_REMATCH[1]//,/ }
do
- SELECTED_TESTS+=("tests/providers/${provider//./\/}")
+ providers_dir="tests/providers/${provider//./\/}"
+ if [[ -d ${providers_dir} ]]; then
+ SELECTED_TESTS+=("${providers_dir}")
+ else
+ echo "${COLOR_YELLOW}Skip ${providers_dir} as the directory does not exist.${COLOR_RESET}"
+ fi
done
else
echo
diff --git a/airflow/__init__.py b/airflow/__init__.py
index c7e020dc5a..67ef30ba5d 100644
--- a/airflow/__init__.py
+++ b/airflow/__init__.py
@@ -15,8 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
-
"""
Authentication is implemented using flask_login and different environments can
implement their own login mechanisms by providing an `airflow_login` module
@@ -25,11 +23,10 @@ in their PYTHONPATH. airflow_login should be based off the
isort:skip_file
"""
+from __future__ import annotations
# flake8: noqa: F401
-from __future__ import annotations
-
import os
import sys
from typing import Callable
diff --git a/airflow/__main__.py b/airflow/__main__.py
index 334126b2d9..e9366ac42c 100644
--- a/airflow/__main__.py
+++ b/airflow/__main__.py
@@ -17,8 +17,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Main executable module"""
+from __future__ import annotations
+
import os
import argcomplete
diff --git a/airflow/api/__init__.py b/airflow/api/__init__.py
index da07429869..d581a453af 100644
--- a/airflow/api/__init__.py
+++ b/airflow/api/__init__.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Authentication backend"""
+from __future__ import annotations
+
import logging
from importlib import import_module
diff --git a/airflow/api/auth/backend/basic_auth.py b/airflow/api/auth/backend/basic_auth.py
index 12f00b435f..29cb0358be 100644
--- a/airflow/api/auth/backend/basic_auth.py
+++ b/airflow/api/auth/backend/basic_auth.py
@@ -15,8 +15,10 @@
# specific language governing permissions and limitations
# under the License.
"""Basic authentication backend"""
+from __future__ import annotations
+
from functools import wraps
-from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+from typing import Any, Callable, TypeVar, cast
from flask import Response, request
from flask_appbuilder.const import AUTH_LDAP
@@ -25,7 +27,7 @@ from flask_login import login_user
from airflow.utils.airflow_flask_app import get_airflow_app
from airflow.www.fab_security.sqla.models import User
-CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = None
+CLIENT_AUTH: tuple[str, str] | Any | None = None
def init_app(_):
@@ -35,7 +37,7 @@ def init_app(_):
T = TypeVar("T", bound=Callable)
-def auth_current_user() -> Optional[User]:
+def auth_current_user() -> User | None:
"""Authenticate and set current user if Authorization header exists"""
auth = request.authorization
if auth is None or not auth.username or not auth.password:
diff --git a/airflow/api/auth/backend/default.py b/airflow/api/auth/backend/default.py
index 6b0a1a6c67..a33b7360dd 100644
--- a/airflow/api/auth/backend/default.py
+++ b/airflow/api/auth/backend/default.py
@@ -16,10 +16,12 @@
# specific language governing permissions and limitations
# under the License.
"""Default authentication backend - everything is allowed"""
+from __future__ import annotations
+
from functools import wraps
-from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+from typing import Any, Callable, TypeVar, cast
-CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = None
+CLIENT_AUTH: tuple[str, str] | Any | None = None
def init_app(_):
diff --git a/airflow/api/auth/backend/deny_all.py b/airflow/api/auth/backend/deny_all.py
index 614e263684..1e01975857 100644
--- a/airflow/api/auth/backend/deny_all.py
+++ b/airflow/api/auth/backend/deny_all.py
@@ -16,12 +16,14 @@
# specific language governing permissions and limitations
# under the License.
"""Authentication backend that denies all requests"""
+from __future__ import annotations
+
from functools import wraps
-from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+from typing import Any, Callable, TypeVar, cast
from flask import Response
-CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = None
+CLIENT_AUTH: tuple[str, str] | Any | None = None
def init_app(_):
diff --git a/airflow/api/auth/backend/kerberos_auth.py b/airflow/api/auth/backend/kerberos_auth.py
index fff5cfb033..505b94f8d0 100644
--- a/airflow/api/auth/backend/kerberos_auth.py
+++ b/airflow/api/auth/backend/kerberos_auth.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
#
# Copyright (c) 2013, Michael Komitee
@@ -43,7 +44,7 @@
import logging
import os
from functools import wraps
-from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+from typing import Any, Callable, TypeVar, cast
import kerberos
from flask import Response, _request_ctx_stack as stack, g, make_response, request # type: ignore
@@ -55,7 +56,7 @@ from airflow.utils.net import getfqdn
log = logging.getLogger(__name__)
-CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = HTTPKerberosAuth(service='airflow')
+CLIENT_AUTH: tuple[str, str] | Any | None = HTTPKerberosAuth(service='airflow')
class KerberosService:
diff --git a/airflow/api/auth/backend/session.py b/airflow/api/auth/backend/session.py
index 3f345c8b38..cfb693f6b7 100644
--- a/airflow/api/auth/backend/session.py
+++ b/airflow/api/auth/backend/session.py
@@ -15,12 +15,14 @@
# specific language governing permissions and limitations
# under the License.
"""Session authentication backend"""
+from __future__ import annotations
+
from functools import wraps
-from typing import Any, Callable, Optional, Tuple, TypeVar, Union, cast
+from typing import Any, Callable, TypeVar, cast
from flask import Response, g
-CLIENT_AUTH: Optional[Union[Tuple[str, str], Any]] = None
+CLIENT_AUTH: tuple[str, str] | Any | None = None
def init_app(_):
diff --git a/airflow/api/client/__init__.py b/airflow/api/client/__init__.py
index 49224b5336..c7feb01f3f 100644
--- a/airflow/api/client/__init__.py
+++ b/airflow/api/client/__init__.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""API Client that allows interacting with Airflow API"""
+from __future__ import annotations
+
from importlib import import_module
from typing import Any
diff --git a/airflow/api/client/api_client.py b/airflow/api/client/api_client.py
index c116d3b75e..28f6f804fc 100644
--- a/airflow/api/client/api_client.py
+++ b/airflow/api/client/api_client.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Client for all the API clients."""
+from __future__ import annotations
+
import httpx
diff --git a/airflow/api/client/json_client.py b/airflow/api/client/json_client.py
index d87a3aeef6..239c4e0cf1 100644
--- a/airflow/api/client/json_client.py
+++ b/airflow/api/client/json_client.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
"""JSON API Client"""
+from __future__ import annotations
from urllib.parse import urljoin
diff --git a/airflow/api/client/local_client.py b/airflow/api/client/local_client.py
index c0050672a8..6b84dc3100 100644
--- a/airflow/api/client/local_client.py
+++ b/airflow/api/client/local_client.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
"""Local client API"""
+from __future__ import annotations
from airflow.api.client import api_client
from airflow.api.common import delete_dag, trigger_dag
diff --git a/airflow/api/common/delete_dag.py b/airflow/api/common/delete_dag.py
index 5e0afa81cb..a8e2bd0a60 100644
--- a/airflow/api/common/delete_dag.py
+++ b/airflow/api/common/delete_dag.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Delete DAGs APIs."""
+from __future__ import annotations
+
import logging
from sqlalchemy import and_, or_
diff --git a/airflow/api/common/experimental/__init__.py b/airflow/api/common/experimental/__init__.py
index b161e04346..aa9316ad2e 100644
--- a/airflow/api/common/experimental/__init__.py
+++ b/airflow/api/common/experimental/__init__.py
@@ -16,14 +16,15 @@
# specific language governing permissions and limitations
# under the License.
"""Experimental APIs."""
+from __future__ import annotations
+
from datetime import datetime
-from typing import Optional
from airflow.exceptions import DagNotFound, DagRunNotFound, TaskNotFound
from airflow.models import DagBag, DagModel, DagRun
-def check_and_get_dag(dag_id: str, task_id: Optional[str] = None) -> DagModel:
+def check_and_get_dag(dag_id: str, task_id: str | None = None) -> DagModel:
"""Checks that DAG exists and in case it is specified that Task exist"""
dag_model = DagModel.get_current(dag_id)
if dag_model is None:
diff --git a/airflow/api/common/experimental/delete_dag.py b/airflow/api/common/experimental/delete_dag.py
index 36bf7dd8c4..821b80aa9a 100644
--- a/airflow/api/common/experimental/delete_dag.py
+++ b/airflow/api/common/experimental/delete_dag.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import warnings
from airflow.api.common.delete_dag import * # noqa
diff --git a/airflow/api/common/experimental/get_code.py b/airflow/api/common/experimental/get_code.py
index d4232b1d09..9e2e8c08a4 100644
--- a/airflow/api/common/experimental/get_code.py
+++ b/airflow/api/common/experimental/get_code.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Get code APIs."""
+from __future__ import annotations
+
from deprecated import deprecated
from airflow.api.common.experimental import check_and_get_dag
diff --git a/airflow/api/common/experimental/get_dag_run_state.py b/airflow/api/common/experimental/get_dag_run_state.py
index 7201186ea9..b4bef6ffa5 100644
--- a/airflow/api/common/experimental/get_dag_run_state.py
+++ b/airflow/api/common/experimental/get_dag_run_state.py
@@ -16,8 +16,9 @@
# specific language governing permissions and limitations
# under the License.
"""DAG run APIs."""
+from __future__ import annotations
+
from datetime import datetime
-from typing import Dict
from deprecated import deprecated
@@ -25,7 +26,7 @@ from airflow.api.common.experimental import check_and_get_dag, check_and_get_dag
@deprecated(reason="Use DagRun().get_state() instead", version="2.2.4")
-def get_dag_run_state(dag_id: str, execution_date: datetime) -> Dict[str, str]:
+def get_dag_run_state(dag_id: str, execution_date: datetime) -> dict[str, str]:
"""Return the Dag Run state identified by the given dag_id and execution_date.
:param dag_id: DAG id
diff --git a/airflow/api/common/experimental/get_dag_runs.py b/airflow/api/common/experimental/get_dag_runs.py
index 2064d6eb51..5c1938659a 100644
--- a/airflow/api/common/experimental/get_dag_runs.py
+++ b/airflow/api/common/experimental/get_dag_runs.py
@@ -16,7 +16,9 @@
# specific language governing permissions and limitations
# under the License.
"""DAG runs APIs."""
-from typing import Any, Dict, List, Optional
+from __future__ import annotations
+
+from typing import Any
from flask import url_for
@@ -25,7 +27,7 @@ from airflow.models import DagRun
from airflow.utils.state import DagRunState
-def get_dag_runs(dag_id: str, state: Optional[str] = None) -> List[Dict[str, Any]]:
+def get_dag_runs(dag_id: str, state: str | None = None) -> list[dict[str, Any]]:
"""
Returns a list of Dag Runs for a specific DAG ID.
diff --git a/airflow/api/common/experimental/get_lineage.py b/airflow/api/common/experimental/get_lineage.py
index 461590b702..bf1bb1a727 100644
--- a/airflow/api/common/experimental/get_lineage.py
+++ b/airflow/api/common/experimental/get_lineage.py
@@ -16,9 +16,11 @@
# specific language governing permissions and limitations
# under the License.
"""Lineage apis"""
+from __future__ import annotations
+
import collections
import datetime
-from typing import Any, Dict
+from typing import Any
from sqlalchemy.orm import Session
@@ -31,7 +33,7 @@ from airflow.utils.session import NEW_SESSION, provide_session
@provide_session
def get_lineage(
dag_id: str, execution_date: datetime.datetime, *, session: Session = NEW_SESSION
-) -> Dict[str, Dict[str, Any]]:
+) -> dict[str, dict[str, Any]]:
"""Gets the lineage information for dag specified."""
dag = check_and_get_dag(dag_id)
dagrun = check_and_get_dagrun(dag, execution_date)
@@ -39,7 +41,7 @@ def get_lineage(
inlets = XCom.get_many(dag_ids=dag_id, run_id=dagrun.run_id, key=PIPELINE_INLETS, session=session)
outlets = XCom.get_many(dag_ids=dag_id, run_id=dagrun.run_id, key=PIPELINE_OUTLETS, session=session)
- lineage: Dict[str, Dict[str, Any]] = collections.defaultdict(dict)
+ lineage: dict[str, dict[str, Any]] = collections.defaultdict(dict)
for meta in inlets:
lineage[meta.task_id]["inlets"] = meta.value
for meta in outlets:
diff --git a/airflow/api/common/experimental/get_task.py b/airflow/api/common/experimental/get_task.py
index 4589cc6ce4..52e92ea522 100644
--- a/airflow/api/common/experimental/get_task.py
+++ b/airflow/api/common/experimental/get_task.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Task APIs.."""
+from __future__ import annotations
+
from deprecated import deprecated
from airflow.api.common.experimental import check_and_get_dag
diff --git a/airflow/api/common/experimental/get_task_instance.py b/airflow/api/common/experimental/get_task_instance.py
index 7361efdc4c..e18cd8c9f0 100644
--- a/airflow/api/common/experimental/get_task_instance.py
+++ b/airflow/api/common/experimental/get_task_instance.py
@@ -15,7 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-"""Task Instance APIs."""
+from __future__ import annotations
+
from datetime import datetime
from deprecated import deprecated
diff --git a/airflow/api/common/experimental/mark_tasks.py b/airflow/api/common/experimental/mark_tasks.py
index 81cff3e30d..303c9f98ee 100644
--- a/airflow/api/common/experimental/mark_tasks.py
+++ b/airflow/api/common/experimental/mark_tasks.py
@@ -15,6 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+"""Task Instance APIs."""
+from __future__ import annotations
+
import warnings
from airflow.api.common.mark_tasks import ( # noqa
diff --git a/airflow/api/common/experimental/pool.py b/airflow/api/common/experimental/pool.py
index 12ebc19dfd..a37c3e4086 100644
--- a/airflow/api/common/experimental/pool.py
+++ b/airflow/api/common/experimental/pool.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Pool APIs."""
+from __future__ import annotations
+
from deprecated import deprecated
from airflow.exceptions import AirflowBadRequest, PoolNotFound
diff --git a/airflow/api/common/experimental/trigger_dag.py b/airflow/api/common/experimental/trigger_dag.py
index d52631281f..123b09cb1c 100644
--- a/airflow/api/common/experimental/trigger_dag.py
+++ b/airflow/api/common/experimental/trigger_dag.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import warnings
diff --git a/airflow/api/common/mark_tasks.py b/airflow/api/common/mark_tasks.py
index 83bdb2081f..d445b903b9 100644
--- a/airflow/api/common/mark_tasks.py
+++ b/airflow/api/common/mark_tasks.py
@@ -16,9 +16,10 @@
# specific language governing permissions and limitations
# under the License.
"""Marks tasks APIs."""
+from __future__ import annotations
from datetime import datetime
-from typing import TYPE_CHECKING, Collection, Iterable, Iterator, List, NamedTuple, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Collection, Iterable, Iterator, NamedTuple
from sqlalchemy import or_
from sqlalchemy.orm import lazyload
@@ -38,7 +39,7 @@ from airflow.utils.types import DagRunType
class _DagRunInfo(NamedTuple):
logical_date: datetime
- data_interval: Tuple[datetime, datetime]
+ data_interval: tuple[datetime, datetime]
def _create_dagruns(
@@ -78,9 +79,9 @@ def _create_dagruns(
@provide_session
def set_state(
*,
- tasks: Collection[Union[Operator, Tuple[Operator, int]]],
- run_id: Optional[str] = None,
- execution_date: Optional[datetime] = None,
+ tasks: Collection[Operator | tuple[Operator, int]],
+ run_id: str | None = None,
+ execution_date: datetime | None = None,
upstream: bool = False,
downstream: bool = False,
future: bool = False,
@@ -88,7 +89,7 @@ def set_state(
state: TaskInstanceState = TaskInstanceState.SUCCESS,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
"""
Set the state of a task instance and if needed its relatives. Can set state
for future tasks (calculated from run_id) and retroactively
@@ -163,7 +164,7 @@ def set_state(
def all_subdag_tasks_query(
- sub_dag_run_ids: List[str],
+ sub_dag_run_ids: list[str],
session: SASession,
state: TaskInstanceState,
confirmed_dates: Iterable[datetime],
@@ -181,7 +182,7 @@ def get_all_dag_task_query(
dag: DAG,
session: SASession,
state: TaskInstanceState,
- task_ids: List[Union[str, Tuple[str, int]]],
+ task_ids: list[str | tuple[str, int]],
run_ids: Iterable[str],
):
"""Get all tasks of the main dag that will be affected by a state change"""
@@ -201,7 +202,7 @@ def _iter_subdag_run_ids(
dag: DAG,
session: SASession,
state: DagRunState,
- task_ids: List[str],
+ task_ids: list[str],
commit: bool,
confirmed_infos: Iterable[_DagRunInfo],
) -> Iterator[str]:
@@ -261,7 +262,7 @@ def verify_dagruns(
session.merge(dag_run)
-def _iter_existing_dag_run_infos(dag: DAG, run_ids: List[str], session: SASession) -> Iterator[_DagRunInfo]:
+def _iter_existing_dag_run_infos(dag: DAG, run_ids: list[str], session: SASession) -> Iterator[_DagRunInfo]:
for dag_run in DagRun.find(dag_id=dag.dag_id, run_id=run_ids, session=session):
dag_run.dag = dag
dag_run.verify_integrity(session=session)
@@ -288,7 +289,7 @@ def find_task_relatives(tasks, downstream, upstream):
@provide_session
def get_execution_dates(
dag: DAG, execution_date: datetime, future: bool, past: bool, *, session: SASession = NEW_SESSION
-) -> List[datetime]:
+) -> list[datetime]:
"""Returns dates of DAG execution"""
latest_execution_date = dag.get_latest_execution_date(session=session)
if latest_execution_date is None:
@@ -371,11 +372,11 @@ def _set_dag_run_state(dag_id: str, run_id: str, state: DagRunState, session: SA
def set_dag_run_state_to_success(
*,
dag: DAG,
- execution_date: Optional[datetime] = None,
- run_id: Optional[str] = None,
+ execution_date: datetime | None = None,
+ run_id: str | None = None,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
"""
Set the dag run for a specific execution date and its task instances
to success.
@@ -418,11 +419,11 @@ def set_dag_run_state_to_success(
def set_dag_run_state_to_failed(
*,
dag: DAG,
- execution_date: Optional[datetime] = None,
- run_id: Optional[str] = None,
+ execution_date: datetime | None = None,
+ run_id: str | None = None,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
"""
Set the dag run for a specific execution date or run_id and its running task instances
to failed.
@@ -493,11 +494,11 @@ def __set_dag_run_state_to_running_or_queued(
*,
new_state: DagRunState,
dag: DAG,
- execution_date: Optional[datetime] = None,
- run_id: Optional[str] = None,
+ execution_date: datetime | None = None,
+ run_id: str | None = None,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
"""
Set the dag run for a specific execution date to running.
@@ -509,7 +510,7 @@ def __set_dag_run_state_to_running_or_queued(
:return: If commit is true, list of tasks that have been updated,
otherwise list of tasks that will be updated
"""
- res: List[TaskInstance] = []
+ res: list[TaskInstance] = []
if not (execution_date is None) ^ (run_id is None):
return res
@@ -539,11 +540,11 @@ def __set_dag_run_state_to_running_or_queued(
def set_dag_run_state_to_running(
*,
dag: DAG,
- execution_date: Optional[datetime] = None,
- run_id: Optional[str] = None,
+ execution_date: datetime | None = None,
+ run_id: str | None = None,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
return __set_dag_run_state_to_running_or_queued(
new_state=DagRunState.RUNNING,
dag=dag,
@@ -558,11 +559,11 @@ def set_dag_run_state_to_running(
def set_dag_run_state_to_queued(
*,
dag: DAG,
- execution_date: Optional[datetime] = None,
- run_id: Optional[str] = None,
+ execution_date: datetime | None = None,
+ run_id: str | None = None,
commit: bool = False,
session: SASession = NEW_SESSION,
-) -> List[TaskInstance]:
+) -> list[TaskInstance]:
return __set_dag_run_state_to_running_or_queued(
new_state=DagRunState.QUEUED,
dag=dag,
diff --git a/airflow/api/common/trigger_dag.py b/airflow/api/common/trigger_dag.py
index a76c54ca11..2ed72722e5 100644
--- a/airflow/api/common/trigger_dag.py
+++ b/airflow/api/common/trigger_dag.py
@@ -16,9 +16,10 @@
# specific language governing permissions and limitations
# under the License.
"""Triggering DAG runs APIs."""
+from __future__ import annotations
+
import json
from datetime import datetime
-from typing import List, Optional, Union
from airflow.exceptions import DagNotFound, DagRunAlreadyExists
from airflow.models import DagBag, DagModel, DagRun
@@ -30,11 +31,11 @@ from airflow.utils.types import DagRunType
def _trigger_dag(
dag_id: str,
dag_bag: DagBag,
- run_id: Optional[str] = None,
- conf: Optional[Union[dict, str]] = None,
- execution_date: Optional[datetime] = None,
+ run_id: str | None = None,
+ conf: dict | str | None = None,
+ execution_date: datetime | None = None,
replace_microseconds: bool = True,
-) -> List[Optional[DagRun]]:
+) -> list[DagRun | None]:
"""Triggers DAG run.
:param dag_id: DAG ID
@@ -101,11 +102,11 @@ def _trigger_dag(
def trigger_dag(
dag_id: str,
- run_id: Optional[str] = None,
- conf: Optional[Union[dict, str]] = None,
- execution_date: Optional[datetime] = None,
+ run_id: str | None = None,
+ conf: dict | str | None = None,
+ execution_date: datetime | None = None,
replace_microseconds: bool = True,
-) -> Optional[DagRun]:
+) -> DagRun | None:
"""Triggers execution of DAG specified by dag_id
:param dag_id: DAG ID
diff --git a/airflow/api_connexion/endpoints/config_endpoint.py b/airflow/api_connexion/endpoints/config_endpoint.py
index bdd2b3a959..87c96b8b2f 100644
--- a/airflow/api_connexion/endpoints/config_endpoint.py
+++ b/airflow/api_connexion/endpoints/config_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from http import HTTPStatus
diff --git a/airflow/api_connexion/endpoints/connection_endpoint.py b/airflow/api_connexion/endpoints/connection_endpoint.py
index b196b3236b..e272bb8172 100644
--- a/airflow/api_connexion/endpoints/connection_endpoint.py
+++ b/airflow/api_connexion/endpoints/connection_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import os
from http import HTTPStatus
diff --git a/airflow/api_connexion/endpoints/dag_endpoint.py b/airflow/api_connexion/endpoints/dag_endpoint.py
index 7940a25c8f..35b4504ef9 100644
--- a/airflow/api_connexion/endpoints/dag_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_endpoint.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from http import HTTPStatus
-from typing import Collection, Optional
+from typing import Collection
from connexion import NoContent
from flask import g, request
@@ -70,8 +71,8 @@ def get_dags(
*,
limit: int,
offset: int = 0,
- tags: Optional[Collection[str]] = None,
- dag_id_pattern: Optional[str] = None,
+ tags: Collection[str] | None = None,
+ dag_id_pattern: str | None = None,
only_active: bool = True,
session: Session = NEW_SESSION,
) -> APIResponse:
diff --git a/airflow/api_connexion/endpoints/dag_run_endpoint.py b/airflow/api_connexion/endpoints/dag_run_endpoint.py
index eb7026b5df..eb16aba432 100644
--- a/airflow/api_connexion/endpoints/dag_run_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_run_endpoint.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from http import HTTPStatus
-from typing import List, Optional, Tuple
import pendulum
from connexion import NoContent
@@ -102,7 +103,7 @@ def get_upstream_dataset_events(
*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION
) -> APIResponse:
"""If dag run is dataset-triggered, return the dataset events that triggered it."""
- dag_run: Optional[DagRun] = (
+ dag_run: DagRun | None = (
session.query(DagRun)
.filter(
DagRun.dag_id == dag_id,
@@ -124,16 +125,16 @@ def get_upstream_dataset_events(
def _fetch_dag_runs(
query: Query,
*,
- end_date_gte: Optional[str],
- end_date_lte: Optional[str],
- execution_date_gte: Optional[str],
- execution_date_lte: Optional[str],
- start_date_gte: Optional[str],
- start_date_lte: Optional[str],
- limit: Optional[int],
- offset: Optional[int],
+ end_date_gte: str | None,
+ end_date_lte: str | None,
+ execution_date_gte: str | None,
+ execution_date_lte: str | None,
+ start_date_gte: str | None,
+ start_date_lte: str | None,
+ limit: int | None,
+ offset: int | None,
order_by: str,
-) -> Tuple[List[DagRun], int]:
+) -> tuple[list[DagRun], int]:
if start_date_gte:
query = query.filter(DagRun.start_date >= start_date_gte)
if start_date_lte:
@@ -187,15 +188,15 @@ def _fetch_dag_runs(
def get_dag_runs(
*,
dag_id: str,
- start_date_gte: Optional[str] = None,
- start_date_lte: Optional[str] = None,
- execution_date_gte: Optional[str] = None,
- execution_date_lte: Optional[str] = None,
- end_date_gte: Optional[str] = None,
- end_date_lte: Optional[str] = None,
- state: Optional[List[str]] = None,
- offset: Optional[int] = None,
- limit: Optional[int] = None,
+ start_date_gte: str | None = None,
+ start_date_lte: str | None = None,
+ execution_date_gte: str | None = None,
+ execution_date_lte: str | None = None,
+ end_date_gte: str | None = None,
+ end_date_lte: str | None = None,
+ state: list[str] | None = None,
+ offset: int | None = None,
+ limit: int | None = None,
order_by: str = "id",
session: Session = NEW_SESSION,
):
@@ -340,7 +341,7 @@ def post_dag_run(*, dag_id: str, session: Session = NEW_SESSION) -> APIResponse:
@provide_session
def update_dag_run_state(*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION) -> APIResponse:
"""Set a state of a dag run."""
- dag_run: Optional[DagRun] = (
+ dag_run: DagRun | None = (
session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.run_id == dag_run_id).one_or_none()
)
if dag_run is None:
@@ -372,7 +373,7 @@ def update_dag_run_state(*, dag_id: str, dag_run_id: str, session: Session = NEW
@provide_session
def clear_dag_run(*, dag_id: str, dag_run_id: str, session: Session = NEW_SESSION) -> APIResponse:
"""Clear a dag run."""
- dag_run: Optional[DagRun] = (
+ dag_run: DagRun | None = (
session.query(DagRun).filter(DagRun.dag_id == dag_id, DagRun.run_id == dag_run_id).one_or_none()
)
if dag_run is None:
diff --git a/airflow/api_connexion/endpoints/dag_source_endpoint.py b/airflow/api_connexion/endpoints/dag_source_endpoint.py
index ad6209221e..2afa583dce 100644
--- a/airflow/api_connexion/endpoints/dag_source_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_source_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from http import HTTPStatus
diff --git a/airflow/api_connexion/endpoints/dag_warning_endpoint.py b/airflow/api_connexion/endpoints/dag_warning_endpoint.py
index 9110f69846..ead6c7881c 100644
--- a/airflow/api_connexion/endpoints/dag_warning_endpoint.py
+++ b/airflow/api_connexion/endpoints/dag_warning_endpoint.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from typing import Optional
+from __future__ import annotations
from sqlalchemy.orm import Session
@@ -37,9 +36,9 @@ from airflow.utils.session import NEW_SESSION, provide_session
def get_dag_warnings(
*,
limit: int,
- dag_id: Optional[str] = None,
- warning_type: Optional[str] = None,
- offset: Optional[int] = None,
+ dag_id: str | None = None,
+ warning_type: str | None = None,
+ offset: int | None = None,
order_by: str = "timestamp",
session: Session = NEW_SESSION,
) -> APIResponse:
diff --git a/airflow/api_connexion/endpoints/dataset_endpoint.py b/airflow/api_connexion/endpoints/dataset_endpoint.py
index c68ff7bb76..b9dd9dfae0 100644
--- a/airflow/api_connexion/endpoints/dataset_endpoint.py
+++ b/airflow/api_connexion/endpoints/dataset_endpoint.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from typing import Optional
+from __future__ import annotations
from sqlalchemy import func
from sqlalchemy.orm import Session, joinedload, subqueryload
@@ -61,7 +60,7 @@ def get_datasets(
*,
limit: int,
offset: int = 0,
- uri_pattern: Optional[str] = None,
+ uri_pattern: str | None = None,
order_by: str = "id",
session: Session = NEW_SESSION,
) -> APIResponse:
@@ -90,11 +89,11 @@ def get_dataset_events(
limit: int,
offset: int = 0,
order_by: str = "timestamp",
- dataset_id: Optional[int] = None,
- source_dag_id: Optional[str] = None,
- source_task_id: Optional[str] = None,
- source_run_id: Optional[str] = None,
- source_map_index: Optional[int] = None,
+ dataset_id: int | None = None,
+ source_dag_id: str | None = None,
+ source_task_id: str | None = None,
+ source_run_id: str | None = None,
+ source_map_index: int | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get dataset events"""
diff --git a/airflow/api_connexion/endpoints/event_log_endpoint.py b/airflow/api_connexion/endpoints/event_log_endpoint.py
index 590190ab98..38a6d9a733 100644
--- a/airflow/api_connexion/endpoints/event_log_endpoint.py
+++ b/airflow/api_connexion/endpoints/event_log_endpoint.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from typing import Optional
+from __future__ import annotations
from sqlalchemy import func
from sqlalchemy.orm import Session
@@ -50,7 +49,7 @@ def get_event_log(*, event_log_id: int, session: Session = NEW_SESSION) -> APIRe
def get_event_logs(
*,
limit: int,
- offset: Optional[int] = None,
+ offset: int | None = None,
order_by: str = "event_log_id",
session: Session = NEW_SESSION,
) -> APIResponse:
diff --git a/airflow/api_connexion/endpoints/extra_link_endpoint.py b/airflow/api_connexion/endpoints/extra_link_endpoint.py
index c464979719..0bc5b21836 100644
--- a/airflow/api_connexion/endpoints/extra_link_endpoint.py
+++ b/airflow/api_connexion/endpoints/extra_link_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from sqlalchemy.orm.session import Session
diff --git a/airflow/api_connexion/endpoints/health_endpoint.py b/airflow/api_connexion/endpoints/health_endpoint.py
index 380225bf16..a60da3c4bc 100644
--- a/airflow/api_connexion/endpoints/health_endpoint.py
+++ b/airflow/api_connexion/endpoints/health_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from airflow.api_connexion.schemas.health_schema import health_schema
from airflow.api_connexion.types import APIResponse
diff --git a/airflow/api_connexion/endpoints/import_error_endpoint.py b/airflow/api_connexion/endpoints/import_error_endpoint.py
index 9a46b3aac4..165f1629ae 100644
--- a/airflow/api_connexion/endpoints/import_error_endpoint.py
+++ b/airflow/api_connexion/endpoints/import_error_endpoint.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from typing import Optional
+from __future__ import annotations
from sqlalchemy import func
from sqlalchemy.orm import Session
@@ -54,7 +53,7 @@ def get_import_error(*, import_error_id: int, session: Session = NEW_SESSION) ->
def get_import_errors(
*,
limit: int,
- offset: Optional[int] = None,
+ offset: int | None = None,
order_by: str = "import_error_id",
session: Session = NEW_SESSION,
) -> APIResponse:
diff --git a/airflow/api_connexion/endpoints/log_endpoint.py b/airflow/api_connexion/endpoints/log_endpoint.py
index 0ae7b2540f..a184d8a5c4 100644
--- a/airflow/api_connexion/endpoints/log_endpoint.py
+++ b/airflow/api_connexion/endpoints/log_endpoint.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Optional
+from __future__ import annotations
+
+from typing import Any
from flask import Response, request
from itsdangerous.exc import BadSignature
@@ -49,7 +51,7 @@ def get_log(
task_try_number: int,
full_content: bool = False,
map_index: int = -1,
- token: Optional[str] = None,
+ token: str | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get logs for specific task instance"""
diff --git a/airflow/api_connexion/endpoints/plugin_endpoint.py b/airflow/api_connexion/endpoints/plugin_endpoint.py
index 3bdb17a99f..b6efb774bb 100644
--- a/airflow/api_connexion/endpoints/plugin_endpoint.py
+++ b/airflow/api_connexion/endpoints/plugin_endpoint.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from airflow.api_connexion import security
from airflow.api_connexion.parameters import check_limit, format_parameters
from airflow.api_connexion.schemas.plugin_schema import PluginCollection, plugin_collection_schema
diff --git a/airflow/api_connexion/endpoints/pool_endpoint.py b/airflow/api_connexion/endpoints/pool_endpoint.py
index 594afeb49b..1e02151414 100644
--- a/airflow/api_connexion/endpoints/pool_endpoint.py
+++ b/airflow/api_connexion/endpoints/pool_endpoint.py
@@ -14,9 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from http import HTTPStatus
-from typing import Optional
from flask import Response
from marshmallow import ValidationError
@@ -64,7 +64,7 @@ def get_pools(
*,
limit: int,
order_by: str = "id",
- offset: Optional[int] = None,
+ offset: int | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get all pools"""
diff --git a/airflow/api_connexion/endpoints/provider_endpoint.py b/airflow/api_connexion/endpoints/provider_endpoint.py
index 7526e284be..ab81c16e46 100644
--- a/airflow/api_connexion/endpoints/provider_endpoint.py
+++ b/airflow/api_connexion/endpoints/provider_endpoint.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import re
diff --git a/airflow/api_connexion/endpoints/request_dict.py b/airflow/api_connexion/endpoints/request_dict.py
index 4d7ad21250..0e3a85402e 100644
--- a/airflow/api_connexion/endpoints/request_dict.py
+++ b/airflow/api_connexion/endpoints/request_dict.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from typing import Any, Mapping, cast
diff --git a/airflow/api_connexion/endpoints/role_and_permission_endpoint.py b/airflow/api_connexion/endpoints/role_and_permission_endpoint.py
index 1b25769af7..e69d7d34e2 100644
--- a/airflow/api_connexion/endpoints/role_and_permission_endpoint.py
+++ b/airflow/api_connexion/endpoints/role_and_permission_endpoint.py
@@ -14,9 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from http import HTTPStatus
-from typing import List, Optional, Tuple
from connexion import NoContent
from flask import request
@@ -40,7 +40,7 @@ from airflow.www.fab_security.sqla.models import Action, Role
from airflow.www.security import AirflowSecurityManager
-def _check_action_and_resource(sm: AirflowSecurityManager, perms: List[Tuple[str, str]]) -> None:
+def _check_action_and_resource(sm: AirflowSecurityManager, perms: list[tuple[str, str]]) -> None:
"""
Checks if the action or resource exists and raise 400 if not
@@ -65,7 +65,7 @@ def get_role(*, role_name: str) -> APIResponse:
@security.requires_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_ROLE)])
@format_parameters({"limit": check_limit})
-def get_roles(*, order_by: str = "name", limit: int, offset: Optional[int] = None) -> APIResponse:
+def get_roles(*, order_by: str = "name", limit: int, offset: int | None = None) -> APIResponse:
"""Get roles"""
appbuilder = get_airflow_app().appbuilder
session = appbuilder.get_session
@@ -89,7 +89,7 @@ def get_roles(*, order_by: str = "name", limit: int, offset: Optional[int] = Non
@security.requires_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_ACTION)])
@format_parameters({'limit': check_limit})
-def get_permissions(*, limit: int, offset: Optional[int] = None) -> APIResponse:
+def get_permissions(*, limit: int, offset: int | None = None) -> APIResponse:
"""Get permissions"""
session = get_airflow_app().appbuilder.get_session
total_entries = session.query(func.count(Action.id)).scalar()
diff --git a/airflow/api_connexion/endpoints/task_endpoint.py b/airflow/api_connexion/endpoints/task_endpoint.py
index 74b6e7e9ee..72add1492e 100644
--- a/airflow/api_connexion/endpoints/task_endpoint.py
+++ b/airflow/api_connexion/endpoints/task_endpoint.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from operator import attrgetter
from airflow import DAG
diff --git a/airflow/api_connexion/endpoints/task_instance_endpoint.py b/airflow/api_connexion/endpoints/task_instance_endpoint.py
index 910df4c929..85fe4d66cd 100644
--- a/airflow/api_connexion/endpoints/task_instance_endpoint.py
+++ b/airflow/api_connexion/endpoints/task_instance_endpoint.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Iterable, List, Optional, Tuple, TypeVar
+from __future__ import annotations
+
+from typing import Any, Iterable, TypeVar
from marshmallow import ValidationError
from sqlalchemy import and_, func, or_
@@ -161,20 +163,20 @@ def get_mapped_task_instances(
dag_id: str,
dag_run_id: str,
task_id: str,
- execution_date_gte: Optional[str] = None,
- execution_date_lte: Optional[str] = None,
- start_date_gte: Optional[str] = None,
- start_date_lte: Optional[str] = None,
- end_date_gte: Optional[str] = None,
- end_date_lte: Optional[str] = None,
- duration_gte: Optional[float] = None,
- duration_lte: Optional[float] = None,
- state: Optional[List[str]] = None,
- pool: Optional[List[str]] = None,
- queue: Optional[List[str]] = None,
- limit: Optional[int] = None,
- offset: Optional[int] = None,
- order_by: Optional[str] = None,
+ execution_date_gte: str | None = None,
+ execution_date_lte: str | None = None,
+ start_date_gte: str | None = None,
+ start_date_lte: str | None = None,
+ end_date_gte: str | None = None,
+ end_date_lte: str | None = None,
+ duration_gte: float | None = None,
+ duration_lte: float | None = None,
+ state: list[str] | None = None,
+ pool: list[str] | None = None,
+ queue: list[str] | None = None,
+ limit: int | None = None,
+ offset: int | None = None,
+ order_by: str | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get list of task instances."""
@@ -250,20 +252,20 @@ def get_mapped_task_instances(
)
-def _convert_state(states: Optional[Iterable[str]]) -> Optional[List[Optional[str]]]:
+def _convert_state(states: Iterable[str] | None) -> list[str | None] | None:
if not states:
return None
return [State.NONE if s == "none" else s for s in states]
-def _apply_array_filter(query: Query, key: ClauseElement, values: Optional[Iterable[Any]]) -> Query:
+def _apply_array_filter(query: Query, key: ClauseElement, values: Iterable[Any] | None) -> Query:
if values is not None:
cond = ((key == v) for v in values)
query = query.filter(or_(*cond))
return query
-def _apply_range_filter(query: Query, key: ClauseElement, value_range: Tuple[T, T]) -> Query:
+def _apply_range_filter(query: Query, key: ClauseElement, value_range: tuple[T, T]) -> Query:
gte_value, lte_value = value_range
if gte_value is not None:
query = query.filter(key >= gte_value)
@@ -293,20 +295,20 @@ def _apply_range_filter(query: Query, key: ClauseElement, value_range: Tuple[T,
def get_task_instances(
*,
limit: int,
- dag_id: Optional[str] = None,
- dag_run_id: Optional[str] = None,
- execution_date_gte: Optional[str] = None,
- execution_date_lte: Optional[str] = None,
- start_date_gte: Optional[str] = None,
- start_date_lte: Optional[str] = None,
- end_date_gte: Optional[str] = None,
- end_date_lte: Optional[str] = None,
- duration_gte: Optional[float] = None,
- duration_lte: Optional[float] = None,
- state: Optional[List[str]] = None,
- pool: Optional[List[str]] = None,
- queue: Optional[List[str]] = None,
- offset: Optional[int] = None,
+ dag_id: str | None = None,
+ dag_run_id: str | None = None,
+ execution_date_gte: str | None = None,
+ execution_date_lte: str | None = None,
+ start_date_gte: str | None = None,
+ start_date_lte: str | None = None,
+ end_date_gte: str | None = None,
+ end_date_lte: str | None = None,
+ duration_gte: float | None = None,
+ duration_lte: float | None = None,
+ state: list[str] | None = None,
+ pool: list[str] | None = None,
+ queue: list[str] | None = None,
+ offset: int | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get list of task instances."""
@@ -443,7 +445,7 @@ def post_clear_task_instances(*, dag_id: str, session: Session = NEW_SESSION) ->
downstream = data.pop('include_downstream', False)
upstream = data.pop('include_upstream', False)
if dag_run_id is not None:
- dag_run: Optional[DR] = (
+ dag_run: DR | None = (
session.query(DR).filter(DR.dag_id == dag_id, DR.run_id == dag_run_id).one_or_none()
)
if dag_run is None:
diff --git a/airflow/api_connexion/endpoints/user_endpoint.py b/airflow/api_connexion/endpoints/user_endpoint.py
index 3ab476e219..a47790789c 100644
--- a/airflow/api_connexion/endpoints/user_endpoint.py
+++ b/airflow/api_connexion/endpoints/user_endpoint.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from http import HTTPStatus
-from typing import List, Optional
from connexion import NoContent
from flask import request
@@ -50,7 +51,7 @@ def get_user(*, username: str) -> APIResponse:
@security.requires_access([(permissions.ACTION_CAN_READ, permissions.RESOURCE_USER)])
@format_parameters({"limit": check_limit})
-def get_users(*, limit: int, order_by: str = "id", offset: Optional[str] = None) -> APIResponse:
+def get_users(*, limit: int, order_by: str = "id", offset: str | None = None) -> APIResponse:
"""Get users"""
appbuilder = get_airflow_app().appbuilder
session = appbuilder.get_session
@@ -164,7 +165,7 @@ def patch_user(*, username: str, update_mask: UpdateMask = None) -> APIResponse:
raise BadRequest(detail=detail)
data = masked_data
- roles_to_update: Optional[List[Role]]
+ roles_to_update: list[Role] | None
if "roles" in data:
roles_to_update = []
missing_role_names = []
diff --git a/airflow/api_connexion/endpoints/variable_endpoint.py b/airflow/api_connexion/endpoints/variable_endpoint.py
index ba9584a688..25f8969d46 100644
--- a/airflow/api_connexion/endpoints/variable_endpoint.py
+++ b/airflow/api_connexion/endpoints/variable_endpoint.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from http import HTTPStatus
-from typing import Optional
from flask import Response
from marshmallow import ValidationError
@@ -56,9 +57,9 @@ def get_variable(*, variable_key: str, session: Session = NEW_SESSION) -> Respon
@provide_session
def get_variables(
*,
- limit: Optional[int],
+ limit: int | None,
order_by: str = "id",
- offset: Optional[int] = None,
+ offset: int | None = None,
session: Session = NEW_SESSION,
) -> Response:
"""Get all variable values"""
diff --git a/airflow/api_connexion/endpoints/version_endpoint.py b/airflow/api_connexion/endpoints/version_endpoint.py
index 077d7f8a1c..15edd15670 100644
--- a/airflow/api_connexion/endpoints/version_endpoint.py
+++ b/airflow/api_connexion/endpoints/version_endpoint.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import NamedTuple, Optional
+from typing import NamedTuple
import airflow
from airflow.api_connexion.schemas.version_schema import version_info_schema
@@ -27,7 +28,7 @@ class VersionInfo(NamedTuple):
"""Version information"""
version: str
- git_version: Optional[str]
+ git_version: str | None
def get_version() -> APIResponse:
diff --git a/airflow/api_connexion/endpoints/xcom_endpoint.py b/airflow/api_connexion/endpoints/xcom_endpoint.py
index 6114d4d98b..e346a1b12d 100644
--- a/airflow/api_connexion/endpoints/xcom_endpoint.py
+++ b/airflow/api_connexion/endpoints/xcom_endpoint.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import copy
-from typing import Optional
from flask import g
from sqlalchemy import and_
@@ -47,8 +48,8 @@ def get_xcom_entries(
dag_id: str,
dag_run_id: str,
task_id: str,
- limit: Optional[int],
- offset: Optional[int] = None,
+ limit: int | None,
+ offset: int | None = None,
session: Session = NEW_SESSION,
) -> APIResponse:
"""Get all XCom values"""
diff --git a/airflow/api_connexion/exceptions.py b/airflow/api_connexion/exceptions.py
index 8fb7f2e788..5e96c3c0f1 100644
--- a/airflow/api_connexion/exceptions.py
+++ b/airflow/api_connexion/exceptions.py
@@ -14,8 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from http import HTTPStatus
-from typing import Any, Dict, Optional
+from typing import Any
import flask
import werkzeug
@@ -76,8 +78,8 @@ class NotFound(ProblemException):
def __init__(
self,
title: str = 'Not Found',
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
) -> None:
super().__init__(
@@ -96,8 +98,8 @@ class BadRequest(ProblemException):
def __init__(
self,
title: str = "Bad Request",
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
) -> None:
super().__init__(
@@ -116,8 +118,8 @@ class Unauthenticated(ProblemException):
def __init__(
self,
title: str = "Unauthorized",
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
):
super().__init__(
@@ -136,8 +138,8 @@ class PermissionDenied(ProblemException):
def __init__(
self,
title: str = "Forbidden",
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
) -> None:
super().__init__(
@@ -156,8 +158,8 @@ class AlreadyExists(ProblemException):
def __init__(
self,
title="Conflict",
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
):
super().__init__(
@@ -176,8 +178,8 @@ class Unknown(ProblemException):
def __init__(
self,
title: str = "Internal Server Error",
- detail: Optional[str] = None,
- headers: Optional[Dict] = None,
+ detail: str | None = None,
+ headers: dict | None = None,
**kwargs: Any,
) -> None:
super().__init__(
diff --git a/airflow/api_connexion/parameters.py b/airflow/api_connexion/parameters.py
index 81d1bd9280..cc809ebba4 100644
--- a/airflow/api_connexion/parameters.py
+++ b/airflow/api_connexion/parameters.py
@@ -14,9 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from datetime import datetime
from functools import wraps
-from typing import Any, Callable, Container, Dict, Optional, TypeVar, cast
+from typing import Any, Callable, Container, TypeVar, cast
from pendulum.parsing import ParserError
from sqlalchemy import text
@@ -69,7 +71,7 @@ def check_limit(value: int) -> int:
T = TypeVar("T", bound=Callable)
-def format_parameters(params_formatters: Dict[str, Callable[[Any], Any]]) -> Callable[[T], T]:
+def format_parameters(params_formatters: dict[str, Callable[[Any], Any]]) -> Callable[[T], T]:
"""
Decorator factory that create decorator that convert parameters using given formatters.
@@ -94,8 +96,8 @@ def format_parameters(params_formatters: Dict[str, Callable[[Any], Any]]) -> Cal
def apply_sorting(
query: Query,
order_by: str,
- to_replace: Optional[Dict[str, str]] = None,
- allowed_attrs: Optional[Container[str]] = None,
+ to_replace: dict[str, str] | None = None,
+ allowed_attrs: Container[str] | None = None,
) -> Query:
"""Apply sorting to query"""
lstriped_orderby = order_by.lstrip('-')
diff --git a/airflow/api_connexion/schemas/common_schema.py b/airflow/api_connexion/schemas/common_schema.py
index 502d5b60bd..bd6b5ccb74 100644
--- a/airflow/api_connexion/schemas/common_schema.py
+++ b/airflow/api_connexion/schemas/common_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import datetime
import inspect
diff --git a/airflow/api_connexion/schemas/config_schema.py b/airflow/api_connexion/schemas/config_schema.py
index 2eb459ce14..938b5cd784 100644
--- a/airflow/api_connexion/schemas/config_schema.py
+++ b/airflow/api_connexion/schemas/config_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
@@ -45,7 +46,7 @@ class ConfigSection(NamedTuple):
"""List of config options within a section"""
name: str
- options: List[ConfigOption]
+ options: list[ConfigOption]
class ConfigSchema(Schema):
@@ -57,7 +58,7 @@ class ConfigSchema(Schema):
class Config(NamedTuple):
"""List of config sections with their options"""
- sections: List[ConfigSection]
+ sections: list[ConfigSection]
config_schema = ConfigSchema()
diff --git a/airflow/api_connexion/schemas/connection_schema.py b/airflow/api_connexion/schemas/connection_schema.py
index f06da92bac..f1334e941b 100644
--- a/airflow/api_connexion/schemas/connection_schema.py
+++ b/airflow/api_connexion/schemas/connection_schema.py
@@ -15,8 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import json
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -68,7 +70,7 @@ class ConnectionSchema(ConnectionCollectionItemSchema):
class ConnectionCollection(NamedTuple):
"""List of Connections with meta"""
- connections: List[Connection]
+ connections: list[Connection]
total_entries: int
diff --git a/airflow/api_connexion/schemas/dag_run_schema.py b/airflow/api_connexion/schemas/dag_run_schema.py
index 5cd79b2022..6744df3d2f 100644
--- a/airflow/api_connexion/schemas/dag_run_schema.py
+++ b/airflow/api_connexion/schemas/dag_run_schema.py
@@ -15,8 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import json
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import fields, post_dump, pre_load, validate
from marshmallow.schema import Schema
@@ -128,7 +130,7 @@ class ClearDagRunStateFormSchema(Schema):
class DAGRunCollection(NamedTuple):
"""List of DAGRuns with metadata"""
- dag_runs: List[DagRun]
+ dag_runs: list[DagRun]
total_entries: int
diff --git a/airflow/api_connexion/schemas/dag_schema.py b/airflow/api_connexion/schemas/dag_schema.py
index 6e7410dc4f..9c3de25c53 100644
--- a/airflow/api_connexion/schemas/dag_schema.py
+++ b/airflow/api_connexion/schemas/dag_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from itsdangerous import URLSafeSerializer
from marshmallow import Schema, fields
@@ -149,7 +150,7 @@ class DAGDetailSchema(DAGSchema):
class DAGCollection(NamedTuple):
"""List of DAGs with metadata"""
- dags: List[DagModel]
+ dags: list[DagModel]
total_entries: int
diff --git a/airflow/api_connexion/schemas/dag_source_schema.py b/airflow/api_connexion/schemas/dag_source_schema.py
index d142454bc1..1520d58296 100644
--- a/airflow/api_connexion/schemas/dag_source_schema.py
+++ b/airflow/api_connexion/schemas/dag_source_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow import Schema, fields
diff --git a/airflow/api_connexion/schemas/dag_warning_schema.py b/airflow/api_connexion/schemas/dag_warning_schema.py
index 9531eb6b36..211f251e7d 100644
--- a/airflow/api_connexion/schemas/dag_warning_schema.py
+++ b/airflow/api_connexion/schemas/dag_warning_schema.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, NamedTuple
+from __future__ import annotations
+
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -39,7 +41,7 @@ class DagWarningSchema(SQLAlchemySchema):
class DagWarningCollection(NamedTuple):
"""List of dag warnings with metadata"""
- dag_warnings: List[DagWarning]
+ dag_warnings: list[DagWarning]
total_entries: int
diff --git a/airflow/api_connexion/schemas/dataset_schema.py b/airflow/api_connexion/schemas/dataset_schema.py
index 56118462b6..2610f58248 100644
--- a/airflow/api_connexion/schemas/dataset_schema.py
+++ b/airflow/api_connexion/schemas/dataset_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -77,7 +78,7 @@ class DatasetSchema(SQLAlchemySchema):
class DatasetCollection(NamedTuple):
"""List of Datasets with meta"""
- datasets: List[DatasetModel]
+ datasets: list[DatasetModel]
total_entries: int
@@ -134,7 +135,7 @@ class DatasetEventSchema(SQLAlchemySchema):
class DatasetEventCollection(NamedTuple):
"""List of Dataset events with meta"""
- dataset_events: List[DatasetEvent]
+ dataset_events: list[DatasetEvent]
total_entries: int
diff --git a/airflow/api_connexion/schemas/enum_schemas.py b/airflow/api_connexion/schemas/enum_schemas.py
index 71faf9fa20..63c66000a7 100644
--- a/airflow/api_connexion/schemas/enum_schemas.py
+++ b/airflow/api_connexion/schemas/enum_schemas.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow import fields, validate
diff --git a/airflow/api_connexion/schemas/error_schema.py b/airflow/api_connexion/schemas/error_schema.py
index c9462b5f96..ebc6f6c85b 100644
--- a/airflow/api_connexion/schemas/error_schema.py
+++ b/airflow/api_connexion/schemas/error_schema.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, NamedTuple
+from __future__ import annotations
+
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -41,7 +43,7 @@ class ImportErrorSchema(SQLAlchemySchema):
class ImportErrorCollection(NamedTuple):
"""List of import errors with metadata"""
- import_errors: List[ImportError]
+ import_errors: list[ImportError]
total_entries: int
diff --git a/airflow/api_connexion/schemas/event_log_schema.py b/airflow/api_connexion/schemas/event_log_schema.py
index d97c223bff..8d3314f916 100644
--- a/airflow/api_connexion/schemas/event_log_schema.py
+++ b/airflow/api_connexion/schemas/event_log_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -44,7 +45,7 @@ class EventLogSchema(SQLAlchemySchema):
class EventLogCollection(NamedTuple):
"""List of import errors with metadata"""
- event_logs: List[Log]
+ event_logs: list[Log]
total_entries: int
diff --git a/airflow/api_connexion/schemas/health_schema.py b/airflow/api_connexion/schemas/health_schema.py
index 7089babb62..9f4d773cc0 100644
--- a/airflow/api_connexion/schemas/health_schema.py
+++ b/airflow/api_connexion/schemas/health_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow import Schema, fields
diff --git a/airflow/api_connexion/schemas/log_schema.py b/airflow/api_connexion/schemas/log_schema.py
index eff97e1723..5ba649f4d6 100644
--- a/airflow/api_connexion/schemas/log_schema.py
+++ b/airflow/api_connexion/schemas/log_schema.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import NamedTuple, Optional
+from __future__ import annotations
+
+from typing import NamedTuple
from marshmallow import Schema, fields
@@ -30,7 +32,7 @@ class LogResponseObject(NamedTuple):
"""Log Response Object"""
content: str
- continuation_token: Optional[str]
+ continuation_token: str | None
logs_schema = LogsSchema()
diff --git a/airflow/api_connexion/schemas/plugin_schema.py b/airflow/api_connexion/schemas/plugin_schema.py
index 9fb75c7d7c..88c546012f 100644
--- a/airflow/api_connexion/schemas/plugin_schema.py
+++ b/airflow/api_connexion/schemas/plugin_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
@@ -38,7 +39,7 @@ class PluginSchema(Schema):
class PluginCollection(NamedTuple):
"""Plugin List"""
- plugins: List
+ plugins: list
total_entries: int
diff --git a/airflow/api_connexion/schemas/pool_schema.py b/airflow/api_connexion/schemas/pool_schema.py
index c942485082..8e2f49cc36 100644
--- a/airflow/api_connexion/schemas/pool_schema.py
+++ b/airflow/api_connexion/schemas/pool_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -69,7 +70,7 @@ class PoolSchema(SQLAlchemySchema):
class PoolCollection(NamedTuple):
"""List of Pools with metadata"""
- pools: List[Pool]
+ pools: list[Pool]
total_entries: int
diff --git a/airflow/api_connexion/schemas/provider_schema.py b/airflow/api_connexion/schemas/provider_schema.py
index 4c9867380b..ad62f4ae26 100644
--- a/airflow/api_connexion/schemas/provider_schema.py
+++ b/airflow/api_connexion/schemas/provider_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
@@ -41,7 +42,7 @@ class Provider(TypedDict):
class ProviderCollection(NamedTuple):
"""List of Providers."""
- providers: List[Provider]
+ providers: list[Provider]
total_entries: int
diff --git a/airflow/api_connexion/schemas/role_and_permission_schema.py b/airflow/api_connexion/schemas/role_and_permission_schema.py
index 4031750199..f67ce6119d 100644
--- a/airflow/api_connexion/schemas/role_and_permission_schema.py
+++ b/airflow/api_connexion/schemas/role_and_permission_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -48,7 +49,7 @@ class ResourceSchema(SQLAlchemySchema):
class ActionCollection(NamedTuple):
"""Action Action Collection"""
- actions: List[Action]
+ actions: list[Action]
total_entries: int
@@ -86,7 +87,7 @@ class RoleSchema(SQLAlchemySchema):
class RoleCollection(NamedTuple):
"""List of roles"""
- roles: List[Role]
+ roles: list[Role]
total_entries: int
diff --git a/airflow/api_connexion/schemas/sla_miss_schema.py b/airflow/api_connexion/schemas/sla_miss_schema.py
index 9413e37cbd..e5ddc6df31 100644
--- a/airflow/api_connexion/schemas/sla_miss_schema.py
+++ b/airflow/api_connexion/schemas/sla_miss_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
diff --git a/airflow/api_connexion/schemas/task_instance_schema.py b/airflow/api_connexion/schemas/task_instance_schema.py
index 26851b02f6..bbe09dae29 100644
--- a/airflow/api_connexion/schemas/task_instance_schema.py
+++ b/airflow/api_connexion/schemas/task_instance_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple, Optional, Tuple
+from typing import NamedTuple
from marshmallow import Schema, ValidationError, fields, validate, validates_schema
from marshmallow.utils import get_value
@@ -77,7 +78,7 @@ class TaskInstanceSchema(SQLAlchemySchema):
class TaskInstanceCollection(NamedTuple):
"""List of task instances with metadata"""
- task_instances: List[Tuple[TaskInstance, Optional[SlaMiss]]]
+ task_instances: list[tuple[TaskInstance, SlaMiss | None]]
total_entries: int
@@ -173,7 +174,7 @@ class TaskInstanceReferenceSchema(Schema):
class TaskInstanceReferenceCollection(NamedTuple):
"""List of objects with metadata about taskinstance and dag_run_id"""
- task_instances: List[Tuple[TaskInstance, str]]
+ task_instances: list[tuple[TaskInstance, str]]
class TaskInstanceReferenceCollectionSchema(Schema):
diff --git a/airflow/api_connexion/schemas/task_schema.py b/airflow/api_connexion/schemas/task_schema.py
index 4815dffb72..89c1fa68c5 100644
--- a/airflow/api_connexion/schemas/task_schema.py
+++ b/airflow/api_connexion/schemas/task_schema.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import List, NamedTuple
+from typing import NamedTuple
from marshmallow import Schema, fields
@@ -78,7 +79,7 @@ class TaskSchema(Schema):
class TaskCollection(NamedTuple):
"""List of Tasks with metadata"""
- tasks: List[Operator]
+ tasks: list[Operator]
total_entries: int
diff --git a/airflow/api_connexion/schemas/user_schema.py b/airflow/api_connexion/schemas/user_schema.py
index 3d36aa91c8..fad16b6701 100644
--- a/airflow/api_connexion/schemas/user_schema.py
+++ b/airflow/api_connexion/schemas/user_schema.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, NamedTuple
+from __future__ import annotations
+
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -55,7 +57,7 @@ class UserSchema(UserCollectionItemSchema):
class UserCollection(NamedTuple):
"""User collection"""
- users: List[User]
+ users: list[User]
total_entries: int
diff --git a/airflow/api_connexion/schemas/variable_schema.py b/airflow/api_connexion/schemas/variable_schema.py
index 00284ce9f2..8c2b97d06b 100644
--- a/airflow/api_connexion/schemas/variable_schema.py
+++ b/airflow/api_connexion/schemas/variable_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow import Schema, fields
diff --git a/airflow/api_connexion/schemas/version_schema.py b/airflow/api_connexion/schemas/version_schema.py
index 24bd9337c1..91f75815cd 100644
--- a/airflow/api_connexion/schemas/version_schema.py
+++ b/airflow/api_connexion/schemas/version_schema.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from marshmallow import Schema, fields
diff --git a/airflow/api_connexion/schemas/xcom_schema.py b/airflow/api_connexion/schemas/xcom_schema.py
index b3f3f0dd02..d730f40554 100644
--- a/airflow/api_connexion/schemas/xcom_schema.py
+++ b/airflow/api_connexion/schemas/xcom_schema.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import List, NamedTuple
+from __future__ import annotations
+
+from typing import NamedTuple
from marshmallow import Schema, fields
from marshmallow_sqlalchemy import SQLAlchemySchema, auto_field
@@ -46,7 +48,7 @@ class XComSchema(XComCollectionItemSchema):
class XComCollection(NamedTuple):
"""List of XComs with meta"""
- xcom_entries: List[XCom]
+ xcom_entries: list[XCom]
total_entries: int
diff --git a/airflow/api_connexion/security.py b/airflow/api_connexion/security.py
index 6c84181f91..6076f17f9c 100644
--- a/airflow/api_connexion/security.py
+++ b/airflow/api_connexion/security.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from functools import wraps
-from typing import Callable, Optional, Sequence, Tuple, TypeVar, cast
+from typing import Callable, Sequence, TypeVar, cast
from flask import Response
@@ -37,7 +38,7 @@ def check_authentication() -> None:
raise Unauthenticated(headers=response.headers)
-def requires_access(permissions: Optional[Sequence[Tuple[str, str]]] = None) -> Callable[[T], T]:
+def requires_access(permissions: Sequence[tuple[str, str]] | None = None) -> Callable[[T], T]:
"""Factory for decorator that checks current user's permissions against required permissions."""
appbuilder = get_airflow_app().appbuilder
appbuilder.sm.sync_resource_permissions(permissions)
diff --git a/airflow/api_connexion/types.py b/airflow/api_connexion/types.py
index f640d14bd7..3a6f89d9bb 100644
--- a/airflow/api_connexion/types.py
+++ b/airflow/api_connexion/types.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from typing import Any, Mapping, Optional, Sequence, Tuple, Union
diff --git a/airflow/callbacks/base_callback_sink.py b/airflow/callbacks/base_callback_sink.py
index e7cbf23e7b..c243f0fbd6 100644
--- a/airflow/callbacks/base_callback_sink.py
+++ b/airflow/callbacks/base_callback_sink.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
from airflow.callbacks.callback_requests import CallbackRequest
diff --git a/airflow/callbacks/callback_requests.py b/airflow/callbacks/callback_requests.py
index 3e274368fb..d8c36cb753 100644
--- a/airflow/callbacks/callback_requests.py
+++ b/airflow/callbacks/callback_requests.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import json
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
if TYPE_CHECKING:
from airflow.models.taskinstance import SimpleTaskInstance
@@ -34,8 +35,8 @@ class CallbackRequest:
def __init__(
self,
full_filepath: str,
- processor_subdir: Optional[str] = None,
- msg: Optional[str] = None,
+ processor_subdir: str | None = None,
+ msg: str | None = None,
):
self.full_filepath = full_filepath
self.processor_subdir = processor_subdir
@@ -73,10 +74,10 @@ class TaskCallbackRequest(CallbackRequest):
def __init__(
self,
full_filepath: str,
- simple_task_instance: "SimpleTaskInstance",
- is_failure_callback: Optional[bool] = True,
- processor_subdir: Optional[str] = None,
- msg: Optional[str] = None,
+ simple_task_instance: SimpleTaskInstance,
+ is_failure_callback: bool | None = True,
+ processor_subdir: str | None = None,
+ msg: str | None = None,
):
super().__init__(full_filepath=full_filepath, processor_subdir=processor_subdir, msg=msg)
self.simple_task_instance = simple_task_instance
@@ -113,9 +114,9 @@ class DagCallbackRequest(CallbackRequest):
full_filepath: str,
dag_id: str,
run_id: str,
- processor_subdir: Optional[str],
- is_failure_callback: Optional[bool] = True,
- msg: Optional[str] = None,
+ processor_subdir: str | None,
+ is_failure_callback: bool | None = True,
+ msg: str | None = None,
):
super().__init__(full_filepath=full_filepath, processor_subdir=processor_subdir, msg=msg)
self.dag_id = dag_id
@@ -136,8 +137,8 @@ class SlaCallbackRequest(CallbackRequest):
self,
full_filepath: str,
dag_id: str,
- processor_subdir: Optional[str],
- msg: Optional[str] = None,
+ processor_subdir: str | None,
+ msg: str | None = None,
):
super().__init__(full_filepath, processor_subdir=processor_subdir, msg=msg)
self.dag_id = dag_id
diff --git a/airflow/callbacks/database_callback_sink.py b/airflow/callbacks/database_callback_sink.py
index 0a7ba21c78..24306170df 100644
--- a/airflow/callbacks/database_callback_sink.py
+++ b/airflow/callbacks/database_callback_sink.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
from sqlalchemy.orm import Session
diff --git a/airflow/callbacks/pipe_callback_sink.py b/airflow/callbacks/pipe_callback_sink.py
index 1e11ffd4f5..d702a781fa 100644
--- a/airflow/callbacks/pipe_callback_sink.py
+++ b/airflow/callbacks/pipe_callback_sink.py
@@ -15,7 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
+
from multiprocessing.connection import Connection as MultiprocessingConnection
from typing import Callable
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
index 21031bd1c2..6581cfe632 100644
--- a/airflow/cli/cli_parser.py
+++ b/airflow/cli/cli_parser.py
@@ -17,6 +17,7 @@
# specific language governing permissions and limitations
# under the License.
"""Command-line interface"""
+from __future__ import annotations
import argparse
import json
@@ -24,7 +25,7 @@ import os
import textwrap
from argparse import Action, ArgumentError, RawTextHelpFormatter
from functools import lru_cache
-from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Union
+from typing import Callable, Iterable, NamedTuple, Union
import lazy_object_proxy
@@ -960,8 +961,8 @@ class ActionCommand(NamedTuple):
help: str
func: Callable
args: Iterable[Arg]
- description: Optional[str] = None
- epilog: Optional[str] = None
+ description: str | None = None
+ epilog: str | None = None
class GroupCommand(NamedTuple):
@@ -970,8 +971,8 @@ class GroupCommand(NamedTuple):
name: str
help: str
subcommands: Iterable
- description: Optional[str] = None
- epilog: Optional[str] = None
+ description: str | None = None
+ epilog: str | None = None
CLICommand = Union[ActionCommand, GroupCommand]
@@ -1801,7 +1802,7 @@ JOBS_COMMANDS = (
),
)
-airflow_commands: List[CLICommand] = [
+airflow_commands: list[CLICommand] = [
GroupCommand(
name='dags',
help='Manage DAGs',
@@ -1999,7 +2000,7 @@ airflow_commands: List[CLICommand] = [
args=tuple(),
),
]
-ALL_COMMANDS_DICT: Dict[str, CLICommand] = {sp.name: sp for sp in airflow_commands}
+ALL_COMMANDS_DICT: dict[str, CLICommand] = {sp.name: sp for sp in airflow_commands}
def _remove_dag_id_opt(command: ActionCommand):
@@ -2008,7 +2009,7 @@ def _remove_dag_id_opt(command: ActionCommand):
return ActionCommand(**cmd)
-dag_cli_commands: List[CLICommand] = [
+dag_cli_commands: list[CLICommand] = [
GroupCommand(
name='dags',
help='Manage DAGs',
@@ -2024,7 +2025,7 @@ dag_cli_commands: List[CLICommand] = [
subcommands=[_remove_dag_id_opt(sp) for sp in TASKS_COMMANDS if sp.name in ['list', 'test', 'run']],
),
]
-DAG_CLI_DICT: Dict[str, CLICommand] = {sp.name: sp for sp in dag_cli_commands}
+DAG_CLI_DICT: dict[str, CLICommand] = {sp.name: sp for sp in dag_cli_commands}
class AirflowHelpFormatter(argparse.HelpFormatter):
diff --git a/airflow/cli/commands/celery_command.py b/airflow/cli/commands/celery_command.py
index c2deb16ff4..4515d8cf95 100644
--- a/airflow/cli/commands/celery_command.py
+++ b/airflow/cli/commands/celery_command.py
@@ -16,9 +16,9 @@
# specific language governing permissions and limitations
# under the License.
"""Celery command"""
+from __future__ import annotations
from multiprocessing import Process
-from typing import Optional
import daemon
import psutil
@@ -83,7 +83,7 @@ def flower(args):
celery_app.start(options)
-def _serve_logs(skip_serve_logs: bool = False) -> Optional[Process]:
+def _serve_logs(skip_serve_logs: bool = False) -> Process | None:
"""Starts serve_logs sub-process"""
if skip_serve_logs is False:
sub_proc = Process(target=serve_logs)
diff --git a/airflow/cli/commands/cheat_sheet_command.py b/airflow/cli/commands/cheat_sheet_command.py
index 001a872133..88d9c5940c 100644
--- a/airflow/cli/commands/cheat_sheet_command.py
+++ b/airflow/cli/commands/cheat_sheet_command.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Iterable, List, Optional, Union
+from __future__ import annotations
+
+from typing import Iterable
from airflow.cli.cli_parser import ActionCommand, GroupCommand, airflow_commands
from airflow.cli.simple_table import AirflowConsole, SimpleTable
@@ -31,12 +33,12 @@ def display_commands_index():
"""Display list of all commands."""
def display_recursive(
- prefix: List[str],
- commands: Iterable[Union[GroupCommand, ActionCommand]],
- help_msg: Optional[str] = None,
+ prefix: list[str],
+ commands: Iterable[GroupCommand | ActionCommand],
+ help_msg: str | None = None,
):
- actions: List[ActionCommand] = []
- groups: List[GroupCommand] = []
+ actions: list[ActionCommand] = []
+ groups: list[GroupCommand] = []
for command in commands:
if isinstance(command, GroupCommand):
groups.append(command)
diff --git a/airflow/cli/commands/config_command.py b/airflow/cli/commands/config_command.py
index 1c2674fc81..70c194e4af 100644
--- a/airflow/cli/commands/config_command.py
+++ b/airflow/cli/commands/config_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Config sub-commands"""
+from __future__ import annotations
+
import io
import pygments
diff --git a/airflow/cli/commands/connection_command.py b/airflow/cli/commands/connection_command.py
index 8a0c0a3acb..3d5bf74113 100644
--- a/airflow/cli/commands/connection_command.py
+++ b/airflow/cli/commands/connection_command.py
@@ -15,13 +15,15 @@
# specific language governing permissions and limitations
# under the License.
"""Connection sub-commands"""
+from __future__ import annotations
+
import io
import json
import os
import sys
import warnings
from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any
from urllib.parse import urlparse, urlunparse
from sqlalchemy.orm import exc
@@ -38,7 +40,7 @@ from airflow.utils.cli import suppress_logs_and_warning
from airflow.utils.session import create_session
-def _connection_mapper(conn: Connection) -> Dict[str, Any]:
+def _connection_mapper(conn: Connection) -> dict[str, Any]:
return {
'id': conn.id,
'conn_id': conn.conn_id,
@@ -99,7 +101,7 @@ def _connection_to_dict(conn: Connection) -> dict:
)
-def _format_connections(conns: List[Connection], file_format: str, serialization_format: str) -> str:
+def _format_connections(conns: list[Connection], file_format: str, serialization_format: str) -> str:
if serialization_format == 'json':
serializer_func = lambda x: json.dumps(_connection_to_dict(x))
elif serialization_format == 'uri':
diff --git a/airflow/cli/commands/dag_command.py b/airflow/cli/commands/dag_command.py
index 29c4a4f3ff..dac0e7b0fc 100644
--- a/airflow/cli/commands/dag_command.py
+++ b/airflow/cli/commands/dag_command.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Dag sub-commands"""
+from __future__ import annotations
+
import ast
import errno
import json
@@ -23,7 +24,6 @@ import logging
import signal
import subprocess
import sys
-from typing import Optional
from graphviz.dot import Dot
from sqlalchemy.orm import Session
@@ -295,7 +295,7 @@ def dag_next_execution(args):
.filter(DagRun.dag_id == dag.dag_id)
.subquery()
)
- max_date_run: Optional[DagRun] = (
+ max_date_run: DagRun | None = (
session.query(DagRun)
.filter(DagRun.dag_id == dag.dag_id, DagRun.execution_date == max_date_subq.c.max_date)
.one_or_none()
diff --git a/airflow/cli/commands/dag_processor_command.py b/airflow/cli/commands/dag_processor_command.py
index 92f7f37ee1..e4493ea536 100644
--- a/airflow/cli/commands/dag_processor_command.py
+++ b/airflow/cli/commands/dag_processor_command.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""DagProcessor command"""
+from __future__ import annotations
+
import logging
from datetime import timedelta
diff --git a/airflow/cli/commands/db_command.py b/airflow/cli/commands/db_command.py
index 5f6a84c8a4..6d981a3520 100644
--- a/airflow/cli/commands/db_command.py
+++ b/airflow/cli/commands/db_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Database sub-commands"""
+from __future__ import annotations
+
import os
import textwrap
from tempfile import NamedTemporaryFile
diff --git a/airflow/cli/commands/info_command.py b/airflow/cli/commands/info_command.py
index fc03615210..f49af3e7dd 100644
--- a/airflow/cli/commands/info_command.py
+++ b/airflow/cli/commands/info_command.py
@@ -15,13 +15,14 @@
# specific language governing permissions and limitations
# under the License.
"""Config sub-commands"""
+from __future__ import annotations
+
import locale
import logging
import os
import platform
import subprocess
import sys
-from typing import List, Optional
from urllib.parse import urlsplit, urlunsplit
import httpx
@@ -132,7 +133,7 @@ class OperatingSystem:
CYGWIN = "Cygwin"
@staticmethod
- def get_current() -> Optional[str]:
+ def get_current() -> str | None:
"""Get current operating system"""
if os.name == "nt":
return OperatingSystem.WINDOWS
@@ -185,7 +186,7 @@ class AirflowInfo:
self.anonymizer = anonymizer
@staticmethod
- def _get_version(cmd: List[str], grep: Optional[bytes] = None):
+ def _get_version(cmd: list[str], grep: bytes | None = None):
"""Return tools version."""
try:
with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc:
@@ -306,7 +307,7 @@ class AirflowInfo:
def _providers_info(self):
return [(p.data['package-name'], p.version) for p in ProvidersManager().providers.values()]
- def show(self, output: str, console: Optional[AirflowConsole] = None) -> None:
+ def show(self, output: str, console: AirflowConsole | None = None) -> None:
"""Shows information about Airflow instance"""
all_info = {
"Apache Airflow": self._airflow_info,
diff --git a/airflow/cli/commands/jobs_command.py b/airflow/cli/commands/jobs_command.py
index f6d8d55fb6..76ef0390b9 100644
--- a/airflow/cli/commands/jobs_command.py
+++ b/airflow/cli/commands/jobs_command.py
@@ -14,8 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-from typing import List
+from __future__ import annotations
from airflow.jobs.base_job import BaseJob
from airflow.utils.session import provide_session
@@ -39,7 +38,7 @@ def check(args, session=None):
if args.limit > 0:
query = query.limit(args.limit)
- jobs: List[BaseJob] = query.all()
+ jobs: list[BaseJob] = query.all()
alive_jobs = [job for job in jobs if job.is_alive()]
count_alive_jobs = len(alive_jobs)
diff --git a/airflow/cli/commands/kerberos_command.py b/airflow/cli/commands/kerberos_command.py
index 4ccd07092f..eec77f6d6f 100644
--- a/airflow/cli/commands/kerberos_command.py
+++ b/airflow/cli/commands/kerberos_command.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Kerberos command"""
+from __future__ import annotations
+
import daemon
from daemon.pidfile import TimeoutPIDLockFile
diff --git a/airflow/cli/commands/kubernetes_command.py b/airflow/cli/commands/kubernetes_command.py
index 7c26821780..76523b3cb3 100644
--- a/airflow/cli/commands/kubernetes_command.py
+++ b/airflow/cli/commands/kubernetes_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Kubernetes sub-commands"""
+from __future__ import annotations
+
import os
import sys
from datetime import datetime, timedelta
diff --git a/airflow/cli/commands/legacy_commands.py b/airflow/cli/commands/legacy_commands.py
index 94f9b69032..5643879dc5 100644
--- a/airflow/cli/commands/legacy_commands.py
+++ b/airflow/cli/commands/legacy_commands.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from argparse import ArgumentError
diff --git a/airflow/cli/commands/plugins_command.py b/airflow/cli/commands/plugins_command.py
index 2d59e901a1..2c9f1488c4 100644
--- a/airflow/cli/commands/plugins_command.py
+++ b/airflow/cli/commands/plugins_command.py
@@ -14,8 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import inspect
-from typing import Any, Dict, List, Union
+from typing import Any
from airflow import plugins_manager
from airflow.cli.simple_table import AirflowConsole
@@ -31,7 +33,7 @@ def _get_name(class_like_object) -> str:
return class_like_object.__class__.__name__
-def _join_plugins_names(value: Union[List[Any], Any]) -> str:
+def _join_plugins_names(value: list[Any] | Any) -> str:
value = value if isinstance(value, list) else [value]
return ",".join(_get_name(v) for v in value)
@@ -39,7 +41,7 @@ def _join_plugins_names(value: Union[List[Any], Any]) -> str:
@suppress_logs_and_warning
def dump_plugins(args):
"""Dump plugins information"""
- plugins_info: List[Dict[str, str]] = get_plugin_info()
+ plugins_info: list[dict[str, str]] = get_plugin_info()
if not plugins_manager.plugins:
print("No plugins loaded")
return
diff --git a/airflow/cli/commands/pool_command.py b/airflow/cli/commands/pool_command.py
index e435c2a483..91e0170775 100644
--- a/airflow/cli/commands/pool_command.py
+++ b/airflow/cli/commands/pool_command.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Pools sub-commands"""
+from __future__ import annotations
+
import json
import os
from json import JSONDecodeError
diff --git a/airflow/cli/commands/provider_command.py b/airflow/cli/commands/provider_command.py
index 124c8cea8a..51a192f61d 100644
--- a/airflow/cli/commands/provider_command.py
+++ b/airflow/cli/commands/provider_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Providers sub-commands"""
+from __future__ import annotations
+
import re
from airflow.cli.simple_table import AirflowConsole
diff --git a/airflow/cli/commands/role_command.py b/airflow/cli/commands/role_command.py
index 51ba2d27d8..ed3e4e30b8 100644
--- a/airflow/cli/commands/role_command.py
+++ b/airflow/cli/commands/role_command.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
"""Roles sub-commands"""
+from __future__ import annotations
+
import json
import os
diff --git a/airflow/cli/commands/rotate_fernet_key_command.py b/airflow/cli/commands/rotate_fernet_key_command.py
index 9334344d96..ca7e956587 100644
--- a/airflow/cli/commands/rotate_fernet_key_command.py
+++ b/airflow/cli/commands/rotate_fernet_key_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Rotate Fernet key command"""
+from __future__ import annotations
+
from airflow.models import Connection, Variable
from airflow.utils import cli as cli_utils
from airflow.utils.session import create_session
diff --git a/airflow/cli/commands/scheduler_command.py b/airflow/cli/commands/scheduler_command.py
index 22a185794f..168018887e 100644
--- a/airflow/cli/commands/scheduler_command.py
+++ b/airflow/cli/commands/scheduler_command.py
@@ -14,11 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Scheduler command"""
+from __future__ import annotations
+
import signal
from multiprocessing import Process
-from typing import Optional
import daemon
from daemon.pidfile import TimeoutPIDLockFile
@@ -85,7 +85,7 @@ def scheduler(args):
_run_scheduler_job(args=args)
-def _serve_logs(skip_serve_logs: bool = False) -> Optional[Process]:
+def _serve_logs(skip_serve_logs: bool = False) -> Process | None:
"""Starts serve_logs sub-process"""
from airflow.configuration import conf
from airflow.utils.serve_logs import serve_logs
@@ -98,7 +98,7 @@ def _serve_logs(skip_serve_logs: bool = False) -> Optional[Process]:
return None
-def _serve_health_check(enable_health_check: bool = False) -> Optional[Process]:
+def _serve_health_check(enable_health_check: bool = False) -> Process | None:
"""Starts serve_health_check sub-process"""
if enable_health_check:
sub_proc = Process(target=serve_health_check)
diff --git a/airflow/cli/commands/standalone_command.py b/airflow/cli/commands/standalone_command.py
index 3860942adb..34eae28c23 100644
--- a/airflow/cli/commands/standalone_command.py
+++ b/airflow/cli/commands/standalone_command.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import logging
import os
@@ -23,7 +24,6 @@ import subprocess
import threading
import time
from collections import deque
-from typing import Dict, List
from termcolor import colored
@@ -265,7 +265,7 @@ class SubCommand(threading.Thread):
complex logic that brings doing line buffering.
"""
- def __init__(self, parent, name: str, command: List[str], env: Dict[str, str]):
+ def __init__(self, parent, name: str, command: list[str], env: dict[str, str]):
super().__init__()
self.parent = parent
self.name = name
diff --git a/airflow/cli/commands/sync_perm_command.py b/airflow/cli/commands/sync_perm_command.py
index d580631fcc..2d3b03133d 100644
--- a/airflow/cli/commands/sync_perm_command.py
+++ b/airflow/cli/commands/sync_perm_command.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Sync permission command"""
+from __future__ import annotations
+
from airflow.utils import cli as cli_utils
from airflow.www.app import cached_app
diff --git a/airflow/cli/commands/task_command.py b/airflow/cli/commands/task_command.py
index f8916f0466..9caa8bb4bd 100644
--- a/airflow/cli/commands/task_command.py
+++ b/airflow/cli/commands/task_command.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Task sub-commands"""
+from __future__ import annotations
+
import datetime
import importlib
import json
@@ -23,7 +25,7 @@ import logging
import os
import textwrap
from contextlib import contextmanager, redirect_stderr, redirect_stdout, suppress
-from typing import Dict, Generator, List, Optional, Tuple, Union
+from typing import Generator, Union
from pendulum.parsing.exceptions import ParserError
from sqlalchemy.orm.exc import NoResultFound
@@ -76,9 +78,9 @@ def _get_dag_run(
*,
dag: DAG,
create_if_necessary: CreateIfNecessary,
- exec_date_or_run_id: Optional[str] = None,
+ exec_date_or_run_id: str | None = None,
session: Session,
-) -> Tuple[DagRun, bool]:
+) -> tuple[DagRun, bool]:
"""Try to retrieve a DAG run from a string representing either a run ID or logical date.
This checks DAG runs like this:
@@ -94,7 +96,7 @@ def _get_dag_run(
"""
if not exec_date_or_run_id and not create_if_necessary:
raise ValueError("Must provide `exec_date_or_run_id` if not `create_if_necessary`.")
- execution_date: Optional[datetime.datetime] = None
+ execution_date: datetime.datetime | None = None
if exec_date_or_run_id:
dag_run = dag.get_dagrun(run_id=exec_date_or_run_id, session=session)
if dag_run:
@@ -140,11 +142,11 @@ def _get_ti(
task: BaseOperator,
map_index: int,
*,
- exec_date_or_run_id: Optional[str] = None,
- pool: Optional[str] = None,
+ exec_date_or_run_id: str | None = None,
+ pool: str | None = None,
create_if_necessary: CreateIfNecessary = False,
session: Session = NEW_SESSION,
-) -> Tuple[TaskInstance, bool]:
+) -> tuple[TaskInstance, bool]:
"""Get the task instance through DagRun.run_id, if that fails, get the TI the old way"""
if not exec_date_or_run_id and not create_if_necessary:
raise ValueError("Must provide `exec_date_or_run_id` if not `create_if_necessary`.")
@@ -267,7 +269,7 @@ def _run_raw_task(args, ti: TaskInstance) -> None:
)
-def _extract_external_executor_id(args) -> Optional[str]:
+def _extract_external_executor_id(args) -> str | None:
if hasattr(args, "external_executor_id"):
return getattr(args, "external_executor_id")
return os.environ.get("external_executor_id", None)
@@ -441,7 +443,7 @@ def task_list(args, dag=None):
print("\n".join(tasks))
-SUPPORTED_DEBUGGER_MODULES: List[str] = [
+SUPPORTED_DEBUGGER_MODULES: list[str] = [
"pudb",
"web_pdb",
"ipdb",
@@ -498,7 +500,7 @@ def task_states_for_dag_run(args, session=None):
has_mapped_instances = any(ti.map_index >= 0 for ti in dag_run.task_instances)
- def format_task_instance(ti: TaskInstance) -> Dict[str, str]:
+ def format_task_instance(ti: TaskInstance) -> dict[str, str]:
data = {
"dag_id": ti.dag_id,
"execution_date": dag_run.execution_date.isoformat(),
diff --git a/airflow/cli/commands/triggerer_command.py b/airflow/cli/commands/triggerer_command.py
index 70fbbda2fc..f364f708c4 100644
--- a/airflow/cli/commands/triggerer_command.py
+++ b/airflow/cli/commands/triggerer_command.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Triggerer command"""
+from __future__ import annotations
+
import signal
import daemon
diff --git a/airflow/cli/commands/user_command.py b/airflow/cli/commands/user_command.py
index ddbb7cfc82..75a6899662 100644
--- a/airflow/cli/commands/user_command.py
+++ b/airflow/cli/commands/user_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""User sub-commands"""
+from __future__ import annotations
+
import functools
import getpass
import json
@@ -22,7 +24,7 @@ import os
import random
import re
import string
-from typing import Any, Dict, List
+from typing import Any
from marshmallow import Schema, fields, validate
from marshmallow.exceptions import ValidationError
@@ -189,7 +191,7 @@ def users_import(args):
print("Updated the following users:\n\t{}".format("\n\t".join(users_updated)))
-def _import_users(users_list: List[Dict[str, Any]]):
+def _import_users(users_list: list[dict[str, Any]]):
appbuilder = cached_app().appbuilder
users_created = []
users_updated = []
diff --git a/airflow/cli/commands/variable_command.py b/airflow/cli/commands/variable_command.py
index 40eb193148..4269723281 100644
--- a/airflow/cli/commands/variable_command.py
+++ b/airflow/cli/commands/variable_command.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Variable subcommands"""
+from __future__ import annotations
+
import json
import os
from json import JSONDecodeError
diff --git a/airflow/cli/commands/version_command.py b/airflow/cli/commands/version_command.py
index 7e51901858..365e9b2316 100644
--- a/airflow/cli/commands/version_command.py
+++ b/airflow/cli/commands/version_command.py
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
"""Version command"""
+from __future__ import annotations
+
import airflow
diff --git a/airflow/cli/commands/webserver_command.py b/airflow/cli/commands/webserver_command.py
index d93d44a2b7..9d15d44b7d 100644
--- a/airflow/cli/commands/webserver_command.py
+++ b/airflow/cli/commands/webserver_command.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Webserver command"""
+from __future__ import annotations
+
import hashlib
import logging
import os
@@ -26,7 +27,7 @@ import textwrap
import time
from contextlib import suppress
from time import sleep
-from typing import Dict, List, NoReturn
+from typing import NoReturn
import daemon
import psutil
@@ -102,7 +103,7 @@ class GunicornMonitor(LoggingMixin):
self._last_plugin_state = self._generate_plugin_state() if reload_on_plugin_change else None
self._restart_on_next_plugin_check = False
- def _generate_plugin_state(self) -> Dict[str, float]:
+ def _generate_plugin_state(self) -> dict[str, float]:
"""
Generate dict of filenames and last modification time of all files in settings.PLUGINS_FOLDER
directory.
@@ -110,7 +111,7 @@ class GunicornMonitor(LoggingMixin):
if not settings.PLUGINS_FOLDER:
return {}
- all_filenames: List[str] = []
+ all_filenames: list[str] = []
for (root, _, filenames) in os.walk(settings.PLUGINS_FOLDER):
all_filenames.extend(os.path.join(root, f) for f in filenames)
plugin_state = {f: self._get_file_hash(f) for f in sorted(all_filenames)}
diff --git a/airflow/cli/simple_table.py b/airflow/cli/simple_table.py
index efd418d21f..7c372ea1f2 100644
--- a/airflow/cli/simple_table.py
+++ b/airflow/cli/simple_table.py
@@ -14,9 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import inspect
import json
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable
from rich.box import ASCII_DOUBLE_HEAD
from rich.console import Console
@@ -40,17 +42,17 @@ class AirflowConsole(Console):
# If show header in tables
self.show_header = show_header
- def print_as_json(self, data: Dict):
+ def print_as_json(self, data: dict):
"""Renders dict as json text representation"""
json_content = json.dumps(data)
self.print(Syntax(json_content, "json", theme="ansi_dark"), soft_wrap=True)
- def print_as_yaml(self, data: Dict):
+ def print_as_yaml(self, data: dict):
"""Renders dict as yaml text representation"""
yaml_content = yaml.dump(data)
self.print(Syntax(yaml_content, "yaml", theme="ansi_dark"), soft_wrap=True)
- def print_as_table(self, data: List[Dict]):
+ def print_as_table(self, data: list[dict]):
"""Renders list of dictionaries as table"""
if not data:
self.print("No data found")
@@ -64,7 +66,7 @@ class AirflowConsole(Console):
table.add_row(*(str(d) for d in row.values()))
self.print(table)
- def print_as_plain_table(self, data: List[Dict]):
+ def print_as_plain_table(self, data: list[dict]):
"""Renders list of dictionaries as a simple table than can be easily piped"""
if not data:
self.print("No data found")
@@ -73,7 +75,7 @@ class AirflowConsole(Console):
output = tabulate(rows, tablefmt="plain", headers=list(data[0].keys()))
print(output)
- def _normalize_data(self, value: Any, output: str) -> Optional[Union[list, str, dict]]:
+ def _normalize_data(self, value: Any, output: str) -> list | str | dict | None:
if isinstance(value, (tuple, list)):
if output == "table":
return ",".join(str(self._normalize_data(x, output)) for x in value)
@@ -86,9 +88,9 @@ class AirflowConsole(Console):
return None
return str(value)
- def print_as(self, data: List[Union[Dict, Any]], output: str, mapper: Optional[Callable] = None):
+ def print_as(self, data: list[dict | Any], output: str, mapper: Callable | None = None):
"""Prints provided using format specified by output argument"""
- output_to_renderer: Dict[str, Callable[[Any], None]] = {
+ output_to_renderer: dict[str, Callable[[Any], None]] = {
"json": self.print_as_json,
"yaml": self.print_as_yaml,
"table": self.print_as_table,
@@ -104,7 +106,7 @@ class AirflowConsole(Console):
raise ValueError("To tabulate non-dictionary data you need to provide `mapper` function")
if mapper:
- dict_data: List[Dict] = [mapper(d) for d in data]
+ dict_data: list[dict] = [mapper(d) for d in data]
else:
dict_data = data
dict_data = [{k: self._normalize_data(v, output) for k, v in d.items()} for d in dict_data]
diff --git a/airflow/compat/functools.py b/airflow/compat/functools.py
index e3dea0a660..dc0c520b79 100644
--- a/airflow/compat/functools.py
+++ b/airflow/compat/functools.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import sys
if sys.version_info >= (3, 8):
diff --git a/airflow/compat/sqlalchemy.py b/airflow/compat/sqlalchemy.py
index 427db90a73..b222af463f 100644
--- a/airflow/compat/sqlalchemy.py
+++ b/airflow/compat/sqlalchemy.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from sqlalchemy import Table
from sqlalchemy.engine import Connection
diff --git a/airflow/config_templates/airflow_local_settings.py b/airflow/config_templates/airflow_local_settings.py
index 9e145a0b8b..e08274de31 100644
--- a/airflow/config_templates/airflow_local_settings.py
+++ b/airflow/config_templates/airflow_local_settings.py
@@ -16,10 +16,11 @@
# specific language governing permissions and limitations
# under the License.
"""Airflow logging settings"""
+from __future__ import annotations
import os
from pathlib import Path
-from typing import Any, Dict, Optional, Union
+from typing import Any
from urllib.parse import urlparse
from airflow.configuration import conf
@@ -62,11 +63,11 @@ DAG_PROCESSOR_MANAGER_LOG_LOCATION: str = conf.get_mandatory_value(
# FILENAME_TEMPLATE only uses in Remote Logging Handlers since Airflow 2.3.3
# All of these handlers inherited from FileTaskHandler and providing any value rather than None
# would raise deprecation warning.
-FILENAME_TEMPLATE: Optional[str] = None
+FILENAME_TEMPLATE: str | None = None
PROCESSOR_FILENAME_TEMPLATE: str = conf.get_mandatory_value('logging', 'LOG_PROCESSOR_FILENAME_TEMPLATE')
-DEFAULT_LOGGING_CONFIG: Dict[str, Any] = {
+DEFAULT_LOGGING_CONFIG: dict[str, Any] = {
'version': 1,
'disable_existing_loggers': False,
'formatters': {
@@ -140,7 +141,7 @@ DEFAULT_LOGGING_CONFIG: Dict[str, Any] = {
},
}
-EXTRA_LOGGER_NAMES: Optional[str] = conf.get('logging', 'EXTRA_LOGGER_NAMES', fallback=None)
+EXTRA_LOGGER_NAMES: str | None = conf.get('logging', 'EXTRA_LOGGER_NAMES', fallback=None)
if EXTRA_LOGGER_NAMES:
new_loggers = {
logger_name.strip(): {
@@ -152,7 +153,7 @@ if EXTRA_LOGGER_NAMES:
}
DEFAULT_LOGGING_CONFIG['loggers'].update(new_loggers)
-DEFAULT_DAG_PARSING_LOGGING_CONFIG: Dict[str, Dict[str, Dict[str, Any]]] = {
+DEFAULT_DAG_PARSING_LOGGING_CONFIG: dict[str, dict[str, dict[str, Any]]] = {
'handlers': {
'processor_manager': {
'class': 'logging.handlers.RotatingFileHandler',
@@ -181,7 +182,7 @@ if os.environ.get('CONFIG_PROCESSOR_MANAGER_LOGGER') == 'True':
# Manually create log directory for processor_manager handler as RotatingFileHandler
# will only create file but not the directory.
- processor_manager_handler_config: Dict[str, Any] = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][
+ processor_manager_handler_config: dict[str, Any] = DEFAULT_DAG_PARSING_LOGGING_CONFIG['handlers'][
'processor_manager'
]
directory: str = os.path.dirname(processor_manager_handler_config['filename'])
@@ -195,7 +196,7 @@ REMOTE_LOGGING: bool = conf.getboolean('logging', 'remote_logging')
if REMOTE_LOGGING:
- ELASTICSEARCH_HOST: Optional[str] = conf.get('elasticsearch', 'HOST')
+ ELASTICSEARCH_HOST: str | None = conf.get('elasticsearch', 'HOST')
# Storage bucket URL for remote logging
# S3 buckets should start with "s3://"
@@ -206,7 +207,7 @@ if REMOTE_LOGGING:
REMOTE_BASE_LOG_FOLDER: str = conf.get_mandatory_value('logging', 'REMOTE_BASE_LOG_FOLDER')
if REMOTE_BASE_LOG_FOLDER.startswith('s3://'):
- S3_REMOTE_HANDLERS: Dict[str, Dict[str, Optional[str]]] = {
+ S3_REMOTE_HANDLERS: dict[str, dict[str, str | None]] = {
'task': {
'class': 'airflow.providers.amazon.aws.log.s3_task_handler.S3TaskHandler',
'formatter': 'airflow',
@@ -219,7 +220,7 @@ if REMOTE_LOGGING:
DEFAULT_LOGGING_CONFIG['handlers'].update(S3_REMOTE_HANDLERS)
elif REMOTE_BASE_LOG_FOLDER.startswith('cloudwatch://'):
url_parts = urlparse(REMOTE_BASE_LOG_FOLDER)
- CLOUDWATCH_REMOTE_HANDLERS: Dict[str, Dict[str, Optional[str]]] = {
+ CLOUDWATCH_REMOTE_HANDLERS: dict[str, dict[str, str | None]] = {
'task': {
'class': 'airflow.providers.amazon.aws.log.cloudwatch_task_handler.CloudwatchTaskHandler',
'formatter': 'airflow',
@@ -232,7 +233,7 @@ if REMOTE_LOGGING:
DEFAULT_LOGGING_CONFIG['handlers'].update(CLOUDWATCH_REMOTE_HANDLERS)
elif REMOTE_BASE_LOG_FOLDER.startswith('gs://'):
key_path = conf.get_mandatory_value('logging', 'GOOGLE_KEY_PATH', fallback=None)
- GCS_REMOTE_HANDLERS: Dict[str, Dict[str, Optional[str]]] = {
+ GCS_REMOTE_HANDLERS: dict[str, dict[str, str | None]] = {
'task': {
'class': 'airflow.providers.google.cloud.log.gcs_task_handler.GCSTaskHandler',
'formatter': 'airflow',
@@ -245,7 +246,7 @@ if REMOTE_LOGGING:
DEFAULT_LOGGING_CONFIG['handlers'].update(GCS_REMOTE_HANDLERS)
elif REMOTE_BASE_LOG_FOLDER.startswith('wasb'):
- WASB_REMOTE_HANDLERS: Dict[str, Dict[str, Optional[Union[str, bool]]]] = {
+ WASB_REMOTE_HANDLERS: dict[str, dict[str, str | bool | None]] = {
'task': {
'class': 'airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler',
'formatter': 'airflow',
@@ -293,7 +294,7 @@ if REMOTE_LOGGING:
ELASTICSEARCH_HOST_FIELD: str = conf.get_mandatory_value('elasticsearch', 'HOST_FIELD')
ELASTICSEARCH_OFFSET_FIELD: str = conf.get_mandatory_value('elasticsearch', 'OFFSET_FIELD')
- ELASTIC_REMOTE_HANDLERS: Dict[str, Dict[str, Optional[Union[str, bool]]]] = {
+ ELASTIC_REMOTE_HANDLERS: dict[str, dict[str, str | bool | None]] = {
'task': {
'class': 'airflow.providers.elasticsearch.log.es_task_handler.ElasticsearchTaskHandler',
'formatter': 'airflow',
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index 2c88a759c3..dd51e04848 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -16,7 +16,6 @@
# specific language governing permissions and limitations
# under the License.
-
# This is the template for Airflow's default configuration. When Airflow is
# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If
# it doesn't exist, Airflow uses this template to generate it by replacing
diff --git a/airflow/config_templates/default_celery.py b/airflow/config_templates/default_celery.py
index 0ca5a16769..5faefb6571 100644
--- a/airflow/config_templates/default_celery.py
+++ b/airflow/config_templates/default_celery.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Default celery configuration."""
+from __future__ import annotations
+
import logging
import ssl
diff --git a/airflow/config_templates/default_webserver_config.py b/airflow/config_templates/default_webserver_config.py
index 5c7eaa1b16..0f3ac255ce 100644
--- a/airflow/config_templates/default_webserver_config.py
+++ b/airflow/config_templates/default_webserver_config.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Default configuration for the Airflow webserver"""
+from __future__ import annotations
+
import os
from airflow.www.fab_security.manager import AUTH_DB
diff --git a/airflow/configuration.py b/airflow/configuration.py
index f71ee36ee9..77164ab904 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import datetime
import functools
import json
@@ -34,7 +36,7 @@ from configparser import _UNSET, ConfigParser, NoOptionError, NoSectionError #
from contextlib import suppress
from json.decoder import JSONDecodeError
from re import Pattern
-from typing import IO, Any, Dict, Iterable, List, Optional, Set, Tuple, Union
+from typing import IO, Any, Dict, Iterable, Tuple, Union
from urllib.parse import urlparse
from typing_extensions import overload
@@ -62,7 +64,7 @@ ConfigSourcesType = Dict[str, ConfigSectionSourcesType]
ENV_VAR_PREFIX = 'AIRFLOW__'
-def _parse_sqlite_version(s: str) -> Tuple[int, ...]:
+def _parse_sqlite_version(s: str) -> tuple[int, ...]:
match = _SQLITE3_VERSION_PATTERN.match(s)
if match is None:
return ()
@@ -79,7 +81,7 @@ def expand_env_var(env_var: str) -> str:
...
-def expand_env_var(env_var: Union[str, None]) -> Optional[Union[str, None]]:
+def expand_env_var(env_var: str | None) -> str | None | None:
"""
Expands (potentially nested) env vars by repeatedly applying
`expandvars` and `expanduser` until interpolation stops having
@@ -111,7 +113,7 @@ def run_command(command: str) -> str:
return output
-def _get_config_value_from_secret_backend(config_key: str) -> Optional[str]:
+def _get_config_value_from_secret_backend(config_key: str) -> str | None:
"""Get Config option values from Secret Backend"""
try:
secrets_client = get_custom_secret_backend()
@@ -132,7 +134,7 @@ def _default_config_file_path(file_name: str) -> str:
return os.path.join(templates_dir, file_name)
-def default_config_yaml() -> List[Dict[str, Any]]:
+def default_config_yaml() -> list[dict[str, Any]]:
"""
Read Airflow configs from YAML file
@@ -165,12 +167,12 @@ class AirflowConfigParser(ConfigParser):
# These configs can also be fetched from Secrets backend
# following the "{section}__{name}__secret" pattern
- sensitive_config_values: Set[Tuple[str, str]] = SENSITIVE_CONFIG_VALUES
+ sensitive_config_values: set[tuple[str, str]] = SENSITIVE_CONFIG_VALUES
# A mapping of (new section, new option) -> (old section, old option, since_version).
# When reading new option, the old option will be checked to see if it exists. If it does a
# DeprecationWarning will be issued and the old option will be used instead
- deprecated_options: Dict[Tuple[str, str], Tuple[str, str, str]] = {
+ deprecated_options: dict[tuple[str, str], tuple[str, str, str]] = {
('celery', 'worker_precheck'): ('core', 'worker_precheck', '2.0.0'),
('logging', 'base_log_folder'): ('core', 'base_log_folder', '2.0.0'),
('logging', 'remote_logging'): ('core', 'remote_logging', '2.0.0'),
@@ -229,7 +231,7 @@ class AirflowConfigParser(ConfigParser):
# A mapping of old default values that we want to change and warn the user
# about. Mapping of section -> setting -> { old, replace, by_version }
- deprecated_values: Dict[str, Dict[str, Tuple[Pattern, str, str]]] = {
+ deprecated_values: dict[str, dict[str, tuple[Pattern, str, str]]] = {
'core': {
'hostname_callable': (re.compile(r':'), r'.', '2.1'),
},
@@ -280,7 +282,7 @@ class AirflowConfigParser(ConfigParser):
("webserver", "analytical_tool"): ['google_analytics', 'metarouter', 'segment', ''],
}
- upgraded_values: Dict[Tuple[str, str], str]
+ upgraded_values: dict[tuple[str, str], str]
"""Mapping of (section,option) to the old value that was upgraded"""
# This method transforms option names on every read, get, or set operation.
@@ -289,7 +291,7 @@ class AirflowConfigParser(ConfigParser):
def optionxform(self, optionstr: str) -> str:
return optionstr
- def __init__(self, default_config: Optional[str] = None, *args, **kwargs):
+ def __init__(self, default_config: str | None = None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.upgraded_values = {}
@@ -431,7 +433,7 @@ class AirflowConfigParser(ConfigParser):
def _using_old_value(self, old: Pattern, current_value: str) -> bool:
return old.search(current_value) is not None
- def _update_env_var(self, section: str, name: str, new_value: Union[str]):
+ def _update_env_var(self, section: str, name: str, new_value: str):
env_var = self._env_var_name(section, name)
# Set it as an env var so that any subprocesses keep the same override!
os.environ[env_var] = new_value
@@ -477,7 +479,7 @@ class AirflowConfigParser(ConfigParser):
def _get_cmd_option_from_config_sources(
self, config_sources: ConfigSourcesType, section: str, key: str
- ) -> Optional[str]:
+ ) -> str | None:
fallback_key = key + '_cmd'
if (section, key) in self.sensitive_config_values:
section_dict = config_sources.get(section)
@@ -491,7 +493,7 @@ class AirflowConfigParser(ConfigParser):
return run_command(command)
return None
- def _get_secret_option(self, section: str, key: str) -> Optional[str]:
+ def _get_secret_option(self, section: str, key: str) -> str | None:
"""Get Config option values from Secret Backend"""
fallback_key = key + '_secret'
if (section, key) in self.sensitive_config_values:
@@ -502,7 +504,7 @@ class AirflowConfigParser(ConfigParser):
def _get_secret_option_from_config_sources(
self, config_sources: ConfigSourcesType, section: str, key: str
- ) -> Optional[str]:
+ ) -> str | None:
fallback_key = key + '_secret'
if (section, key) in self.sensitive_config_values:
section_dict = config_sources.get(section)
@@ -528,11 +530,11 @@ class AirflowConfigParser(ConfigParser):
...
@overload # type: ignore[override]
- def get(self, section: str, key: str, **kwargs) -> Optional[str]: # type: ignore[override]
+ def get(self, section: str, key: str, **kwargs) -> str | None: # type: ignore[override]
...
- def get(self, section: str, key: str, **kwargs) -> Optional[str]: # type: ignore[override, misc]
+ def get(self, section: str, key: str, **kwargs) -> str | None: # type: ignore[override, misc]
section = str(section).lower()
key = str(key).lower()
@@ -558,7 +560,7 @@ class AirflowConfigParser(ConfigParser):
return self._get_option_from_default_config(section, key, **kwargs)
- def _get_option_from_default_config(self, section: str, key: str, **kwargs) -> Optional[str]:
+ def _get_option_from_default_config(self, section: str, key: str, **kwargs) -> str | None:
# ...then the default config
if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs:
return expand_env_var(self.airflow_defaults.get(section, key, **kwargs))
@@ -569,8 +571,8 @@ class AirflowConfigParser(ConfigParser):
raise AirflowConfigException(f"section/key [{section}/{key}] not found in config")
def _get_option_from_secrets(
- self, deprecated_key: Optional[str], deprecated_section: Optional[str], key: str, section: str
- ) -> Optional[str]:
+ self, deprecated_key: str | None, deprecated_section: str | None, key: str, section: str
+ ) -> str | None:
# ...then from secret backends
option = self._get_secret_option(section, key)
if option:
@@ -583,8 +585,8 @@ class AirflowConfigParser(ConfigParser):
return None
def _get_option_from_commands(
- self, deprecated_key: Optional[str], deprecated_section: Optional[str], key: str, section: str
- ) -> Optional[str]:
+ self, deprecated_key: str | None, deprecated_section: str | None, key: str, section: str
+ ) -> str | None:
# ...then commands
option = self._get_cmd_option(section, key)
if option:
@@ -598,12 +600,12 @@ class AirflowConfigParser(ConfigParser):
def _get_option_from_config_file(
self,
- deprecated_key: Optional[str],
- deprecated_section: Optional[str],
+ deprecated_key: str | None,
+ deprecated_section: str | None,
key: str,
- kwargs: Dict[str, Any],
+ kwargs: dict[str, Any],
section: str,
- ) -> Optional[str]:
+ ) -> str | None:
# ...then the config file
if super().has_option(section, key):
# Use the parent's methods to get the actual config here to be able to
@@ -616,8 +618,8 @@ class AirflowConfigParser(ConfigParser):
return None
def _get_environment_variables(
- self, deprecated_key: Optional[str], deprecated_section: Optional[str], key: str, section: str
- ) -> Optional[str]:
+ self, deprecated_key: str | None, deprecated_section: str | None, key: str, section: str
+ ) -> str | None:
# first check environment variables
option = self._get_env_var_option(section, key)
if option is not None:
@@ -696,7 +698,7 @@ class AirflowConfigParser(ConfigParser):
def getjson(
self, section: str, key: str, fallback=_UNSET, **kwargs
- ) -> Union[dict, list, str, int, float, None]:
+ ) -> dict | list | str | int | float | None:
"""
Return a config value parsed from a JSON string.
@@ -724,7 +726,7 @@ class AirflowConfigParser(ConfigParser):
def gettimedelta(
self, section: str, key: str, fallback: Any = None, **kwargs
- ) -> Optional[datetime.timedelta]:
+ ) -> datetime.timedelta | None:
"""
Gets the config value for the given section and key, and converts it into datetime.timedelta object.
If the key is missing, then it is considered as `None`.
@@ -760,12 +762,7 @@ class AirflowConfigParser(ConfigParser):
def read(
self,
- filenames: Union[
- str,
- bytes,
- os.PathLike,
- Iterable[Union[str, bytes, os.PathLike]],
- ],
+ filenames: (str | bytes | os.PathLike | Iterable[str | bytes | os.PathLike]),
encoding=None,
):
super().read(filenames=filenames, encoding=encoding)
@@ -773,7 +770,7 @@ class AirflowConfigParser(ConfigParser):
# The RawConfigParser defines "Mapping" from abc.collections is not subscriptable - so we have
# to use Dict here.
def read_dict( # type: ignore[override]
- self, dictionary: Dict[str, Dict[str, Any]], source: str = '<dict>'
+ self, dictionary: dict[str, dict[str, Any]], source: str = '<dict>'
):
super().read_dict(dictionary=dictionary, source=source)
@@ -799,7 +796,7 @@ class AirflowConfigParser(ConfigParser):
if self.airflow_defaults.has_option(section, option) and remove_default:
self.airflow_defaults.remove_option(section, option)
- def getsection(self, section: str) -> Optional[ConfigOptionsDictType]:
+ def getsection(self, section: str) -> ConfigOptionsDictType | None:
"""
Returns the section as a dict. Values are converted to int, float, bool
as required.
@@ -969,12 +966,12 @@ class AirflowConfigParser(ConfigParser):
raw: bool,
):
for (section, key) in self.sensitive_config_values:
- value: Optional[str] = self._get_secret_option_from_config_sources(config_sources, section, key)
+ value: str | None = self._get_secret_option_from_config_sources(config_sources, section, key)
if value:
if not display_sensitive:
value = '< hidden >'
if display_source:
- opt: Union[str, Tuple[str, str]] = (value, 'secret')
+ opt: str | tuple[str, str] = (value, 'secret')
elif raw:
opt = value.replace('%', '%%')
else:
@@ -993,7 +990,7 @@ class AirflowConfigParser(ConfigParser):
opt = self._get_cmd_option_from_config_sources(config_sources, section, key)
if not opt:
continue
- opt_to_set: Union[str, Tuple[str, str], None] = opt
+ opt_to_set: str | tuple[str, str] | None = opt
if not display_sensitive:
opt_to_set = '< hidden >'
if display_source:
@@ -1001,7 +998,7 @@ class AirflowConfigParser(ConfigParser):
elif raw:
opt_to_set = str(opt_to_set).replace('%', '%%')
if opt_to_set is not None:
- dict_to_update: Dict[str, Union[str, Tuple[str, str]]] = {key: opt_to_set}
+ dict_to_update: dict[str, str | tuple[str, str]] = {key: opt_to_set}
config_sources.setdefault(section, OrderedDict()).update(dict_to_update)
del config_sources[section][key + '_cmd']
@@ -1093,10 +1090,10 @@ class AirflowConfigParser(ConfigParser):
@staticmethod
def _replace_config_with_display_sources(
config_sources: ConfigSourcesType,
- configs: Iterable[Tuple[str, ConfigParser]],
+ configs: Iterable[tuple[str, ConfigParser]],
display_source: bool,
raw: bool,
- deprecated_options: Dict[Tuple[str, str], Tuple[str, str, str]],
+ deprecated_options: dict[tuple[str, str], tuple[str, str, str]],
include_env: bool,
include_cmds: bool,
include_secret: bool,
@@ -1121,7 +1118,7 @@ class AirflowConfigParser(ConfigParser):
def _deprecated_value_is_set_in_config(
deprecated_section: str,
deprecated_key: str,
- configs: Iterable[Tuple[str, ConfigParser]],
+ configs: Iterable[tuple[str, ConfigParser]],
) -> bool:
for config_type, config in configs:
if config_type == 'default':
@@ -1144,7 +1141,7 @@ class AirflowConfigParser(ConfigParser):
@staticmethod
def _deprecated_command_is_set_in_config(
- deprecated_section: str, deprecated_key: str, configs: Iterable[Tuple[str, ConfigParser]]
+ deprecated_section: str, deprecated_key: str, configs: Iterable[tuple[str, ConfigParser]]
) -> bool:
return AirflowConfigParser._deprecated_value_is_set_in_config(
deprecated_section=deprecated_section, deprecated_key=deprecated_key + "_cmd", configs=configs
@@ -1159,7 +1156,7 @@ class AirflowConfigParser(ConfigParser):
@staticmethod
def _deprecated_secret_is_set_in_config(
- deprecated_section: str, deprecated_key: str, configs: Iterable[Tuple[str, ConfigParser]]
+ deprecated_section: str, deprecated_key: str, configs: Iterable[tuple[str, ConfigParser]]
) -> bool:
return AirflowConfigParser._deprecated_value_is_set_in_config(
deprecated_section=deprecated_section, deprecated_key=deprecated_key + "_secret", configs=configs
@@ -1180,8 +1177,8 @@ class AirflowConfigParser(ConfigParser):
raw: bool,
section: str,
source_name: str,
- deprecated_options: Dict[Tuple[str, str], Tuple[str, str, str]],
- configs: Iterable[Tuple[str, ConfigParser]],
+ deprecated_options: dict[tuple[str, str], tuple[str, str, str]],
+ configs: Iterable[tuple[str, ConfigParser]],
include_env: bool,
include_cmds: bool,
include_secret: bool,
@@ -1422,7 +1419,7 @@ def load_test_config():
conf.load_test_config()
-def get(*args, **kwargs) -> Optional[ConfigType]:
+def get(*args, **kwargs) -> ConfigType | None:
"""Historical get"""
warnings.warn(
"Accessing configuration method 'get' directly from the configuration module is "
@@ -1470,7 +1467,7 @@ def getint(*args, **kwargs) -> int:
return conf.getint(*args, **kwargs)
-def getsection(*args, **kwargs) -> Optional[ConfigOptionsDictType]:
+def getsection(*args, **kwargs) -> ConfigOptionsDictType | None:
"""Historical getsection"""
warnings.warn(
"Accessing configuration method 'getsection' directly from the configuration module is "
@@ -1530,7 +1527,7 @@ def set(*args, **kwargs) -> None:
conf.set(*args, **kwargs)
-def ensure_secrets_loaded() -> List[BaseSecretsBackend]:
+def ensure_secrets_loaded() -> list[BaseSecretsBackend]:
"""
Ensure that all secrets backends are loaded.
If the secrets_backend_list contains only 2 default backends, reload it.
@@ -1541,7 +1538,7 @@ def ensure_secrets_loaded() -> List[BaseSecretsBackend]:
return secrets_backend_list
-def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
+def get_custom_secret_backend() -> BaseSecretsBackend | None:
"""Get Secret Backend if defined in airflow.cfg"""
secrets_backend_cls = conf.getimport(section='secrets', key='backend')
@@ -1556,7 +1553,7 @@ def get_custom_secret_backend() -> Optional[BaseSecretsBackend]:
return None
-def initialize_secrets_backends() -> List[BaseSecretsBackend]:
+def initialize_secrets_backends() -> list[BaseSecretsBackend]:
"""
* import secrets backend classes
* instantiate them and return them in a list
diff --git a/airflow/contrib/hooks/__init__.py b/airflow/contrib/hooks/__init__.py
index 397cb011c2..cffc937097 100644
--- a/airflow/contrib/hooks/__init__.py
+++ b/airflow/contrib/hooks/__init__.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
"""This package is deprecated. Please use `airflow.hooks` or `airflow.providers.*.hooks`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/contrib/operators/__init__.py b/airflow/contrib/operators/__init__.py
index 03b54fb511..8e02c871b8 100644
--- a/airflow/contrib/operators/__init__.py
+++ b/airflow/contrib/operators/__init__.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
"""This package is deprecated. Please use `airflow.operators` or `airflow.providers.*.operators`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/contrib/secrets/__init__.py b/airflow/contrib/secrets/__init__.py
index 77c53d009d..9e2b4c74ab 100644
--- a/airflow/contrib/secrets/__init__.py
+++ b/airflow/contrib/secrets/__init__.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""This package is deprecated. Please use `airflow.secrets` or `airflow.providers.*.secrets`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/contrib/sensors/__init__.py b/airflow/contrib/sensors/__init__.py
index 259ba1674a..65739f1ed0 100644
--- a/airflow/contrib/sensors/__init__.py
+++ b/airflow/contrib/sensors/__init__.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
"""This package is deprecated. Please use `airflow.sensors` or `airflow.providers.*.sensors`."""
+from __future__ import annotations
import warnings
diff --git a/airflow/contrib/task_runner/__init__.py b/airflow/contrib/task_runner/__init__.py
index a918d5d11d..82bd6bf041 100644
--- a/airflow/contrib/task_runner/__init__.py
+++ b/airflow/contrib/task_runner/__init__.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""This package is deprecated. Please use `airflow.task.task_runner`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/contrib/utils/__init__.py b/airflow/contrib/utils/__init__.py
index 8e921940f6..4057f36868 100644
--- a/airflow/contrib/utils/__init__.py
+++ b/airflow/contrib/utils/__init__.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
"""This package is deprecated. Please use `airflow.utils`."""
+from __future__ import annotations
import warnings
diff --git a/airflow/contrib/utils/log/__init__.py b/airflow/contrib/utils/log/__init__.py
index e5e4e7abfd..baf3a48185 100644
--- a/airflow/contrib/utils/log/__init__.py
+++ b/airflow/contrib/utils/log/__init__.py
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""This package is deprecated. Please use `airflow.utils.log`."""
+from __future__ import annotations
import warnings
diff --git a/airflow/dag_processing/manager.py b/airflow/dag_processing/manager.py
index c45fa91931..270315d5e8 100644
--- a/airflow/dag_processing/manager.py
+++ b/airflow/dag_processing/manager.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""Processes DAGs."""
+from __future__ import annotations
+
import enum
import importlib
import inspect
@@ -32,7 +34,7 @@ from datetime import datetime, timedelta
from importlib import import_module
from multiprocessing.connection import Connection as MultiprocessingConnection
from pathlib import Path
-from typing import Any, Dict, List, NamedTuple, Optional, Union, cast
+from typing import Any, NamedTuple, cast
from setproctitle import setproctitle
from sqlalchemy.orm import Session
@@ -74,8 +76,8 @@ class DagFileStat(NamedTuple):
num_dags: int
import_errors: int
- last_finish_time: Optional[datetime]
- last_duration: Optional[timedelta]
+ last_finish_time: datetime | None
+ last_duration: timedelta | None
run_count: int
@@ -111,12 +113,12 @@ class DagFileProcessorAgent(LoggingMixin, MultiprocessingStartMethodMixin):
dag_directory: os.PathLike,
max_runs: int,
processor_timeout: timedelta,
- dag_ids: Optional[List[str]],
+ dag_ids: list[str] | None,
pickle_dags: bool,
async_mode: bool,
):
super().__init__()
- self._file_path_queue: List[str] = []
+ self._file_path_queue: list[str] = []
self._dag_directory: os.PathLike = dag_directory
self._max_runs = max_runs
self._processor_timeout = processor_timeout
@@ -124,14 +126,14 @@ class DagFileProcessorAgent(LoggingMixin, MultiprocessingStartMethodMixin):
self._pickle_dags = pickle_dags
self._async_mode = async_mode
# Map from file path to the processor
- self._processors: Dict[str, DagFileProcessorProcess] = {}
+ self._processors: dict[str, DagFileProcessorProcess] = {}
# Pipe for communicating signals
- self._process: Optional[multiprocessing.process.BaseProcess] = None
+ self._process: multiprocessing.process.BaseProcess | None = None
self._done: bool = False
# Initialized as true so we do not deactivate w/o any actual DAG parsing.
self._all_files_processed = True
- self._parent_signal_conn: Optional[MultiprocessingConnection] = None
+ self._parent_signal_conn: MultiprocessingConnection | None = None
self._last_parsing_stat_received_at: float = time.monotonic()
@@ -210,7 +212,7 @@ class DagFileProcessorAgent(LoggingMixin, MultiprocessingStartMethodMixin):
max_runs: int,
processor_timeout: timedelta,
signal_conn: MultiprocessingConnection,
- dag_ids: Optional[List[str]],
+ dag_ids: list[str] | None,
pickle_dags: bool,
async_mode: bool,
) -> None:
@@ -372,21 +374,21 @@ class DagFileProcessorManager(LoggingMixin):
dag_directory: os.PathLike,
max_runs: int,
processor_timeout: timedelta,
- dag_ids: Optional[List[str]],
+ dag_ids: list[str] | None,
pickle_dags: bool,
- signal_conn: Optional[MultiprocessingConnection] = None,
+ signal_conn: MultiprocessingConnection | None = None,
async_mode: bool = True,
):
super().__init__()
- self._file_paths: List[str] = []
- self._file_path_queue: List[str] = []
+ self._file_paths: list[str] = []
+ self._file_path_queue: list[str] = []
self._max_runs = max_runs
# signal_conn is None for dag_processor_standalone mode.
self._direct_scheduler_conn = signal_conn
self._pickle_dags = pickle_dags
self._dag_ids = dag_ids
self._async_mode = async_mode
- self._parsing_start_time: Optional[int] = None
+ self._parsing_start_time: int | None = None
self._dag_directory = dag_directory
# Set the signal conn in to non-blocking mode, so that attempting to
@@ -418,12 +420,12 @@ class DagFileProcessorManager(LoggingMixin):
self.print_stats_interval = conf.getint('scheduler', 'print_stats_interval')
# Map from file path to the processor
- self._processors: Dict[str, DagFileProcessorProcess] = {}
+ self._processors: dict[str, DagFileProcessorProcess] = {}
self._num_run = 0
# Map from file path to stats about the file
- self._file_stats: Dict[str, DagFileStat] = {}
+ self._file_stats: dict[str, DagFileStat] = {}
# Last time that the DAG dir was traversed to look for files
self.last_dag_dir_refresh_time = timezone.make_aware(datetime.fromtimestamp(0))
@@ -439,11 +441,11 @@ class DagFileProcessorManager(LoggingMixin):
self.dag_dir_list_interval = conf.getint('scheduler', 'dag_dir_list_interval')
# Mapping file name and callbacks requests
- self._callback_to_execute: Dict[str, List[CallbackRequest]] = defaultdict(list)
+ self._callback_to_execute: dict[str, list[CallbackRequest]] = defaultdict(list)
self._log = logging.getLogger('airflow.processor_manager')
- self.waitables: Dict[Any, Union[MultiprocessingConnection, DagFileProcessorProcess]] = (
+ self.waitables: dict[Any, MultiprocessingConnection | DagFileProcessorProcess] = (
{
self._direct_scheduler_conn: self._direct_scheduler_conn,
}
diff --git a/airflow/dag_processing/processor.py b/airflow/dag_processing/processor.py
index fe56fc7c8e..cebed6a13c 100644
--- a/airflow/dag_processing/processor.py
+++ b/airflow/dag_processing/processor.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import datetime
import logging
@@ -25,7 +26,7 @@ import time
from contextlib import redirect_stderr, redirect_stdout, suppress
from datetime import timedelta
from multiprocessing.connection import Connection as MultiprocessingConnection
-from typing import TYPE_CHECKING, Iterator, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Iterator
from setproctitle import setproctitle
from sqlalchemy import exc, func, or_
@@ -74,9 +75,9 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
self,
file_path: str,
pickle_dags: bool,
- dag_ids: Optional[List[str]],
+ dag_ids: list[str] | None,
dag_directory: str,
- callback_requests: List[CallbackRequest],
+ callback_requests: list[CallbackRequest],
):
super().__init__()
self._file_path = file_path
@@ -86,18 +87,18 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
self._callback_requests = callback_requests
# The process that was launched to process the given .
- self._process: Optional[multiprocessing.process.BaseProcess] = None
+ self._process: multiprocessing.process.BaseProcess | None = None
# The result of DagFileProcessor.process_file(file_path).
- self._result: Optional[Tuple[int, int]] = None
+ self._result: tuple[int, int] | None = None
# Whether the process is done running.
self._done = False
# When the process started.
- self._start_time: Optional[datetime.datetime] = None
+ self._start_time: datetime.datetime | None = None
# This ID is use to uniquely name the process / thread that's launched
# by this processor instance
self._instance_id = DagFileProcessorProcess.class_creation_counter
- self._parent_channel: Optional[MultiprocessingConnection] = None
+ self._parent_channel: MultiprocessingConnection | None = None
DagFileProcessorProcess.class_creation_counter += 1
@property
@@ -110,10 +111,10 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
parent_channel: MultiprocessingConnection,
file_path: str,
pickle_dags: bool,
- dag_ids: Optional[List[str]],
+ dag_ids: list[str] | None,
thread_name: str,
dag_directory: str,
- callback_requests: List[CallbackRequest],
+ callback_requests: list[CallbackRequest],
) -> None:
"""
Process the given file.
@@ -154,7 +155,7 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
log.info("Started process (PID=%s) to work on %s", os.getpid(), file_path)
dag_file_processor = DagFileProcessor(dag_ids=dag_ids, dag_directory=dag_directory, log=log)
- result: Tuple[int, int] = dag_file_processor.process_file(
+ result: tuple[int, int] = dag_file_processor.process_file(
file_path=file_path,
pickle_dags=pickle_dags,
callback_requests=callback_requests,
@@ -269,7 +270,7 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
return self._process.pid
@property
- def exit_code(self) -> Optional[int]:
+ def exit_code(self) -> int | None:
"""
After the process is finished, this can be called to get the return code
@@ -326,7 +327,7 @@ class DagFileProcessorProcess(LoggingMixin, MultiprocessingStartMethodMixin):
return False
@property
- def result(self) -> Optional[Tuple[int, int]]:
+ def result(self) -> tuple[int, int] | None:
"""
:return: result of running DagFileProcessor.process_file()
:rtype: tuple[int, int] or None
@@ -370,12 +371,12 @@ class DagFileProcessor(LoggingMixin):
UNIT_TEST_MODE: bool = conf.getboolean('core', 'UNIT_TEST_MODE')
- def __init__(self, dag_ids: Optional[List[str]], dag_directory: str, log: logging.Logger):
+ def __init__(self, dag_ids: list[str] | None, dag_directory: str, log: logging.Logger):
super().__init__()
self.dag_ids = dag_ids
self._log = log
self._dag_directory = dag_directory
- self.dag_warnings: Set[Tuple[str, str]] = set()
+ self.dag_warnings: set[tuple[str, str]] = set()
@provide_session
def manage_slas(self, dag: DAG, session: Session = None) -> None:
@@ -453,19 +454,19 @@ class DagFileProcessor(LoggingMixin):
session.add_all(sla_misses)
session.commit()
- slas: List[SlaMiss] = (
+ slas: list[SlaMiss] = (
session.query(SlaMiss)
.filter(SlaMiss.notification_sent == False, SlaMiss.dag_id == dag.dag_id) # noqa
.all()
)
if slas:
- sla_dates: List[datetime.datetime] = [sla.execution_date for sla in slas]
- fetched_tis: List[TI] = (
+ sla_dates: list[datetime.datetime] = [sla.execution_date for sla in slas]
+ fetched_tis: list[TI] = (
session.query(TI)
.filter(TI.state != State.SUCCESS, TI.execution_date.in_(sla_dates), TI.dag_id == dag.dag_id)
.all()
)
- blocking_tis: List[TI] = []
+ blocking_tis: list[TI] = []
for ti in fetched_tis:
if ti.task_id in dag.task_ids:
ti.task = dag.get_task(ti.task_id)
@@ -511,7 +512,7 @@ class DagFileProcessor(LoggingMixin):
continue
tasks_missed_sla.append(task)
- emails: Set[str] = set()
+ emails: set[str] = set()
for task in tasks_missed_sla:
if task.email:
if isinstance(task.email, str):
@@ -626,7 +627,7 @@ class DagFileProcessor(LoggingMixin):
@provide_session
def execute_callbacks(
- self, dagbag: DagBag, callback_requests: List[CallbackRequest], session: Session = NEW_SESSION
+ self, dagbag: DagBag, callback_requests: list[CallbackRequest], session: Session = NEW_SESSION
) -> None:
"""
Execute on failure callbacks. These objects can come from SchedulerJob or from
@@ -655,7 +656,7 @@ class DagFileProcessor(LoggingMixin):
session.flush()
def execute_callbacks_without_dag(
- self, callback_requests: List[CallbackRequest], session: Session
+ self, callback_requests: list[CallbackRequest], session: Session
) -> None:
"""
Execute what callbacks we can as "best effort" when the dag cannot be found/had parse errors.
@@ -682,14 +683,12 @@ class DagFileProcessor(LoggingMixin):
dagrun=dag_run, success=not request.is_failure_callback, reason=request.msg, session=session
)
- def _execute_task_callbacks(
- self, dagbag: Optional[DagBag], request: TaskCallbackRequest, session: Session
- ):
+ def _execute_task_callbacks(self, dagbag: DagBag | None, request: TaskCallbackRequest, session: Session):
if not request.is_failure_callback:
return
simple_ti = request.simple_task_instance
- ti: Optional[TI] = (
+ ti: TI | None = (
session.query(TI)
.filter_by(
dag_id=simple_ti.dag_id,
@@ -702,7 +701,7 @@ class DagFileProcessor(LoggingMixin):
if not ti:
return
- task: Optional["Operator"] = None
+ task: Operator | None = None
if dagbag and simple_ti.dag_id in dagbag.dags:
dag = dagbag.dags[simple_ti.dag_id]
@@ -732,10 +731,10 @@ class DagFileProcessor(LoggingMixin):
def process_file(
self,
file_path: str,
- callback_requests: List[CallbackRequest],
+ callback_requests: list[CallbackRequest],
pickle_dags: bool = False,
session: Session = NEW_SESSION,
- ) -> Tuple[int, int]:
+ ) -> tuple[int, int]:
"""
Process a Python file containing Airflow DAGs.
@@ -787,7 +786,7 @@ class DagFileProcessor(LoggingMixin):
if pickle_dags:
paused_dag_ids = DagModel.get_paused_dag_ids(dag_ids=dagbag.dag_ids)
- unpaused_dags: List[DAG] = [
+ unpaused_dags: list[DAG] = [
dag for dag_id, dag in dagbag.dags.items() if dag_id not in paused_dag_ids
]
diff --git a/airflow/datasets/__init__.py b/airflow/datasets/__init__.py
index d8e6451fbf..b61707d8df 100644
--- a/airflow/datasets/__init__.py
+++ b/airflow/datasets/__init__.py
@@ -14,7 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Dict, Optional
+from __future__ import annotations
+
+from typing import Any
import attr
@@ -24,4 +26,4 @@ class Dataset:
"""A Dataset is used for marking data dependencies between workflows."""
uri: str
- extra: Optional[Dict[str, Any]] = None
+ extra: dict[str, Any] | None = None
diff --git a/airflow/datasets/manager.py b/airflow/datasets/manager.py
index f044510863..2c3eda4dcc 100644
--- a/airflow/datasets/manager.py
+++ b/airflow/datasets/manager.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from typing import TYPE_CHECKING
from sqlalchemy import exc
@@ -41,7 +43,7 @@ class DatasetManager(LoggingMixin):
super().__init__(**kwargs)
def register_dataset_change(
- self, *, task_instance: "TaskInstance", dataset: Dataset, extra=None, session: Session, **kwargs
+ self, *, task_instance: TaskInstance, dataset: Dataset, extra=None, session: Session, **kwargs
) -> None:
"""
For local datasets, look them up, record the dataset event, queue dagruns, and broadcast
@@ -104,7 +106,7 @@ class DatasetManager(LoggingMixin):
)
-def resolve_dataset_manager() -> "DatasetManager":
+def resolve_dataset_manager() -> DatasetManager:
_dataset_manager_class = conf.getimport(
section='core',
key='dataset_manager_class',
diff --git a/airflow/decorators/__init__.py b/airflow/decorators/__init__.py
index ad5d6431e5..2485b6645a 100644
--- a/airflow/decorators/__init__.py
+++ b/airflow/decorators/__init__.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from typing import Any
diff --git a/airflow/decorators/base.py b/airflow/decorators/base.py
index c38bcf5921..f64b6a117f 100644
--- a/airflow/decorators/base.py
+++ b/airflow/decorators/base.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import inspect
import re
@@ -26,11 +27,7 @@ from typing import (
Generic,
Iterator,
Mapping,
- Optional,
Sequence,
- Set,
- Tuple,
- Type,
TypeVar,
cast,
overload,
@@ -91,8 +88,8 @@ def validate_python_callable(python_callable: Any) -> None:
def get_unique_task_id(
task_id: str,
- dag: Optional[DAG] = None,
- task_group: Optional[TaskGroup] = None,
+ dag: DAG | None = None,
+ task_group: TaskGroup | None = None,
) -> str:
"""
Generate unique task id given a DAG (or if run in a DAG context)
@@ -159,10 +156,10 @@ class DecoratedOperator(BaseOperator):
*,
python_callable: Callable,
task_id: str,
- op_args: Optional[Collection[Any]] = None,
- op_kwargs: Optional[Mapping[str, Any]] = None,
+ op_args: Collection[Any] | None = None,
+ op_kwargs: Mapping[str, Any] | None = None,
multiple_outputs: bool = False,
- kwargs_to_upstream: Optional[Dict[str, Any]] = None,
+ kwargs_to_upstream: dict[str, Any] | None = None,
**kwargs,
) -> None:
task_id = get_unique_task_id(task_id, kwargs.get('dag'), kwargs.get('task_group'))
@@ -249,9 +246,9 @@ class _TaskDecorator(Generic[FParams, FReturn, OperatorSubclass]):
"""
function: Callable[FParams, FReturn] = attr.ib()
- operator_class: Type[OperatorSubclass]
+ operator_class: type[OperatorSubclass]
multiple_outputs: bool = attr.ib()
- kwargs: Dict[str, Any] = attr.ib(factory=dict)
+ kwargs: dict[str, Any] = attr.ib(factory=dict)
decorator_name: str = attr.ib(repr=False, default="task")
@@ -271,7 +268,7 @@ class _TaskDecorator(Generic[FParams, FReturn, OperatorSubclass]):
raise TypeError(f"@{self.decorator_name} does not support methods")
self.kwargs.setdefault('task_id', self.function.__name__)
- def __call__(self, *args: "FParams.args", **kwargs: "FParams.kwargs") -> XComArg:
+ def __call__(self, *args: FParams.args, **kwargs: FParams.kwargs) -> XComArg:
op = self.operator_class(
python_callable=self.function,
op_args=args,
@@ -297,11 +294,11 @@ class _TaskDecorator(Generic[FParams, FReturn, OperatorSubclass]):
return any(v.kind == inspect.Parameter.VAR_KEYWORD for v in parameters.values())
@cached_property
- def _mappable_function_argument_names(self) -> Set[str]:
+ def _mappable_function_argument_names(self) -> set[str]:
"""Arguments that can be mapped against."""
return set(self.function_signature.parameters)
- def _validate_arg_names(self, func: ValidationSource, kwargs: Dict[str, Any]):
+ def _validate_arg_names(self, func: ValidationSource, kwargs: dict[str, Any]):
# Ensure that context variables are not shadowed.
context_keys_being_mapped = KNOWN_CONTEXT_KEYS.intersection(kwargs)
if len(context_keys_being_mapped) == 1:
@@ -430,14 +427,14 @@ class _TaskDecorator(Generic[FParams, FReturn, OperatorSubclass]):
)
return XComArg(operator=operator)
- def partial(self, **kwargs: Any) -> "_TaskDecorator[FParams, FReturn, OperatorSubclass]":
+ def partial(self, **kwargs: Any) -> _TaskDecorator[FParams, FReturn, OperatorSubclass]:
self._validate_arg_names("partial", kwargs)
old_kwargs = self.kwargs.get("op_kwargs", {})
prevent_duplicates(old_kwargs, kwargs, fail_reason="duplicate partial")
kwargs.update(old_kwargs)
return attr.evolve(self, kwargs={**self.kwargs, "op_kwargs": kwargs})
- def override(self, **kwargs: Any) -> "_TaskDecorator[FParams, FReturn, OperatorSubclass]":
+ def override(self, **kwargs: Any) -> _TaskDecorator[FParams, FReturn, OperatorSubclass]:
return attr.evolve(self, kwargs={**self.kwargs, **kwargs})
@@ -461,13 +458,13 @@ class DecoratedMappedOperator(MappedOperator):
super(DecoratedMappedOperator, DecoratedMappedOperator).__attrs_post_init__(self)
XComArg.apply_upstream_relationship(self, self.op_kwargs_expand_input.value)
- def _expand_mapped_kwargs(self, context: Context, session: Session) -> Tuple[Mapping[str, Any], Set[int]]:
+ def _expand_mapped_kwargs(self, context: Context, session: Session) -> tuple[Mapping[str, Any], set[int]]:
# We only use op_kwargs_expand_input so this must always be empty.
assert self.expand_input is EXPAND_INPUT_EMPTY
op_kwargs, resolved_oids = super()._expand_mapped_kwargs(context, session)
return {"op_kwargs": op_kwargs}, resolved_oids
- def _get_unmap_kwargs(self, mapped_kwargs: Mapping[str, Any], *, strict: bool) -> Dict[str, Any]:
+ def _get_unmap_kwargs(self, mapped_kwargs: Mapping[str, Any], *, strict: bool) -> dict[str, Any]:
partial_op_kwargs = self.partial_kwargs["op_kwargs"]
mapped_op_kwargs = mapped_kwargs["op_kwargs"]
@@ -501,7 +498,7 @@ class Task(Generic[FParams, FReturn]):
def __wrapped__(self) -> Callable[FParams, FReturn]:
...
- def partial(self, **kwargs: Any) -> "Task[FParams, FReturn]":
+ def partial(self, **kwargs: Any) -> Task[FParams, FReturn]:
...
def expand(self, **kwargs: OperatorExpandArgument) -> XComArg:
@@ -510,7 +507,7 @@ class Task(Generic[FParams, FReturn]):
def expand_kwargs(self, kwargs: OperatorExpandKwargsArgument, *, strict: bool = True) -> XComArg:
...
- def override(self, **kwargs: Any) -> "Task[FParams, FReturn]":
+ def override(self, **kwargs: Any) -> Task[FParams, FReturn]:
...
@@ -528,20 +525,20 @@ class TaskDecorator(Protocol):
def __call__(
self,
*,
- multiple_outputs: Optional[bool] = None,
+ multiple_outputs: bool | None = None,
**kwargs: Any,
) -> Callable[[Callable[FParams, FReturn]], Task[FParams, FReturn]]:
"""For the decorator factory ``@task()`` case."""
- def override(self, **kwargs: Any) -> "Task[FParams, FReturn]":
+ def override(self, **kwargs: Any) -> Task[FParams, FReturn]:
...
def task_decorator_factory(
- python_callable: Optional[Callable] = None,
+ python_callable: Callable | None = None,
*,
- multiple_outputs: Optional[bool] = None,
- decorated_operator_class: Type[BaseOperator],
+ multiple_outputs: bool | None = None,
+ decorated_operator_class: type[BaseOperator],
**kwargs,
) -> TaskDecorator:
"""Generate a wrapper that wraps a function into an Airflow operator.
diff --git a/airflow/decorators/branch_python.py b/airflow/decorators/branch_python.py
index d4adce91db..bc00aa4d95 100644
--- a/airflow/decorators/branch_python.py
+++ b/airflow/decorators/branch_python.py
@@ -14,10 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import inspect
from textwrap import dedent
-from typing import Callable, Optional, Sequence
+from typing import Callable, Sequence
from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
from airflow.operators.python import BranchPythonOperator
@@ -66,7 +67,7 @@ class _BranchPythonDecoratedOperator(DecoratedOperator, BranchPythonOperator):
def branch_task(
- python_callable: Optional[Callable] = None, multiple_outputs: Optional[bool] = None, **kwargs
+ python_callable: Callable | None = None, multiple_outputs: bool | None = None, **kwargs
) -> TaskDecorator:
"""
Wraps a python function into a BranchPythonOperator
diff --git a/airflow/decorators/external_python.py b/airflow/decorators/external_python.py
index 7291e19052..a4bab1d099 100644
--- a/airflow/decorators/external_python.py
+++ b/airflow/decorators/external_python.py
@@ -14,10 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import inspect
from textwrap import dedent
-from typing import Callable, Optional, Sequence
+from typing import Callable, Sequence
from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
from airflow.operators.python import ExternalPythonOperator
@@ -71,9 +72,9 @@ class _PythonExternalDecoratedOperator(DecoratedOperator, ExternalPythonOperator
def external_python_task(
- python: Optional[str] = None,
- python_callable: Optional[Callable] = None,
- multiple_outputs: Optional[bool] = None,
+ python: str | None = None,
+ python_callable: Callable | None = None,
+ multiple_outputs: bool | None = None,
**kwargs,
) -> TaskDecorator:
"""Wraps a callable into an Airflow operator to run via a Python virtual environment.
diff --git a/airflow/decorators/python.py b/airflow/decorators/python.py
index 020837930f..0568e4809e 100644
--- a/airflow/decorators/python.py
+++ b/airflow/decorators/python.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import Callable, Optional, Sequence
+from typing import Callable, Sequence
from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
from airflow.operators.python import PythonOperator
@@ -59,8 +60,8 @@ class _PythonDecoratedOperator(DecoratedOperator, PythonOperator):
def python_task(
- python_callable: Optional[Callable] = None,
- multiple_outputs: Optional[bool] = None,
+ python_callable: Callable | None = None,
+ multiple_outputs: bool | None = None,
**kwargs,
) -> TaskDecorator:
"""Wraps a function into an Airflow operator.
diff --git a/airflow/decorators/python_virtualenv.py b/airflow/decorators/python_virtualenv.py
index 523879b063..8891fc112c 100644
--- a/airflow/decorators/python_virtualenv.py
+++ b/airflow/decorators/python_virtualenv.py
@@ -14,10 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import inspect
from textwrap import dedent
-from typing import Callable, Optional, Sequence
+from typing import Callable, Sequence
from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
from airflow.operators.python import PythonVirtualenvOperator
@@ -68,8 +69,8 @@ class _PythonVirtualenvDecoratedOperator(DecoratedOperator, PythonVirtualenvOper
def virtualenv_task(
- python_callable: Optional[Callable] = None,
- multiple_outputs: Optional[bool] = None,
+ python_callable: Callable | None = None,
+ multiple_outputs: bool | None = None,
**kwargs,
) -> TaskDecorator:
"""Wraps a callable into an Airflow operator to run via a Python virtual environment.
diff --git a/airflow/decorators/short_circuit.py b/airflow/decorators/short_circuit.py
index f3aec185b7..b7a87e4eb4 100644
--- a/airflow/decorators/short_circuit.py
+++ b/airflow/decorators/short_circuit.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
-from typing import Callable, Optional, Sequence
+from typing import Callable, Sequence
from airflow.decorators.base import DecoratedOperator, TaskDecorator, task_decorator_factory
from airflow.operators.python import ShortCircuitOperator
@@ -59,8 +60,8 @@ class _ShortCircuitDecoratedOperator(DecoratedOperator, ShortCircuitOperator):
def short_circuit_task(
- python_callable: Optional[Callable] = None,
- multiple_outputs: Optional[bool] = None,
+ python_callable: Callable | None = None,
+ multiple_outputs: bool | None = None,
**kwargs,
) -> TaskDecorator:
"""Wraps a function into an ShortCircuitOperator.
diff --git a/airflow/decorators/task_group.py b/airflow/decorators/task_group.py
index df647d2885..c8d72d0aff 100644
--- a/airflow/decorators/task_group.py
+++ b/airflow/decorators/task_group.py
@@ -19,9 +19,11 @@
A TaskGroup is a collection of closely related tasks on the same DAG that should be grouped
together when the DAG is displayed graphically.
"""
+from __future__ import annotations
+
import functools
from inspect import signature
-from typing import TYPE_CHECKING, Any, Callable, Dict, Generic, Optional, TypeVar, Union, cast, overload
+from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, cast, overload
import attr
@@ -41,8 +43,8 @@ task_group_sig = signature(TaskGroup.__init__)
class TaskGroupDecorator(Generic[R]):
""":meta private:"""
- function: Callable[..., Optional[R]] = attr.ib(validator=attr.validators.is_callable())
- kwargs: Dict[str, Any] = attr.ib(factory=dict)
+ function: Callable[..., R | None] = attr.ib(validator=attr.validators.is_callable())
+ kwargs: dict[str, Any] = attr.ib(factory=dict)
"""kwargs for the TaskGroup"""
@function.validator
@@ -60,7 +62,7 @@ class TaskGroupDecorator(Generic[R]):
def _make_task_group(self, **kwargs) -> TaskGroup:
return TaskGroup(**kwargs)
- def __call__(self, *args, **kwargs) -> Union[R, TaskGroup]:
+ def __call__(self, *args, **kwargs) -> R | TaskGroup:
with self._make_task_group(add_suffix_on_collision=True, **self.kwargs) as task_group:
if self.function.__doc__ and not task_group.tooltip:
task_group.tooltip = self.function.__doc__
@@ -83,7 +85,7 @@ class TaskGroupDecorator(Generic[R]):
# start >> tg >> end
return task_group
- def override(self, **kwargs: Any) -> "TaskGroupDecorator[R]":
+ def override(self, **kwargs: Any) -> TaskGroupDecorator[R]:
return attr.evolve(self, kwargs={**self.kwargs, **kwargs})
@@ -103,10 +105,10 @@ class Group(Generic[F]):
function: F
# Return value should match F's return type, but that's impossible to declare.
- def expand(self, **kwargs: "OperatorExpandArgument") -> Any:
+ def expand(self, **kwargs: OperatorExpandArgument) -> Any:
...
- def partial(self, **kwargs: Any) -> "Group[F]":
+ def partial(self, **kwargs: Any) -> Group[F]:
...
@@ -119,11 +121,11 @@ class Group(Generic[F]):
# disastrous if they go out of sync with TaskGroup.
@overload
def task_group(
- group_id: Optional[str] = None,
+ group_id: str | None = None,
prefix_group_id: bool = True,
- parent_group: Optional[TaskGroup] = None,
- dag: Optional["DAG"] = None,
- default_args: Optional[Dict[str, Any]] = None,
+ parent_group: TaskGroup | None = None,
+ dag: DAG | None = None,
+ default_args: dict[str, Any] | None = None,
tooltip: str = "",
ui_color: str = "CornflowerBlue",
ui_fgcolor: str = "#000",
diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py
index 36a92a0142..90aaace63c 100644
--- a/airflow/example_dags/example_bash_operator.py
+++ b/airflow/example_dags/example_bash_operator.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the BashOperator."""
+from __future__ import annotations
import datetime
diff --git a/airflow/example_dags/example_branch_datetime_operator.py b/airflow/example_dags/example_branch_datetime_operator.py
index dcbee0cef1..3f37db7889 100644
--- a/airflow/example_dags/example_branch_datetime_operator.py
+++ b/airflow/example_dags/example_branch_datetime_operator.py
@@ -15,11 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating the usage of DateTimeBranchOperator with datetime as well as time objects as
targets.
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_branch_day_of_week_operator.py b/airflow/example_dags/example_branch_day_of_week_operator.py
index 76b524b932..f2ee95ec7f 100644
--- a/airflow/example_dags/example_branch_day_of_week_operator.py
+++ b/airflow/example_dags/example_branch_day_of_week_operator.py
@@ -15,10 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating the usage of BranchDayOfWeekOperator.
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_branch_labels.py b/airflow/example_dags/example_branch_labels.py
index 1645abd860..220bb445cf 100644
--- a/airflow/example_dags/example_branch_labels.py
+++ b/airflow/example_dags/example_branch_labels.py
@@ -15,10 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating the usage of labels with different branches.
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_branch_operator.py b/airflow/example_dags/example_branch_operator.py
index e16d06eb13..b0dbd34001 100644
--- a/airflow/example_dags/example_branch_operator.py
+++ b/airflow/example_dags/example_branch_operator.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the BranchPythonOperator."""
+from __future__ import annotations
import random
diff --git a/airflow/example_dags/example_branch_operator_decorator.py b/airflow/example_dags/example_branch_operator_decorator.py
index eecd2c07a0..0cb2cb0b90 100644
--- a/airflow/example_dags/example_branch_operator_decorator.py
+++ b/airflow/example_dags/example_branch_operator_decorator.py
@@ -15,9 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the ``@task.branch`` TaskFlow API decorator."""
-
from __future__ import annotations
import random
diff --git a/airflow/example_dags/example_branch_python_dop_operator_3.py b/airflow/example_dags/example_branch_python_dop_operator_3.py
index e991e16340..a4ec1c3380 100644
--- a/airflow/example_dags/example_branch_python_dop_operator_3.py
+++ b/airflow/example_dags/example_branch_python_dop_operator_3.py
@@ -15,11 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating the usage of ``@task.branch`` TaskFlow API decorator with depends_on_past=True,
where tasks may be run or skipped on alternating runs.
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_complex.py b/airflow/example_dags/example_complex.py
index dffaddd8ab..c287976122 100644
--- a/airflow/example_dags/example_complex.py
+++ b/airflow/example_dags/example_complex.py
@@ -15,10 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example Airflow DAG that shows the complex DAG structure.
"""
+from __future__ import annotations
+
import pendulum
from airflow import models
diff --git a/airflow/example_dags/example_dag_decorator.py b/airflow/example_dags/example_dag_decorator.py
index b43b7f7193..15355f8808 100644
--- a/airflow/example_dags/example_dag_decorator.py
+++ b/airflow/example_dags/example_dag_decorator.py
@@ -15,7 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Dict
+from __future__ import annotations
+
+from typing import Any
import httpx
import pendulum
@@ -53,7 +55,7 @@ def example_dag_decorator(email: str = 'example@example.com'):
get_ip = GetRequestOperator(task_id='get_ip', url="http://httpbin.org/get")
@task(multiple_outputs=True)
- def prepare_email(raw_json: Dict[str, Any]) -> Dict[str, str]:
+ def prepare_email(raw_json: dict[str, Any]) -> dict[str, str]:
external_ip = raw_json['origin']
return {
'subject': f'Server connected from {external_ip}',
diff --git a/airflow/example_dags/example_datasets.py b/airflow/example_dags/example_datasets.py
index 1ea1178086..1d11cf1b76 100644
--- a/airflow/example_dags/example_datasets.py
+++ b/airflow/example_dags/example_datasets.py
@@ -36,6 +36,8 @@ example_dataset_dag4_req_dag1_dag2 should run.
Dags example_dataset_dag5_req_dag1_D and example_dataset_dag6_req_DD should not run because they depend on
datasets that never get updated.
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG, Dataset
diff --git a/airflow/example_dags/example_external_task_marker_dag.py b/airflow/example_dags/example_external_task_marker_dag.py
index 874658d9bb..e193de3330 100644
--- a/airflow/example_dags/example_external_task_marker_dag.py
+++ b/airflow/example_dags/example_external_task_marker_dag.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating setting up inter-DAG dependencies using ExternalTaskSensor and
ExternalTaskMarker.
@@ -38,6 +37,7 @@ ExternalTaskSensor times out. In this case, ExternalTaskSensor will raise Airflo
or AirflowSensorTimeout exception
"""
+from __future__ import annotations
import pendulum
diff --git a/airflow/example_dags/example_kubernetes_executor.py b/airflow/example_dags/example_kubernetes_executor.py
index f14e54e284..81e335c54a 100644
--- a/airflow/example_dags/example_kubernetes_executor.py
+++ b/airflow/example_dags/example_kubernetes_executor.py
@@ -18,6 +18,8 @@
"""
This is an example dag for using a Kubernetes Executor Configuration.
"""
+from __future__ import annotations
+
import logging
import os
diff --git a/airflow/example_dags/example_latest_only.py b/airflow/example_dags/example_latest_only.py
index cecb1dfc18..bf7f5434bb 100644
--- a/airflow/example_dags/example_latest_only.py
+++ b/airflow/example_dags/example_latest_only.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example of the LatestOnlyOperator"""
+from __future__ import annotations
import datetime as dt
diff --git a/airflow/example_dags/example_latest_only_with_trigger.py b/airflow/example_dags/example_latest_only_with_trigger.py
index eb452e1adb..f8a7b2ecf6 100644
--- a/airflow/example_dags/example_latest_only_with_trigger.py
+++ b/airflow/example_dags/example_latest_only_with_trigger.py
@@ -18,6 +18,7 @@
"""
Example LatestOnlyOperator and TriggerRule interactions
"""
+from __future__ import annotations
# [START example]
import datetime
diff --git a/airflow/example_dags/example_local_kubernetes_executor.py b/airflow/example_dags/example_local_kubernetes_executor.py
index 3a3c6607c7..cd1fd6a0c0 100644
--- a/airflow/example_dags/example_local_kubernetes_executor.py
+++ b/airflow/example_dags/example_local_kubernetes_executor.py
@@ -18,6 +18,8 @@
"""
This is an example dag for using a Local Kubernetes Executor Configuration.
"""
+from __future__ import annotations
+
import logging
from datetime import datetime
diff --git a/airflow/example_dags/example_nested_branch_dag.py b/airflow/example_dags/example_nested_branch_dag.py
index 27f758c67b..7c46592455 100644
--- a/airflow/example_dags/example_nested_branch_dag.py
+++ b/airflow/example_dags/example_nested_branch_dag.py
@@ -15,12 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating a workflow with nested branching. The join tasks are created with
``none_failed_min_one_success`` trigger rule such that they are skipped whenever their corresponding
branching tasks are skipped.
"""
+from __future__ import annotations
+
import pendulum
from airflow.decorators import task
diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py
index f851d12349..b5cad5e72e 100644
--- a/airflow/example_dags/example_passing_params_via_test_command.py
+++ b/airflow/example_dags/example_passing_params_via_test_command.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the params arguments in templated arguments."""
+from __future__ import annotations
import datetime
import os
diff --git a/airflow/example_dags/example_python_operator.py b/airflow/example_dags/example_python_operator.py
index 5752d394df..d8284b19ac 100644
--- a/airflow/example_dags/example_python_operator.py
+++ b/airflow/example_dags/example_python_operator.py
@@ -15,11 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating the usage of the TaskFlow API to execute Python functions natively and within a
virtual environment.
"""
+from __future__ import annotations
+
import logging
import os
import shutil
diff --git a/airflow/example_dags/example_short_circuit_decorator.py b/airflow/example_dags/example_short_circuit_decorator.py
index 4e7e098624..41249ae5f2 100644
--- a/airflow/example_dags/example_short_circuit_decorator.py
+++ b/airflow/example_dags/example_short_circuit_decorator.py
@@ -14,8 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the `@task.short_circuit()` TaskFlow decorator."""
+from __future__ import annotations
+
import pendulum
from airflow.decorators import dag, task
diff --git a/airflow/example_dags/example_short_circuit_operator.py b/airflow/example_dags/example_short_circuit_operator.py
index 3fc9f1bd00..a99f7de765 100644
--- a/airflow/example_dags/example_short_circuit_operator.py
+++ b/airflow/example_dags/example_short_circuit_operator.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the ShortCircuitOperator."""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_skip_dag.py b/airflow/example_dags/example_skip_dag.py
index 00d3a3d91e..b081371667 100644
--- a/airflow/example_dags/example_skip_dag.py
+++ b/airflow/example_dags/example_skip_dag.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the EmptyOperator and a custom EmptySkipOperator which skips by default."""
+from __future__ import annotations
import pendulum
diff --git a/airflow/example_dags/example_sla_dag.py b/airflow/example_dags/example_sla_dag.py
index 692b2df735..bdef140572 100644
--- a/airflow/example_dags/example_sla_dag.py
+++ b/airflow/example_dags/example_sla_dag.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+"""Example DAG demonstrating SLA use in Tasks"""
+from __future__ import annotations
import datetime
import time
@@ -22,8 +24,6 @@ import pendulum
from airflow.decorators import dag, task
-"""Example DAG demonstrating SLA use in Tasks"""
-
# [START howto_task_sla]
def sla_callback(dag, task_list, blocking_task_list, slas, blocking_tis):
diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py
index ea45786dfa..600bc5e84b 100644
--- a/airflow/example_dags/example_subdag_operator.py
+++ b/airflow/example_dags/example_subdag_operator.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the SubDagOperator."""
+from __future__ import annotations
# [START example_subdag_operator]
import datetime
diff --git a/airflow/example_dags/example_task_group.py b/airflow/example_dags/example_task_group.py
index 7bc319f98b..5a9bbc7769 100644
--- a/airflow/example_dags/example_task_group.py
+++ b/airflow/example_dags/example_task_group.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the TaskGroup."""
+from __future__ import annotations
+
import pendulum
from airflow.models.dag import DAG
diff --git a/airflow/example_dags/example_task_group_decorator.py b/airflow/example_dags/example_task_group_decorator.py
index 637c721886..b71ec39c41 100644
--- a/airflow/example_dags/example_task_group_decorator.py
+++ b/airflow/example_dags/example_task_group_decorator.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the @taskgroup decorator."""
+from __future__ import annotations
import pendulum
diff --git a/airflow/example_dags/example_time_delta_sensor_async.py b/airflow/example_dags/example_time_delta_sensor_async.py
index 9742833cd6..d1562c5751 100644
--- a/airflow/example_dags/example_time_delta_sensor_async.py
+++ b/airflow/example_dags/example_time_delta_sensor_async.py
@@ -15,11 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example DAG demonstrating ``TimeDeltaSensorAsync``, a drop in replacement for ``TimeDeltaSensor`` that
defers and doesn't occupy a worker slot while it waits
"""
+from __future__ import annotations
import datetime
diff --git a/airflow/example_dags/example_trigger_controller_dag.py b/airflow/example_dags/example_trigger_controller_dag.py
index af6e407a3e..23862166e8 100644
--- a/airflow/example_dags/example_trigger_controller_dag.py
+++ b/airflow/example_dags/example_trigger_controller_dag.py
@@ -15,12 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example usage of the TriggerDagRunOperator. This example holds 2 DAGs:
1. 1st DAG (example_trigger_controller_dag) holds a TriggerDagRunOperator, which will trigger the 2nd DAG
2. 2nd DAG (example_trigger_target_dag) which will be triggered by the TriggerDagRunOperator in the 1st DAG
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_trigger_target_dag.py b/airflow/example_dags/example_trigger_target_dag.py
index 20f73cf4fc..b7eada41d9 100644
--- a/airflow/example_dags/example_trigger_target_dag.py
+++ b/airflow/example_dags/example_trigger_target_dag.py
@@ -15,12 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Example usage of the TriggerDagRunOperator. This example holds 2 DAGs:
1. 1st DAG (example_trigger_controller_dag) holds a TriggerDagRunOperator, which will trigger the 2nd DAG
2. 2nd DAG (example_trigger_target_dag) which will be triggered by the TriggerDagRunOperator in the 1st DAG
"""
+from __future__ import annotations
+
import pendulum
from airflow import DAG
diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py
index 971c8ff58a..47affb67b3 100644
--- a/airflow/example_dags/example_xcom.py
+++ b/airflow/example_dags/example_xcom.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of XComs."""
+from __future__ import annotations
+
import pendulum
from airflow import DAG, XComArg
diff --git a/airflow/example_dags/example_xcomargs.py b/airflow/example_dags/example_xcomargs.py
index 4101ca58ae..6a87980aa8 100644
--- a/airflow/example_dags/example_xcomargs.py
+++ b/airflow/example_dags/example_xcomargs.py
@@ -15,8 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Example DAG demonstrating the usage of the XComArgs."""
+from __future__ import annotations
+
import logging
import pendulum
diff --git a/airflow/example_dags/libs/helper.py b/airflow/example_dags/libs/helper.py
index a3d3a720a0..e6c2e3c458 100644
--- a/airflow/example_dags/libs/helper.py
+++ b/airflow/example_dags/libs/helper.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
def print_stuff():
diff --git a/airflow/example_dags/plugins/workday.py b/airflow/example_dags/plugins/workday.py
index 77111a7939..db68c29541 100644
--- a/airflow/example_dags/plugins/workday.py
+++ b/airflow/example_dags/plugins/workday.py
@@ -15,12 +15,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Plugin to demonstrate timetable registration and accommodate example DAGs."""
+from __future__ import annotations
# [START howto_timetable]
from datetime import timedelta
-from typing import Optional
from pendulum import UTC, Date, DateTime, Time
@@ -47,9 +46,9 @@ class AfterWorkdayTimetable(Timetable):
def next_dagrun_info(
self,
*,
- last_automated_data_interval: Optional[DataInterval],
+ last_automated_data_interval: DataInterval | None,
restriction: TimeRestriction,
- ) -> Optional[DagRunInfo]:
+ ) -> DagRunInfo | None:
if last_automated_data_interval is not None: # There was a previous run on the regular schedule.
last_start = last_automated_data_interval.start
last_start_weekday = last_start.weekday()
diff --git a/airflow/example_dags/subdags/subdag.py b/airflow/example_dags/subdags/subdag.py
index 0686507942..6cc1f4f28f 100644
--- a/airflow/example_dags/subdags/subdag.py
+++ b/airflow/example_dags/subdags/subdag.py
@@ -15,8 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Helper function to generate a DAG and operators given some arguments."""
+from __future__ import annotations
# [START subdag]
import pendulum
diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py
index d6b50128e8..06ca4af693 100644
--- a/airflow/example_dags/tutorial.py
+++ b/airflow/example_dags/tutorial.py
@@ -15,12 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
### Tutorial Documentation
Documentation that goes along with the Airflow tutorial located
[here](https://airflow.apache.org/tutorial.html)
"""
+from __future__ import annotations
+
# [START tutorial]
# [START import_module]
from datetime import datetime, timedelta
diff --git a/airflow/example_dags/tutorial_dag.py b/airflow/example_dags/tutorial_dag.py
index 03157de82f..3c99ce101d 100644
--- a/airflow/example_dags/tutorial_dag.py
+++ b/airflow/example_dags/tutorial_dag.py
@@ -15,12 +15,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
"""
### DAG Tutorial Documentation
This DAG is demonstrating an Extract -> Transform -> Load pipeline
"""
+from __future__ import annotations
+
# [START tutorial]
# [START import_module]
import json
diff --git a/airflow/example_dags/tutorial_taskflow_api.py b/airflow/example_dags/tutorial_taskflow_api.py
index ba8aef09a6..bddf0645ce 100644
--- a/airflow/example_dags/tutorial_taskflow_api.py
+++ b/airflow/example_dags/tutorial_taskflow_api.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+from __future__ import annotations
# [START tutorial]
# [START import_module]
diff --git a/airflow/example_dags/tutorial_taskflow_api_virtualenv.py b/airflow/example_dags/tutorial_taskflow_api_virtualenv.py
index 732d533eb6..627a2ec662 100644
--- a/airflow/example_dags/tutorial_taskflow_api_virtualenv.py
+++ b/airflow/example_dags/tutorial_taskflow_api_virtualenv.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
+from __future__ import annotations
import logging
import shutil
diff --git a/airflow/exceptions.py b/airflow/exceptions.py
index 8754158847..e280b4211a 100644
--- a/airflow/exceptions.py
+++ b/airflow/exceptions.py
@@ -15,14 +15,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
# Note: Any AirflowException raised is expected to cause the TaskInstance
# to be marked in an ERROR state
"""Exceptions used by Airflow"""
+from __future__ import annotations
+
import datetime
import warnings
from http import HTTPStatus
-from typing import Any, Dict, List, NamedTuple, Optional, Sized
+from typing import Any, NamedTuple, Sized
class AirflowException(Exception):
@@ -200,7 +201,7 @@ class DuplicateTaskIdFound(AirflowException):
class TaskAlreadyInTaskGroup(AirflowException):
"""Raise when a Task cannot be added to a TaskGroup since it already belongs to another TaskGroup."""
- def __init__(self, task_id: str, existing_group_id: Optional[str], new_group_id: str) -> None:
+ def __init__(self, task_id: str, existing_group_id: str | None, new_group_id: str) -> None:
super().__init__(task_id, new_group_id)
self.task_id = task_id
self.existing_group_id = existing_group_id
@@ -262,7 +263,7 @@ class BackfillUnfinished(AirflowException):
class FileSyntaxError(NamedTuple):
"""Information about a single error in a file."""
- line_no: Optional[int]
+ line_no: int | None
message: str
def __str__(self):
@@ -278,7 +279,7 @@ class AirflowFileParseException(AirflowException):
:param parse_errors: File syntax errors
"""
- def __init__(self, msg: str, file_path: str, parse_errors: List[FileSyntaxError]) -> None:
+ def __init__(self, msg: str, file_path: str, parse_errors: list[FileSyntaxError]) -> None:
super().__init__(msg)
self.msg = msg
self.file_path = file_path
@@ -316,8 +317,8 @@ class TaskDeferred(BaseException):
*,
trigger,
method_name: str,
- kwargs: Optional[Dict[str, Any]] = None,
- timeout: Optional[datetime.timedelta] = None,
+ kwargs: dict[str, Any] | None = None,
+ timeout: datetime.timedelta | None = None,
):
super().__init__()
self.trigger = trigger
@@ -343,12 +344,12 @@ class PodReconciliationError(AirflowException):
class RemovedInAirflow3Warning(DeprecationWarning):
"""Issued for usage of deprecated features that will be removed in Airflow3."""
- deprecated_since: Optional[str] = None
+ deprecated_since: str | None = None
"Indicates the airflow version that started raising this deprecation warning"
class AirflowProviderDeprecationWarning(DeprecationWarning):
"""Issued for usage of deprecated features of Airflow provider."""
- deprecated_provider_since: Optional[str] = None
+ deprecated_provider_since: str | None = None
"Indicates the provider version that started raising this deprecation warning"
diff --git a/airflow/executors/base_executor.py b/airflow/executors/base_executor.py
index 0168106c2d..1d1b0e2cf7 100644
--- a/airflow/executors/base_executor.py
+++ b/airflow/executors/base_executor.py
@@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.
"""Base executor - this is the base class for all the implemented executors."""
+from __future__ import annotations
+
import sys
import warnings
from collections import OrderedDict
-from typing import Any, Counter, Dict, List, Optional, Sequence, Set, Tuple, Union
+from typing import Any, Counter, List, Optional, Sequence, Tuple
from airflow.callbacks.base_callback_sink import BaseCallbackSink
from airflow.callbacks.callback_requests import CallbackRequest
@@ -64,15 +66,15 @@ class BaseExecutor(LoggingMixin):
``0`` for infinity
"""
- job_id: Union[None, int, str] = None
- callback_sink: Optional[BaseCallbackSink] = None
+ job_id: None | int | str = None
+ callback_sink: BaseCallbackSink | None = None
def __init__(self, parallelism: int = PARALLELISM):
super().__init__()
self.parallelism: int = parallelism
self.queued_tasks: OrderedDict[TaskInstanceKey, QueuedTaskInstanceType] = OrderedDict()
- self.running: Set[TaskInstanceKey] = set()
- self.event_buffer: Dict[TaskInstanceKey, EventBufferValueType] = {}
+ self.running: set[TaskInstanceKey] = set()
+ self.event_buffer: dict[TaskInstanceKey, EventBufferValueType] = {}
self.attempts: Counter[TaskInstanceKey] = Counter()
def __repr__(self):
@@ -86,7 +88,7 @@ class BaseExecutor(LoggingMixin):
task_instance: TaskInstance,
command: CommandType,
priority: int = 1,
- queue: Optional[str] = None,
+ queue: str | None = None,
):
"""Queues command to task"""
if task_instance.key not in self.queued_tasks:
@@ -99,13 +101,13 @@ class BaseExecutor(LoggingMixin):
self,
task_instance: TaskInstance,
mark_success: bool = False,
- pickle_id: Optional[str] = None,
+ pickle_id: str | None = None,
ignore_all_deps: bool = False,
ignore_depends_on_past: bool = False,
ignore_task_deps: bool = False,
ignore_ti_state: bool = False,
- pool: Optional[str] = None,
- cfg_path: Optional[str] = None,
+ pool: str | None = None,
+ cfg_path: str | None = None,
) -> None:
"""Queues task instance."""
pool = pool or task_instance.pool
@@ -172,7 +174,7 @@ class BaseExecutor(LoggingMixin):
self.log.debug("Calling the %s sync method", self.__class__)
self.sync()
- def order_queued_tasks_by_priority(self) -> List[Tuple[TaskInstanceKey, QueuedTaskInstanceType]]:
+ def order_queued_tasks_by_priority(self) -> list[tuple[TaskInstanceKey, QueuedTaskInstanceType]]:
"""
Orders the queued tasks by priority.
@@ -223,7 +225,7 @@ class BaseExecutor(LoggingMixin):
if task_tuples:
self._process_tasks(task_tuples)
- def _process_tasks(self, task_tuples: List[TaskTuple]) -> None:
+ def _process_tasks(self, task_tuples: list[TaskTuple]) -> None:
for key, command, queue, executor_config in task_tuples:
del self.queued_tasks[key]
self.execute_async(key=key, command=command, queue=queue, executor_config=executor_config)
@@ -262,7 +264,7 @@ class BaseExecutor(LoggingMixin):
"""
self.change_state(key, State.SUCCESS, info)
- def get_event_buffer(self, dag_ids=None) -> Dict[TaskInstanceKey, EventBufferValueType]:
+ def get_event_buffer(self, dag_ids=None) -> dict[TaskInstanceKey, EventBufferValueType]:
"""
Returns and flush the event buffer. In case dag_ids is specified
it will only return and flush events for the given dag_ids. Otherwise
@@ -271,7 +273,7 @@ class BaseExecutor(LoggingMixin):
:param dag_ids: the dag_ids to return events for; returns all if given ``None``.
:return: a dict of events
"""
- cleared_events: Dict[TaskInstanceKey, EventBufferValueType] = {}
+ cleared_events: dict[TaskInstanceKey, EventBufferValueType] = {}
if dag_ids is None:
cleared_events = self.event_buffer
self.event_buffer = {}
@@ -286,8 +288,8 @@ class BaseExecutor(LoggingMixin):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None: # pragma: no cover
"""
This method will execute the command asynchronously.
@@ -334,7 +336,7 @@ class BaseExecutor(LoggingMixin):
return sys.maxsize
@staticmethod
- def validate_command(command: List[str]) -> None:
+ def validate_command(command: list[str]) -> None:
"""
Back-compat method to Check if the command to execute is airflow command
@@ -351,7 +353,7 @@ class BaseExecutor(LoggingMixin):
BaseExecutor.validate_airflow_tasks_run_command(command)
@staticmethod
- def validate_airflow_tasks_run_command(command: List[str]) -> Tuple[Optional[str], Optional[str]]:
+ def validate_airflow_tasks_run_command(command: list[str]) -> tuple[str | None, str | None]:
"""
Check if the command to execute is airflow command
@@ -360,8 +362,8 @@ class BaseExecutor(LoggingMixin):
if command[0:3] != ["airflow", "tasks", "run"]:
raise ValueError('The command must start with ["airflow", "tasks", "run"].')
if len(command) > 3 and "--help" not in command:
- dag_id: Optional[str] = None
- task_id: Optional[str] = None
+ dag_id: str | None = None
+ task_id: str | None = None
for arg in command[4:]:
if not arg.startswith("--"):
if dag_id is None:
diff --git a/airflow/executors/celery_executor.py b/airflow/executors/celery_executor.py
index 7edc90ab9d..0554e88999 100644
--- a/airflow/executors/celery_executor.py
+++ b/airflow/executors/celery_executor.py
@@ -21,6 +21,8 @@
For more information on how the CeleryExecutor works, take a look at the guide:
:ref:`executor:CeleryExecutor`
"""
+from __future__ import annotations
+
import datetime
import logging
import math
@@ -33,7 +35,7 @@ from collections import Counter
from concurrent.futures import ProcessPoolExecutor
from enum import Enum
from multiprocessing import cpu_count
-from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Sequence, Set, Tuple, Union
+from typing import Any, Mapping, MutableMapping, Optional, Sequence, Tuple
from celery import Celery, Task, states as celery_states
from celery.backends.base import BaseKeyValueStoreBackend
@@ -97,7 +99,7 @@ def execute_command(command_to_exec: CommandType) -> None:
raise
-def _execute_in_fork(command_to_exec: CommandType, celery_task_id: Optional[str] = None) -> None:
+def _execute_in_fork(command_to_exec: CommandType, celery_task_id: str | None = None) -> None:
pid = os.fork()
if pid:
# In parent, wait for the child
@@ -136,7 +138,7 @@ def _execute_in_fork(command_to_exec: CommandType, celery_task_id: Optional[str]
os._exit(ret)
-def _execute_in_subprocess(command_to_exec: CommandType, celery_task_id: Optional[str] = None) -> None:
+def _execute_in_subprocess(command_to_exec: CommandType, celery_task_id: str | None = None) -> None:
env = os.environ.copy()
if celery_task_id:
env["external_executor_id"] = celery_task_id
@@ -169,7 +171,7 @@ TaskInstanceInCelery = Tuple[TaskInstanceKey, CommandType, Optional[str], Task]
def send_task_to_executor(
task_tuple: TaskInstanceInCelery,
-) -> Tuple[TaskInstanceKey, CommandType, Union[AsyncResult, ExceptionWithTraceback]]:
+) -> tuple[TaskInstanceKey, CommandType, AsyncResult | ExceptionWithTraceback]:
"""Sends task to executor."""
key, command, queue, task_to_run = task_tuple
try:
@@ -238,11 +240,11 @@ class CeleryExecutor(BaseExecutor):
self._sync_parallelism = max(1, cpu_count() - 1)
self.bulk_state_fetcher = BulkStateFetcher(self._sync_parallelism)
self.tasks = {}
- self.stalled_task_timeouts: Dict[TaskInstanceKey, datetime.datetime] = {}
+ self.stalled_task_timeouts: dict[TaskInstanceKey, datetime.datetime] = {}
self.stalled_task_timeout = datetime.timedelta(
seconds=conf.getint('celery', 'stalled_task_timeout', fallback=0)
)
- self.adopted_task_timeouts: Dict[TaskInstanceKey, datetime.datetime] = {}
+ self.adopted_task_timeouts: dict[TaskInstanceKey, datetime.datetime] = {}
self.task_adoption_timeout = (
datetime.timedelta(seconds=conf.getint('celery', 'task_adoption_timeout', fallback=600))
or self.stalled_task_timeout
@@ -262,7 +264,7 @@ class CeleryExecutor(BaseExecutor):
"""
return max(1, int(math.ceil(1.0 * to_send_count / self._sync_parallelism)))
- def _process_tasks(self, task_tuples: List[TaskTuple]) -> None:
+ def _process_tasks(self, task_tuples: list[TaskTuple]) -> None:
task_tuples_to_send = [task_tuple[:3] + (execute_command,) for task_tuple in task_tuples]
first_task = next(t[3] for t in task_tuples_to_send)
@@ -307,7 +309,7 @@ class CeleryExecutor(BaseExecutor):
# If the task runs _really quickly_ we may already have a result!
self.update_task_state(key, result.state, getattr(result, 'info', None))
- def _send_tasks_to_celery(self, task_tuples_to_send: List[TaskInstanceInCelery]):
+ def _send_tasks_to_celery(self, task_tuples_to_send: list[TaskInstanceInCelery]):
if len(task_tuples_to_send) == 1 or self._sync_parallelism == 1:
# One tuple, or max one process -> send it in the main thread.
return list(map(send_task_to_executor, task_tuples_to_send))
@@ -354,8 +356,8 @@ class CeleryExecutor(BaseExecutor):
self._send_stalled_tis_back_to_scheduler(timedout_keys)
def _get_timedout_ti_keys(
- self, task_timeouts: Dict[TaskInstanceKey, datetime.datetime]
- ) -> List[TaskInstanceKey]:
+ self, task_timeouts: dict[TaskInstanceKey, datetime.datetime]
+ ) -> list[TaskInstanceKey]:
"""
These timeouts exist to check to see if any of our tasks have not progressed
in the expected time. This can happen for few different reasons, usually related
@@ -387,7 +389,7 @@ class CeleryExecutor(BaseExecutor):
@provide_session
def _send_stalled_tis_back_to_scheduler(
- self, keys: List[TaskInstanceKey], session: Session = NEW_SESSION
+ self, keys: list[TaskInstanceKey], session: Session = NEW_SESSION
) -> None:
try:
session.query(TaskInstance).filter(
@@ -536,7 +538,7 @@ class CeleryExecutor(BaseExecutor):
return not_adopted_tis
def _set_celery_pending_task_timeout(
- self, key: TaskInstanceKey, timeout_type: Optional[_CeleryPendingTaskTimeoutType]
+ self, key: TaskInstanceKey, timeout_type: _CeleryPendingTaskTimeoutType | None
) -> None:
"""
We use the fact that dicts maintain insertion order, and the the timeout for a
@@ -551,7 +553,7 @@ class CeleryExecutor(BaseExecutor):
self.stalled_task_timeouts[key] = utcnow() + self.stalled_task_timeout
-def fetch_celery_task_state(async_result: AsyncResult) -> Tuple[str, Union[str, ExceptionWithTraceback], Any]:
+def fetch_celery_task_state(async_result: AsyncResult) -> tuple[str, str | ExceptionWithTraceback, Any]:
"""
Fetch and return the state of the given celery task. The scope of this function is
global so that it can be called by subprocesses in the pool.
@@ -586,7 +588,7 @@ class BulkStateFetcher(LoggingMixin):
super().__init__()
self._sync_parallelism = sync_parallelism
- def _tasks_list_to_task_ids(self, async_tasks) -> Set[str]:
+ def _tasks_list_to_task_ids(self, async_tasks) -> set[str]:
return {a.task_id for a in async_tasks}
def get_many(self, async_results) -> Mapping[str, EventBufferValueType]:
diff --git a/airflow/executors/celery_kubernetes_executor.py b/airflow/executors/celery_kubernetes_executor.py
index e19e7d32e0..c959412303 100644
--- a/airflow/executors/celery_kubernetes_executor.py
+++ b/airflow/executors/celery_kubernetes_executor.py
@@ -15,7 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Dict, List, Optional, Sequence, Set, Union
+from __future__ import annotations
+
+from typing import Sequence
from airflow.callbacks.base_callback_sink import BaseCallbackSink
from airflow.callbacks.callback_requests import CallbackRequest
@@ -37,19 +39,19 @@ class CeleryKubernetesExecutor(LoggingMixin):
"""
supports_ad_hoc_ti_run: bool = True
- callback_sink: Optional[BaseCallbackSink] = None
+ callback_sink: BaseCallbackSink | None = None
KUBERNETES_QUEUE = conf.get('celery_kubernetes_executor', 'kubernetes_queue')
def __init__(self, celery_executor: CeleryExecutor, kubernetes_executor: KubernetesExecutor):
super().__init__()
- self._job_id: Optional[int] = None
+ self._job_id: int | None = None
self.celery_executor = celery_executor
self.kubernetes_executor = kubernetes_executor
self.kubernetes_executor.kubernetes_queue = self.KUBERNETES_QUEUE
@property
- def queued_tasks(self) -> Dict[TaskInstanceKey, QueuedTaskInstanceType]:
+ def queued_tasks(self) -> dict[TaskInstanceKey, QueuedTaskInstanceType]:
"""Return queued tasks from celery and kubernetes executor"""
queued_tasks = self.celery_executor.queued_tasks.copy()
queued_tasks.update(self.kubernetes_executor.queued_tasks)
@@ -57,12 +59,12 @@ class CeleryKubernetesExecutor(LoggingMixin):
return queued_tasks
@property
- def running(self) -> Set[TaskInstanceKey]:
+ def running(self) -> set[TaskInstanceKey]:
"""Return running tasks from celery and kubernetes executor"""
return self.celery_executor.running.union(self.kubernetes_executor.running)
@property
- def job_id(self) -> Optional[int]:
+ def job_id(self) -> int | None:
"""
This is a class attribute in BaseExecutor but since this is not really an executor, but a wrapper
of executors we implement as property so we can have custom setter.
@@ -70,7 +72,7 @@ class CeleryKubernetesExecutor(LoggingMixin):
return self._job_id
@job_id.setter
- def job_id(self, value: Optional[int]) -> None:
+ def job_id(self, value: int | None) -> None:
"""job_id is manipulated by SchedulerJob. We must propagate the job_id to wrapped executors."""
self._job_id = value
self.kubernetes_executor.job_id = value
@@ -91,7 +93,7 @@ class CeleryKubernetesExecutor(LoggingMixin):
task_instance: TaskInstance,
command: CommandType,
priority: int = 1,
- queue: Optional[str] = None,
+ queue: str | None = None,
) -> None:
"""Queues command via celery or kubernetes executor"""
executor = self._router(task_instance)
@@ -102,13 +104,13 @@ class CeleryKubernetesExecutor(LoggingMixin):
self,
task_instance: TaskInstance,
mark_success: bool = False,
- pickle_id: Optional[str] = None,
+ pickle_id: str | None = None,
ignore_all_deps: bool = False,
ignore_depends_on_past: bool = False,
ignore_task_deps: bool = False,
ignore_ti_state: bool = False,
- pool: Optional[str] = None,
- cfg_path: Optional[str] = None,
+ pool: str | None = None,
+ cfg_path: str | None = None,
) -> None:
"""Queues task instance via celery or kubernetes executor"""
executor = self._router(SimpleTaskInstance.from_ti(task_instance))
@@ -144,8 +146,8 @@ class CeleryKubernetesExecutor(LoggingMixin):
self.kubernetes_executor.heartbeat()
def get_event_buffer(
- self, dag_ids: Optional[List[str]] = None
- ) -> Dict[TaskInstanceKey, EventBufferValueType]:
+ self, dag_ids: list[str] | None = None
+ ) -> dict[TaskInstanceKey, EventBufferValueType]:
"""
Returns and flush the event buffer from celery and kubernetes executor
@@ -184,7 +186,7 @@ class CeleryKubernetesExecutor(LoggingMixin):
self.celery_executor.terminate()
self.kubernetes_executor.terminate()
- def _router(self, simple_task_instance: SimpleTaskInstance) -> Union[CeleryExecutor, KubernetesExecutor]:
+ def _router(self, simple_task_instance: SimpleTaskInstance) -> CeleryExecutor | KubernetesExecutor:
"""
Return either celery_executor or kubernetes_executor
diff --git a/airflow/executors/dask_executor.py b/airflow/executors/dask_executor.py
index 6488da9649..9357ad62d4 100644
--- a/airflow/executors/dask_executor.py
+++ b/airflow/executors/dask_executor.py
@@ -22,8 +22,10 @@ DaskExecutor
For more information on how the DaskExecutor works, take a look at the guide:
:ref:`executor:DaskExecutor`
"""
+from __future__ import annotations
+
import subprocess
-from typing import Any, Dict, Optional
+from typing import Any
from distributed import Client, Future, as_completed
from distributed.security import Security
@@ -52,8 +54,8 @@ class DaskExecutor(BaseExecutor):
self.tls_ca = conf.get('dask', 'tls_ca')
self.tls_key = conf.get('dask', 'tls_key')
self.tls_cert = conf.get('dask', 'tls_cert')
- self.client: Optional[Client] = None
- self.futures: Optional[Dict[Future, TaskInstanceKey]] = None
+ self.client: Client | None = None
+ self.futures: dict[Future, TaskInstanceKey] | None = None
def start(self) -> None:
if self.tls_ca or self.tls_key or self.tls_cert:
@@ -73,8 +75,8 @@ class DaskExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
self.validate_airflow_tasks_run_command(command)
diff --git a/airflow/executors/debug_executor.py b/airflow/executors/debug_executor.py
index 0177602b19..d727ee8795 100644
--- a/airflow/executors/debug_executor.py
+++ b/airflow/executors/debug_executor.py
@@ -22,9 +22,10 @@ DebugExecutor
For more information on how the DebugExecutor works, take a look at the guide:
:ref:`executor:DebugExecutor`
"""
+from __future__ import annotations
import threading
-from typing import Any, Dict, List, Optional
+from typing import Any
from airflow.configuration import conf
from airflow.executors.base_executor import BaseExecutor
@@ -44,9 +45,9 @@ class DebugExecutor(BaseExecutor):
def __init__(self):
super().__init__()
- self.tasks_to_run: List[TaskInstance] = []
+ self.tasks_to_run: list[TaskInstance] = []
# Place where we keep information for task instance raw run
- self.tasks_params: Dict[TaskInstanceKey, Dict[str, Any]] = {}
+ self.tasks_params: dict[TaskInstanceKey, dict[str, Any]] = {}
self.fail_fast = conf.getboolean("debug", "fail_fast")
def execute_async(self, *args, **kwargs) -> None:
@@ -88,13 +89,13 @@ class DebugExecutor(BaseExecutor):
self,
task_instance: TaskInstance,
mark_success: bool = False,
- pickle_id: Optional[str] = None,
+ pickle_id: str | None = None,
ignore_all_deps: bool = False,
ignore_depends_on_past: bool = False,
ignore_task_deps: bool = False,
ignore_ti_state: bool = False,
- pool: Optional[str] = None,
- cfg_path: Optional[str] = None,
+ pool: str | None = None,
+ cfg_path: str | None = None,
) -> None:
"""Queues task instance with empty command because we do not need it."""
self.queue_command(
diff --git a/airflow/executors/executor_constants.py b/airflow/executors/executor_constants.py
index 55a3a7f766..c6219f3b4a 100644
--- a/airflow/executors/executor_constants.py
+++ b/airflow/executors/executor_constants.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
LOCAL_EXECUTOR = "LocalExecutor"
LOCAL_KUBERNETES_EXECUTOR = "LocalKubernetesExecutor"
diff --git a/airflow/executors/executor_loader.py b/airflow/executors/executor_loader.py
index 153b8ed243..0b7b63cfb1 100644
--- a/airflow/executors/executor_loader.py
+++ b/airflow/executors/executor_loader.py
@@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.
"""All executors."""
+from __future__ import annotations
+
import logging
from contextlib import suppress
from enum import Enum, unique
-from typing import TYPE_CHECKING, Optional, Tuple, Type
+from typing import TYPE_CHECKING
from airflow.exceptions import AirflowConfigException
from airflow.executors.executor_constants import (
@@ -51,7 +53,7 @@ class ConnectorSource(Enum):
class ExecutorLoader:
"""Keeps constants for all the currently available executors."""
- _default_executor: Optional["BaseExecutor"] = None
+ _default_executor: BaseExecutor | None = None
executors = {
LOCAL_EXECUTOR: 'airflow.executors.local_executor.LocalExecutor',
LOCAL_KUBERNETES_EXECUTOR: 'airflow.executors.local_kubernetes_executor.LocalKubernetesExecutor',
@@ -64,7 +66,7 @@ class ExecutorLoader:
}
@classmethod
- def get_default_executor(cls) -> "BaseExecutor":
+ def get_default_executor(cls) -> BaseExecutor:
"""Creates a new instance of the configured executor if none exists and returns it."""
if cls._default_executor is not None:
return cls._default_executor
@@ -77,7 +79,7 @@ class ExecutorLoader:
return cls._default_executor
@classmethod
- def load_executor(cls, executor_name: str) -> "BaseExecutor":
+ def load_executor(cls, executor_name: str) -> BaseExecutor:
"""
Loads the executor.
@@ -107,7 +109,7 @@ class ExecutorLoader:
return executor_cls()
@classmethod
- def import_executor_cls(cls, executor_name: str) -> Tuple[Type["BaseExecutor"], ConnectorSource]:
+ def import_executor_cls(cls, executor_name: str) -> tuple[type[BaseExecutor], ConnectorSource]:
"""
Imports the executor class.
@@ -133,7 +135,7 @@ class ExecutorLoader:
return import_string(executor_name), ConnectorSource.CUSTOM_PATH
@classmethod
- def __load_celery_kubernetes_executor(cls) -> "BaseExecutor":
+ def __load_celery_kubernetes_executor(cls) -> BaseExecutor:
celery_executor = import_string(cls.executors[CELERY_EXECUTOR])()
kubernetes_executor = import_string(cls.executors[KUBERNETES_EXECUTOR])()
@@ -141,7 +143,7 @@ class ExecutorLoader:
return celery_kubernetes_executor_cls(celery_executor, kubernetes_executor)
@classmethod
- def __load_local_kubernetes_executor(cls) -> "BaseExecutor":
+ def __load_local_kubernetes_executor(cls) -> BaseExecutor:
local_executor = import_string(cls.executors[LOCAL_EXECUTOR])()
kubernetes_executor = import_string(cls.executors[KUBERNETES_EXECUTOR])()
diff --git a/airflow/executors/kubernetes_executor.py b/airflow/executors/kubernetes_executor.py
index 7a9e0552a0..a718aa1907 100644
--- a/airflow/executors/kubernetes_executor.py
+++ b/airflow/executors/kubernetes_executor.py
@@ -21,6 +21,7 @@ KubernetesExecutor
For more information on how the KubernetesExecutor works, take a look at the guide:
:ref:`executor:KubernetesExecutor`
"""
+from __future__ import annotations
import functools
import json
@@ -28,7 +29,7 @@ import multiprocessing
import time
from datetime import timedelta
from queue import Empty, Queue
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from typing import Any, Dict, Optional, Sequence, Tuple
from kubernetes import client, watch
from kubernetes.client import Configuration, models as k8s
@@ -77,10 +78,10 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
def __init__(
self,
- namespace: Optional[str],
+ namespace: str | None,
multi_namespace_mode: bool,
- watcher_queue: 'Queue[KubernetesWatchType]',
- resource_version: Optional[str],
+ watcher_queue: Queue[KubernetesWatchType],
+ resource_version: str | None,
scheduler_job_id: str,
kube_config: Configuration,
):
@@ -121,10 +122,10 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
def _run(
self,
kube_client: client.CoreV1Api,
- resource_version: Optional[str],
+ resource_version: str | None,
scheduler_job_id: str,
kube_config: Any,
- ) -> Optional[str]:
+ ) -> str | None:
self.log.info('Event: and now my watch begins starting at resource_version: %s', resource_version)
watcher = watch.Watch()
@@ -135,7 +136,7 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
for key, value in kube_config.kube_client_request_args.items():
kwargs[key] = value
- last_resource_version: Optional[str] = None
+ last_resource_version: str | None = None
if self.multi_namespace_mode:
list_worker_pods = functools.partial(
watcher.stream, kube_client.list_pod_for_all_namespaces, **kwargs
@@ -193,7 +194,7 @@ class KubernetesJobWatcher(multiprocessing.Process, LoggingMixin):
pod_id: str,
namespace: str,
status: str,
- annotations: Dict[str, str],
+ annotations: dict[str, str],
resource_version: str,
event: Any,
) -> None:
@@ -234,8 +235,8 @@ class AirflowKubernetesScheduler(LoggingMixin):
def __init__(
self,
kube_config: Any,
- task_queue: 'Queue[KubernetesJobType]',
- result_queue: 'Queue[KubernetesResultsType]',
+ task_queue: Queue[KubernetesJobType],
+ result_queue: Queue[KubernetesResultsType],
kube_client: client.CoreV1Api,
scheduler_job_id: str,
):
@@ -412,7 +413,7 @@ class AirflowKubernetesScheduler(LoggingMixin):
self._manager.shutdown()
-def get_base_pod_from_template(pod_template_file: Optional[str], kube_config: Any) -> k8s.V1Pod:
+def get_base_pod_from_template(pod_template_file: str | None, kube_config: Any) -> k8s.V1Pod:
"""
Reads either the pod_template_file set in the executor_config or the base pod_template_file
set in the airflow.cfg to craft a "base pod" that will be used by the KubernetesExecutor
@@ -435,14 +436,14 @@ class KubernetesExecutor(BaseExecutor):
def __init__(self):
self.kube_config = KubeConfig()
self._manager = multiprocessing.Manager()
- self.task_queue: 'Queue[KubernetesJobType]' = self._manager.Queue()
- self.result_queue: 'Queue[KubernetesResultsType]' = self._manager.Queue()
- self.kube_scheduler: Optional[AirflowKubernetesScheduler] = None
- self.kube_client: Optional[client.CoreV1Api] = None
- self.scheduler_job_id: Optional[str] = None
- self.event_scheduler: Optional[EventScheduler] = None
- self.last_handled: Dict[TaskInstanceKey, float] = {}
- self.kubernetes_queue: Optional[str] = None
+ self.task_queue: Queue[KubernetesJobType] = self._manager.Queue()
+ self.result_queue: Queue[KubernetesResultsType] = self._manager.Queue()
+ self.kube_scheduler: AirflowKubernetesScheduler | None = None
+ self.kube_client: client.CoreV1Api | None = None
+ self.scheduler_job_id: str | None = None
+ self.event_scheduler: EventScheduler | None = None
+ self.last_handled: dict[TaskInstanceKey, float] = {}
+ self.kubernetes_queue: str | None = None
super().__init__(parallelism=self.kube_config.parallelism)
@provide_session
@@ -465,7 +466,7 @@ class KubernetesExecutor(BaseExecutor):
query = session.query(TaskInstance).filter(TaskInstance.state == State.QUEUED)
if self.kubernetes_queue:
query = query.filter(TaskInstance.queue == self.kubernetes_queue)
- queued_tis: List[TaskInstance] = query.all()
+ queued_tis: list[TaskInstance] = query.all()
self.log.info('Found %s queued task instances', len(queued_tis))
# Go through the "last seen" dictionary and clean out old entries
@@ -543,8 +544,8 @@ class KubernetesExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
"""Executes task asynchronously"""
self.log.info('Add task %s with command %s with executor_config %s', key, command, executor_config)
@@ -685,7 +686,7 @@ class KubernetesExecutor(BaseExecutor):
)
self.kube_scheduler.delete_pod(pod.metadata.name, pod.metadata.namespace)
- def _change_state(self, key: TaskInstanceKey, state: Optional[str], pod_id: str, namespace: str) -> None:
+ def _change_state(self, key: TaskInstanceKey, state: str | None, pod_id: str, namespace: str) -> None:
if state != State.RUNNING:
if self.kube_config.delete_worker_pods:
if not self.kube_scheduler:
@@ -715,7 +716,7 @@ class KubernetesExecutor(BaseExecutor):
return tis_to_flush
def adopt_launched_task(
- self, kube_client: client.CoreV1Api, pod: k8s.V1Pod, pod_ids: Dict[TaskInstanceKey, k8s.V1Pod]
+ self, kube_client: client.CoreV1Api, pod: k8s.V1Pod, pod_ids: dict[TaskInstanceKey, k8s.V1Pod]
) -> None:
"""
Patch existing pod so that the current KubernetesJobWatcher can monitor it via label selectors
diff --git a/airflow/executors/local_executor.py b/airflow/executors/local_executor.py
index 10628708d1..4fae990bc3 100644
--- a/airflow/executors/local_executor.py
+++ b/airflow/executors/local_executor.py
@@ -22,6 +22,8 @@ LocalExecutor
For more information on how the LocalExecutor works, take a look at the guide:
:ref:`executor:LocalExecutor`
"""
+from __future__ import annotations
+
import logging
import os
import subprocess
@@ -29,7 +31,7 @@ from abc import abstractmethod
from multiprocessing import Manager, Process
from multiprocessing.managers import SyncManager
from queue import Empty, Queue
-from typing import Any, List, Optional, Tuple, Union
+from typing import Any, Optional, Tuple
from setproctitle import getproctitle, setproctitle
@@ -54,10 +56,10 @@ class LocalWorkerBase(Process, LoggingMixin):
:param result_queue: the queue to store result state
"""
- def __init__(self, result_queue: 'Queue[TaskInstanceStateType]'):
+ def __init__(self, result_queue: Queue[TaskInstanceStateType]):
super().__init__(target=self.do_work)
self.daemon: bool = True
- self.result_queue: 'Queue[TaskInstanceStateType]' = result_queue
+ self.result_queue: Queue[TaskInstanceStateType] = result_queue
def run(self):
# We know we've just started a new process, so lets disconnect from the metadata db now
@@ -148,7 +150,7 @@ class LocalWorker(LocalWorkerBase):
"""
def __init__(
- self, result_queue: 'Queue[TaskInstanceStateType]', key: TaskInstanceKey, command: CommandType
+ self, result_queue: Queue[TaskInstanceStateType], key: TaskInstanceKey, command: CommandType
):
super().__init__(result_queue)
self.key: TaskInstanceKey = key
@@ -168,7 +170,7 @@ class QueuedLocalWorker(LocalWorkerBase):
:param result_queue: queue where worker puts results after finishing tasks
"""
- def __init__(self, task_queue: 'Queue[ExecutorWorkType]', result_queue: 'Queue[TaskInstanceStateType]'):
+ def __init__(self, task_queue: Queue[ExecutorWorkType], result_queue: Queue[TaskInstanceStateType]):
super().__init__(result_queue=result_queue)
self.task_queue = task_queue
@@ -205,14 +207,12 @@ class LocalExecutor(BaseExecutor):
super().__init__(parallelism=parallelism)
if self.parallelism < 0:
raise AirflowException("parallelism must be bigger than or equal to 0")
- self.manager: Optional[SyncManager] = None
- self.result_queue: Optional['Queue[TaskInstanceStateType]'] = None
- self.workers: List[QueuedLocalWorker] = []
+ self.manager: SyncManager | None = None
+ self.result_queue: Queue[TaskInstanceStateType] | None = None
+ self.workers: list[QueuedLocalWorker] = []
self.workers_used: int = 0
self.workers_active: int = 0
- self.impl: Optional[
- Union['LocalExecutor.UnlimitedParallelism', 'LocalExecutor.LimitedParallelism']
- ] = None
+ self.impl: None | (LocalExecutor.UnlimitedParallelism | LocalExecutor.LimitedParallelism) = None
class UnlimitedParallelism:
"""
@@ -222,8 +222,8 @@ class LocalExecutor(BaseExecutor):
:param executor: the executor instance to implement.
"""
- def __init__(self, executor: 'LocalExecutor'):
- self.executor: 'LocalExecutor' = executor
+ def __init__(self, executor: LocalExecutor):
+ self.executor: LocalExecutor = executor
def start(self) -> None:
"""Starts the executor."""
@@ -234,8 +234,8 @@ class LocalExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
"""
Executes task asynchronously.
@@ -278,9 +278,9 @@ class LocalExecutor(BaseExecutor):
:param executor: the executor instance to implement.
"""
- def __init__(self, executor: 'LocalExecutor'):
- self.executor: 'LocalExecutor' = executor
- self.queue: Optional['Queue[ExecutorWorkType]'] = None
+ def __init__(self, executor: LocalExecutor):
+ self.executor: LocalExecutor = executor
+ self.queue: Queue[ExecutorWorkType] | None = None
def start(self) -> None:
"""Starts limited parallelism implementation."""
@@ -303,8 +303,8 @@ class LocalExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
"""
Executes task asynchronously.
@@ -361,8 +361,8 @@ class LocalExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
"""Execute asynchronously."""
if not self.impl:
diff --git a/airflow/executors/local_kubernetes_executor.py b/airflow/executors/local_kubernetes_executor.py
index 749348d249..dcb4b2c392 100644
--- a/airflow/executors/local_kubernetes_executor.py
+++ b/airflow/executors/local_kubernetes_executor.py
@@ -15,7 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import Dict, List, Optional, Sequence, Set, Union
+from __future__ import annotations
+
+from typing import Sequence
from airflow.callbacks.base_callback_sink import BaseCallbackSink
from airflow.callbacks.callback_requests import CallbackRequest
@@ -37,19 +39,19 @@ class LocalKubernetesExecutor(LoggingMixin):
"""
supports_ad_hoc_ti_run: bool = True
- callback_sink: Optional[BaseCallbackSink] = None
+ callback_sink: BaseCallbackSink | None = None
KUBERNETES_QUEUE = conf.get('local_kubernetes_executor', 'kubernetes_queue')
def __init__(self, local_executor: LocalExecutor, kubernetes_executor: KubernetesExecutor):
super().__init__()
- self._job_id: Optional[str] = None
+ self._job_id: str | None = None
self.local_executor = local_executor
self.kubernetes_executor = kubernetes_executor
self.kubernetes_executor.kubernetes_queue = self.KUBERNETES_QUEUE
@property
- def queued_tasks(self) -> Dict[TaskInstanceKey, QueuedTaskInstanceType]:
+ def queued_tasks(self) -> dict[TaskInstanceKey, QueuedTaskInstanceType]:
"""Return queued tasks from local and kubernetes executor"""
queued_tasks = self.local_executor.queued_tasks.copy()
queued_tasks.update(self.kubernetes_executor.queued_tasks)
@@ -57,12 +59,12 @@ class LocalKubernetesExecutor(LoggingMixin):
return queued_tasks
@property
- def running(self) -> Set[TaskInstanceKey]:
+ def running(self) -> set[TaskInstanceKey]:
"""Return running tasks from local and kubernetes executor"""
return self.local_executor.running.union(self.kubernetes_executor.running)
@property
- def job_id(self) -> Optional[str]:
+ def job_id(self) -> str | None:
"""
This is a class attribute in BaseExecutor but since this is not really an executor, but a wrapper
of executors we implement as property so we can have custom setter.
@@ -70,7 +72,7 @@ class LocalKubernetesExecutor(LoggingMixin):
return self._job_id
@job_id.setter
- def job_id(self, value: Optional[str]) -> None:
+ def job_id(self, value: str | None) -> None:
"""job_id is manipulated by SchedulerJob. We must propagate the job_id to wrapped executors."""
self._job_id = value
self.kubernetes_executor.job_id = value
@@ -92,7 +94,7 @@ class LocalKubernetesExecutor(LoggingMixin):
task_instance: TaskInstance,
command: CommandType,
priority: int = 1,
- queue: Optional[str] = None,
+ queue: str | None = None,
) -> None:
"""Queues command via local or kubernetes executor"""
executor = self._router(task_instance)
@@ -103,13 +105,13 @@ class LocalKubernetesExecutor(LoggingMixin):
self,
task_instance: TaskInstance,
mark_success: bool = False,
- pickle_id: Optional[str] = None,
+ pickle_id: str | None = None,
ignore_all_deps: bool = False,
ignore_depends_on_past: bool = False,
ignore_task_deps: bool = False,
ignore_ti_state: bool = False,
- pool: Optional[str] = None,
- cfg_path: Optional[str] = None,
+ pool: str | None = None,
+ cfg_path: str | None = None,
) -> None:
"""Queues task instance via local or kubernetes executor"""
executor = self._router(SimpleTaskInstance.from_ti(task_instance))
@@ -143,8 +145,8 @@ class LocalKubernetesExecutor(LoggingMixin):
self.kubernetes_executor.heartbeat()
def get_event_buffer(
- self, dag_ids: Optional[List[str]] = None
- ) -> Dict[TaskInstanceKey, EventBufferValueType]:
+ self, dag_ids: list[str] | None = None
+ ) -> dict[TaskInstanceKey, EventBufferValueType]:
"""
Returns and flush the event buffer from local and kubernetes executor
@@ -183,7 +185,7 @@ class LocalKubernetesExecutor(LoggingMixin):
self.local_executor.terminate()
self.kubernetes_executor.terminate()
- def _router(self, simple_task_instance: SimpleTaskInstance) -> Union[LocalExecutor, KubernetesExecutor]:
+ def _router(self, simple_task_instance: SimpleTaskInstance) -> LocalExecutor | KubernetesExecutor:
"""
Return either local_executor or kubernetes_executor
diff --git a/airflow/executors/sequential_executor.py b/airflow/executors/sequential_executor.py
index 6ce3002f02..c7c2f00417 100644
--- a/airflow/executors/sequential_executor.py
+++ b/airflow/executors/sequential_executor.py
@@ -22,8 +22,10 @@ SequentialExecutor
For more information on how the SequentialExecutor works, take a look at the guide:
:ref:`executor:SequentialExecutor`
"""
+from __future__ import annotations
+
import subprocess
-from typing import Any, Optional
+from typing import Any
from airflow.executors.base_executor import BaseExecutor, CommandType
from airflow.models.taskinstance import TaskInstanceKey
@@ -48,8 +50,8 @@ class SequentialExecutor(BaseExecutor):
self,
key: TaskInstanceKey,
command: CommandType,
- queue: Optional[str] = None,
- executor_config: Optional[Any] = None,
+ queue: str | None = None,
+ executor_config: Any | None = None,
) -> None:
self.validate_airflow_tasks_run_command(command)
self.commands_to_run.append((key, command))
diff --git a/airflow/hooks/__init__.py b/airflow/hooks/__init__.py
index 44556107cd..83338b9750 100644
--- a/airflow/hooks/__init__.py
+++ b/airflow/hooks/__init__.py
@@ -15,8 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# fmt: off
+# fmt:, off
"""Hooks."""
+from __future__ import annotations
+
from airflow.utils.deprecation_tools import add_deprecated_classes
__deprecated_classes = {
diff --git a/airflow/hooks/base.py b/airflow/hooks/base.py
index 9bd7854ab2..a0c9f3d8f6 100644
--- a/airflow/hooks/base.py
+++ b/airflow/hooks/base.py
@@ -16,9 +16,11 @@
# specific language governing permissions and limitations
# under the License.
"""Base class for all hooks"""
+from __future__ import annotations
+
import logging
import warnings
-from typing import TYPE_CHECKING, Any, Dict, List
+from typing import TYPE_CHECKING, Any
from airflow.exceptions import RemovedInAirflow3Warning
from airflow.typing_compat import Protocol
@@ -40,7 +42,7 @@ class BaseHook(LoggingMixin):
"""
@classmethod
- def get_connections(cls, conn_id: str) -> List["Connection"]:
+ def get_connections(cls, conn_id: str) -> list[Connection]:
"""
Get all connections as an iterable, given the connection id.
@@ -56,7 +58,7 @@ class BaseHook(LoggingMixin):
return [cls.get_connection(conn_id)]
@classmethod
- def get_connection(cls, conn_id: str) -> "Connection":
+ def get_connection(cls, conn_id: str) -> Connection:
"""
Get connection, given connection id.
@@ -70,7 +72,7 @@ class BaseHook(LoggingMixin):
return conn
@classmethod
- def get_hook(cls, conn_id: str) -> "BaseHook":
+ def get_hook(cls, conn_id: str) -> BaseHook:
"""
Returns default hook for this connection id.
@@ -87,11 +89,11 @@ class BaseHook(LoggingMixin):
raise NotImplementedError()
@classmethod
- def get_connection_form_widgets(cls) -> Dict[str, Any]:
+ def get_connection_form_widgets(cls) -> dict[str, Any]:
...
@classmethod
- def get_ui_field_behaviour(cls) -> Dict[str, Any]:
+ def get_ui_field_behaviour(cls) -> dict[str, Any]:
...
@@ -140,7 +142,7 @@ class DiscoverableHook(Protocol):
hook_name: str
@staticmethod
- def get_connection_form_widgets() -> Dict[str, Any]:
+ def get_connection_form_widgets() -> dict[str, Any]:
"""
Returns dictionary of widgets to be added for the hook to handle extra values.
@@ -156,7 +158,7 @@ class DiscoverableHook(Protocol):
...
@staticmethod
- def get_ui_field_behaviour() -> Dict[str, Any]:
+ def get_ui_field_behaviour() -> dict[str, Any]:
"""
Returns dictionary describing customizations to implement in javascript handling the
connection form. Should be compliant with airflow/customized_form_field_behaviours.schema.json'
diff --git a/airflow/hooks/dbapi.py b/airflow/hooks/dbapi.py
index a5e38bad6f..cd4a39af8d 100644
--- a/airflow/hooks/dbapi.py
+++ b/airflow/hooks/dbapi.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/hooks/filesystem.py b/airflow/hooks/filesystem.py
index c694940c2a..34c1119655 100644
--- a/airflow/hooks/filesystem.py
+++ b/airflow/hooks/filesystem.py
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
from airflow.hooks.base import BaseHook
diff --git a/airflow/hooks/subprocess.py b/airflow/hooks/subprocess.py
index fa8c706c69..7f29af305a 100644
--- a/airflow/hooks/subprocess.py
+++ b/airflow/hooks/subprocess.py
@@ -14,13 +14,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import contextlib
import os
import signal
from collections import namedtuple
from subprocess import PIPE, STDOUT, Popen
from tempfile import TemporaryDirectory, gettempdir
-from typing import Dict, List, Optional
from airflow.hooks.base import BaseHook
@@ -31,15 +32,15 @@ class SubprocessHook(BaseHook):
"""Hook for running processes with the ``subprocess`` module"""
def __init__(self) -> None:
- self.sub_process: Optional[Popen[bytes]] = None
+ self.sub_process: Popen[bytes] | None = None
super().__init__()
def run_command(
self,
- command: List[str],
- env: Optional[Dict[str, str]] = None,
+ command: list[str],
+ env: dict[str, str] | None = None,
output_encoding: str = 'utf-8',
- cwd: Optional[str] = None,
+ cwd: str | None = None,
) -> SubprocessResult:
"""
Execute the command.
diff --git a/airflow/jobs/__init__.py b/airflow/jobs/__init__.py
index fe95886d5c..217e5db960 100644
--- a/airflow/jobs/__init__.py
+++ b/airflow/jobs/__init__.py
@@ -15,4 +15,3 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
diff --git a/airflow/jobs/backfill_job.py b/airflow/jobs/backfill_job.py
index cb8fda6eca..4a2d3e7bd4 100644
--- a/airflow/jobs/backfill_job.py
+++ b/airflow/jobs/backfill_job.py
@@ -15,10 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
import time
-from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Sequence, Set, Tuple
+from typing import TYPE_CHECKING, Any, Iterable, Iterator, Sequence
import attr
import pendulum
@@ -88,15 +88,15 @@ class BackfillJob(BaseJob):
:param total_runs: Number of total dag runs able to run
"""
- to_run: Dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict)
- running: Dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict)
- skipped: Set[TaskInstanceKey] = attr.ib(factory=set)
- succeeded: Set[TaskInstanceKey] = attr.ib(factory=set)
- failed: Set[TaskInstanceKey] = attr.ib(factory=set)
- not_ready: Set[TaskInstanceKey] = attr.ib(factory=set)
- deadlocked: Set[TaskInstance] = attr.ib(factory=set)
- active_runs: List[DagRun] = attr.ib(factory=list)
- executed_dag_run_dates: Set[pendulum.DateTime] = attr.ib(factory=set)
+ to_run: dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict)
+ running: dict[TaskInstanceKey, TaskInstance] = attr.ib(factory=dict)
+ skipped: set[TaskInstanceKey] = attr.ib(factory=set)
+ succeeded: set[TaskInstanceKey] = attr.ib(factory=set)
+ failed: set[TaskInstanceKey] = attr.ib(factory=set)
+ not_ready: set[TaskInstanceKey] = attr.ib(factory=set)
+ deadlocked: set[TaskInstance] = attr.ib(factory=set)
+ active_runs: list[DagRun] = attr.ib(factory=list)
+ executed_dag_run_dates: set[pendulum.DateTime] = attr.ib(factory=set)
finished_runs: int = 0
total_runs: int = 0
@@ -228,7 +228,7 @@ class BackfillJob(BaseJob):
def _manage_executor_state(
self, running, session
- ) -> Iterator[Tuple["MappedOperator", str, Sequence[TaskInstance], int]]:
+ ) -> Iterator[tuple[MappedOperator, str, Sequence[TaskInstance], int]]:
"""
Checks if the executor agrees with the state of task instances
that are running.
@@ -292,7 +292,7 @@ class BackfillJob(BaseJob):
# check if we are scheduling on top of a already existing dag_run
# we could find a "scheduled" run instead of a "backfill"
runs = DagRun.find(dag_id=dag.dag_id, execution_date=run_date, session=session)
- run: Optional[DagRun]
+ run: DagRun | None
if runs:
run = runs[0]
if run.state == DagRunState.RUNNING:
@@ -703,7 +703,7 @@ class BackfillJob(BaseJob):
return err
- def _get_dag_with_subdags(self) -> List[DAG]:
+ def _get_dag_with_subdags(self) -> list[DAG]:
return [self.dag] + self.dag.subdags
@provide_session
diff --git a/airflow/jobs/base_job.py b/airflow/jobs/base_job.py
index a796e3d105..8eb9e46c2d 100644
--- a/airflow/jobs/base_job.py
+++ b/airflow/jobs/base_job.py
@@ -15,10 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
from time import sleep
-from typing import Optional
from sqlalchemy import Column, Index, Integer, String
from sqlalchemy.exc import OperationalError
@@ -119,7 +118,7 @@ class BaseJob(Base, LoggingMixin):
@classmethod
@provide_session
- def most_recent_job(cls, session=None) -> Optional['BaseJob']:
+ def most_recent_job(cls, session=None) -> BaseJob | None:
"""
Return the most recent job of this type, if any, based on last
heartbeat received.
diff --git a/airflow/jobs/local_task_job.py b/airflow/jobs/local_task_job.py
index 147475da4b..1881511f9b 100644
--- a/airflow/jobs/local_task_job.py
+++ b/airflow/jobs/local_task_job.py
@@ -15,9 +15,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
+
import signal
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
import psutil
from sqlalchemy.exc import OperationalError
@@ -52,9 +53,9 @@ class LocalTaskJob(BaseJob):
ignore_task_deps: bool = False,
ignore_ti_state: bool = False,
mark_success: bool = False,
- pickle_id: Optional[str] = None,
- pool: Optional[str] = None,
- external_executor_id: Optional[str] = None,
+ pickle_id: str | None = None,
+ pool: str | None = None,
+ external_executor_id: str | None = None,
*args,
**kwargs,
):
diff --git a/airflow/jobs/scheduler_job.py b/airflow/jobs/scheduler_job.py
index 5a24b530bf..1354374505 100644
--- a/airflow/jobs/scheduler_job.py
+++ b/airflow/jobs/scheduler_job.py
@@ -15,7 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
+
import itertools
import logging
import multiprocessing
@@ -27,7 +28,7 @@ import warnings
from collections import defaultdict
from datetime import datetime, timedelta
from pathlib import Path
-from typing import TYPE_CHECKING, Collection, DefaultDict, Dict, Iterator, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Collection, DefaultDict, Iterator
from sqlalchemy import func, not_, or_, text
from sqlalchemy.exc import OperationalError
@@ -115,8 +116,8 @@ class SchedulerJob(BaseJob):
num_times_parse_dags: int = -1,
scheduler_idle_sleep_time: float = conf.getfloat('scheduler', 'scheduler_idle_sleep_time'),
do_pickle: bool = False,
- log: Optional[logging.Logger] = None,
- processor_poll_interval: Optional[float] = None,
+ log: logging.Logger | None = None,
+ processor_poll_interval: float | None = None,
*args,
**kwargs,
):
@@ -152,10 +153,10 @@ class SchedulerJob(BaseJob):
self.using_sqlite = sql_conn.startswith('sqlite')
self.using_mysql = sql_conn.startswith('mysql')
# Dag Processor agent - not used in Dag Processor standalone mode.
- self.processor_agent: Optional["DagFileProcessorAgent"] = None
+ self.processor_agent: DagFileProcessorAgent | None = None
self.dagbag = DagBag(dag_folder=self.subdir, read_dags_from_db=True, load_op_links=False)
- self._paused_dag_without_running_dagruns: Set = set()
+ self._paused_dag_without_running_dagruns: set = set()
def register_signals(self) -> None:
"""Register signals that stop child processes"""
@@ -163,7 +164,7 @@ class SchedulerJob(BaseJob):
signal.signal(signal.SIGTERM, self._exit_gracefully)
signal.signal(signal.SIGUSR2, self._debug_dump)
- def _exit_gracefully(self, signum: int, frame: Optional["FrameType"]) -> None:
+ def _exit_gracefully(self, signum: int, frame: FrameType | None) -> None:
"""Helper method to clean up processor_agent to avoid leaving orphan processes."""
if not _is_parent_process():
# Only the parent process should perform the cleanup.
@@ -174,7 +175,7 @@ class SchedulerJob(BaseJob):
self.processor_agent.end()
sys.exit(os.EX_OK)
- def _debug_dump(self, signum: int, frame: Optional["FrameType"]) -> None:
+ def _debug_dump(self, signum: int, frame: FrameType | None) -> None:
if not _is_parent_process():
# Only the parent process should perform the debug dump.
return
@@ -189,7 +190,7 @@ class SchedulerJob(BaseJob):
self.executor.debug_dump()
self.log.info("-" * 80)
- def is_alive(self, grace_multiplier: Optional[float] = None) -> bool:
+ def is_alive(self, grace_multiplier: float | None = None) -> bool:
"""
Is this SchedulerJob alive?
@@ -211,8 +212,8 @@ class SchedulerJob(BaseJob):
)
def __get_concurrency_maps(
- self, states: List[TaskInstanceState], session: Session
- ) -> Tuple[DefaultDict[str, int], DefaultDict[Tuple[str, str], int]]:
+ self, states: list[TaskInstanceState], session: Session
+ ) -> tuple[DefaultDict[str, int], DefaultDict[tuple[str, str], int]]:
"""
Get the concurrency maps.
@@ -221,20 +222,20 @@ class SchedulerJob(BaseJob):
a map from (dag_id, task_id) to # of task instances in the given state list
:rtype: tuple[dict[str, int], dict[tuple[str, str], int]]
"""
- ti_concurrency_query: List[Tuple[str, str, int]] = (
+ ti_concurrency_query: list[tuple[str, str, int]] = (
session.query(TI.task_id, TI.dag_id, func.count('*'))
.filter(TI.state.in_(states))
.group_by(TI.task_id, TI.dag_id)
).all()
dag_map: DefaultDict[str, int] = defaultdict(int)
- task_map: DefaultDict[Tuple[str, str], int] = defaultdict(int)
+ task_map: DefaultDict[tuple[str, str], int] = defaultdict(int)
for result in ti_concurrency_query:
task_id, dag_id, count = result
dag_map[dag_id] += count
task_map[(dag_id, task_id)] = count
return dag_map, task_map
- def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -> List[TI]:
+ def _executable_task_instances_to_queued(self, max_tis: int, session: Session) -> list[TI]:
"""
Finds TIs that are ready for execution with respect to pool limits,
dag max_active_tasks, executor state, and priority.
@@ -245,7 +246,7 @@ class SchedulerJob(BaseJob):
from airflow.models.pool import Pool
from airflow.utils.db import DBLocks
- executable_tis: List[TI] = []
+ executable_tis: list[TI] = []
if session.get_bind().dialect.name == "postgresql":
# Optimization: to avoid littering the DB errors of "ERROR: canceling statement due to lock
@@ -280,7 +281,7 @@ class SchedulerJob(BaseJob):
# dag_id to # of running tasks and (dag_id, task_id) to # of running tasks.
dag_active_tasks_map: DefaultDict[str, int]
- task_concurrency_map: DefaultDict[Tuple[str, str], int]
+ task_concurrency_map: DefaultDict[tuple[str, str], int]
dag_active_tasks_map, task_concurrency_map = self.__get_concurrency_maps(
states=list(EXECUTION_STATES), session=session
)
@@ -290,8 +291,8 @@ class SchedulerJob(BaseJob):
num_starving_tasks_total = 0
# dag and task ids that can't be queued because of concurrency limits
- starved_dags: Set[str] = set()
- starved_tasks: Set[Tuple[str, str]] = set()
+ starved_dags: set[str] = set()
+ starved_tasks: set[tuple[str, str]] = set()
pool_num_starving_tasks: DefaultDict[str, int] = defaultdict(int)
@@ -328,7 +329,7 @@ class SchedulerJob(BaseJob):
query = query.limit(max_tis)
- task_instances_to_examine: List[TI] = with_row_locks(
+ task_instances_to_examine: list[TI] = with_row_locks(
query,
of=TI,
session=session,
@@ -441,7 +442,7 @@ class SchedulerJob(BaseJob):
).update({TI.state: TaskInstanceState.FAILED}, synchronize_session='fetch')
continue
- task_concurrency_limit: Optional[int] = None
+ task_concurrency_limit: int | None = None
if serialized_dag.has_task(task_instance.task_id):
task_concurrency_limit = serialized_dag.get_task(
task_instance.task_id
@@ -513,7 +514,7 @@ class SchedulerJob(BaseJob):
make_transient(ti)
return executable_tis
- def _enqueue_task_instances_with_queued_state(self, task_instances: List[TI], session: Session) -> None:
+ def _enqueue_task_instances_with_queued_state(self, task_instances: list[TI], session: Session) -> None:
"""
Takes task_instances, which should have been set to queued, and enqueues them
with the executor.
@@ -574,9 +575,9 @@ class SchedulerJob(BaseJob):
"""Respond to executor events."""
if not self._standalone_dag_processor and not self.processor_agent:
raise ValueError("Processor agent is not started.")
- ti_primary_key_to_try_number_map: Dict[Tuple[str, str, str, int], int] = {}
+ ti_primary_key_to_try_number_map: dict[tuple[str, str, str, int], int] = {}
event_buffer = self.executor.get_event_buffer()
- tis_with_right_state: List[TaskInstanceKey] = []
+ tis_with_right_state: list[TaskInstanceKey] = []
# Report execution
for ti_key, value in event_buffer.items():
@@ -1075,7 +1076,7 @@ class SchedulerJob(BaseJob):
def _create_dag_runs_dataset_triggered(
self,
dag_models: Collection[DagModel],
- dataset_triggered_dag_info: Dict[str, Tuple[datetime, datetime]],
+ dataset_triggered_dag_info: dict[str, tuple[datetime, datetime]],
session: Session,
) -> None:
"""For DAGs that are triggered by datasets, create dag runs."""
@@ -1087,7 +1088,7 @@ class SchedulerJob(BaseJob):
dag_id: timezone.coerce_datetime(last_time)
for dag_id, (_, last_time) in dataset_triggered_dag_info.items()
}
- existing_dagruns: Set[Tuple[str, timezone.DateTime]] = set(
+ existing_dagruns: set[tuple[str, timezone.DateTime]] = set(
session.query(DagRun.dag_id, DagRun.execution_date).filter(
tuple_in_condition((DagRun.dag_id, DagRun.execution_date), exec_dates.items())
)
@@ -1230,14 +1231,14 @@ class SchedulerJob(BaseJob):
self,
dag_run: DagRun,
session: Session,
- ) -> Optional[DagCallbackRequest]:
+ ) -> DagCallbackRequest | None:
"""
Make scheduling decisions about an individual dag run
:param dag_run: The DagRun to schedule
:return: Callback that needs to be executed
"""
- callback: Optional[DagCallbackRequest] = None
+ callback: DagCallbackRequest | None = None
dag = dag_run.dag = self.dagbag.get_dag(dag_run.dag_id, session=session)
@@ -1314,9 +1315,7 @@ class SchedulerJob(BaseJob):
# Verify integrity also takes care of session.flush
dag_run.verify_integrity(session=session)
- def _send_dag_callbacks_to_processor(
- self, dag: DAG, callback: Optional[DagCallbackRequest] = None
- ) -> None:
+ def _send_dag_callbacks_to_processor(self, dag: DAG, callback: DagCallbackRequest | None = None) -> None:
self._send_sla_callbacks_to_processor(dag)
if callback:
self.executor.send_callback(callback)
diff --git a/airflow/jobs/triggerer_job.py b/airflow/jobs/triggerer_job.py
index ac7d22a6b1..49e8191363 100644
--- a/airflow/jobs/triggerer_job.py
+++ b/airflow/jobs/triggerer_job.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import asyncio
import os
@@ -22,7 +23,7 @@ import sys
import threading
import time
from collections import deque
-from typing import Deque, Dict, Set, Tuple, Type
+from typing import Deque
from sqlalchemy import func
@@ -195,22 +196,22 @@ class TriggerRunner(threading.Thread, LoggingMixin):
"""
# Maps trigger IDs to their running tasks and other info
- triggers: Dict[int, TriggerDetails]
+ triggers: dict[int, TriggerDetails]
# Cache for looking up triggers by classpath
- trigger_cache: Dict[str, Type[BaseTrigger]]
+ trigger_cache: dict[str, type[BaseTrigger]]
# Inbound queue of new triggers
- to_create: Deque[Tuple[int, BaseTrigger]]
+ to_create: Deque[tuple[int, BaseTrigger]]
# Inbound queue of deleted triggers
to_cancel: Deque[int]
# Outbound queue of events
- events: Deque[Tuple[int, TriggerEvent]]
+ events: Deque[tuple[int, TriggerEvent]]
# Outbound queue of failed triggers
- failed_triggers: Deque[Tuple[int, BaseException]]
+ failed_triggers: Deque[tuple[int, BaseException]]
# Should-we-stop flag
stop: bool = False
@@ -370,7 +371,7 @@ class TriggerRunner(threading.Thread, LoggingMixin):
# Main-thread sync API
- def update_triggers(self, requested_trigger_ids: Set[int]):
+ def update_triggers(self, requested_trigger_ids: set[int]):
"""
Called from the main thread to request that we update what
triggers we're running.
@@ -413,7 +414,7 @@ class TriggerRunner(threading.Thread, LoggingMixin):
for old_id in cancel_trigger_ids:
self.to_cancel.append(old_id)
- def get_trigger_by_classpath(self, classpath: str) -> Type[BaseTrigger]:
+ def get_trigger_by_classpath(self, classpath: str) -> type[BaseTrigger]:
"""
Gets a trigger class by its classpath ("path.to.module.classname")
diff --git a/airflow/kubernetes/k8s_model.py b/airflow/kubernetes/k8s_model.py
index 01e294672a..123294e0bd 100644
--- a/airflow/kubernetes/k8s_model.py
+++ b/airflow/kubernetes/k8s_model.py
@@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.
"""Classes for interacting with Kubernetes API."""
+from __future__ import annotations
+
from abc import ABC, abstractmethod
from functools import reduce
-from typing import List, Optional
from kubernetes.client import models as k8s
@@ -42,7 +43,7 @@ class K8SModel(ABC):
"""
-def append_to_pod(pod: k8s.V1Pod, k8s_objects: Optional[List[K8SModel]]):
+def append_to_pod(pod: k8s.V1Pod, k8s_objects: list[K8SModel] | None):
"""
:param pod: A pod to attach a list of Kubernetes objects to
:param k8s_objects: a potential None list of K8SModels
diff --git a/airflow/kubernetes/kube_client.py b/airflow/kubernetes/kube_client.py
index c42ef6191f..0b8c095f1a 100644
--- a/airflow/kubernetes/kube_client.py
+++ b/airflow/kubernetes/kube_client.py
@@ -15,8 +15,9 @@
# specific language governing permissions and limitations
# under the License.
"""Client for kubernetes communication"""
+from __future__ import annotations
+
import logging
-from typing import Optional
from airflow.configuration import conf
@@ -85,8 +86,8 @@ def _enable_tcp_keepalive() -> None:
def get_kube_client(
in_cluster: bool = conf.getboolean('kubernetes', 'in_cluster'),
- cluster_context: Optional[str] = None,
- config_file: Optional[str] = None,
+ cluster_context: str | None = None,
+ config_file: str | None = None,
) -> client.CoreV1Api:
"""
Retrieves Kubernetes client
diff --git a/airflow/kubernetes/kube_config.py b/airflow/kubernetes/kube_config.py
index 2b55021454..322a6a9a77 100644
--- a/airflow/kubernetes/kube_config.py
+++ b/airflow/kubernetes/kube_config.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from airflow.configuration import conf
from airflow.exceptions import AirflowConfigException
diff --git a/airflow/kubernetes/kubernetes_helper_functions.py b/airflow/kubernetes/kubernetes_helper_functions.py
index 1068a0521b..f1e1cd7f46 100644
--- a/airflow/kubernetes/kubernetes_helper_functions.py
+++ b/airflow/kubernetes/kubernetes_helper_functions.py
@@ -14,9 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import logging
-from typing import Dict, Optional
import pendulum
from slugify import slugify
@@ -56,7 +56,7 @@ def create_pod_id(dag_id: str, task_id: str) -> str:
return safe_dag_id + safe_task_id
-def annotations_to_key(annotations: Dict[str, str]) -> Optional[TaskInstanceKey]:
+def annotations_to_key(annotations: dict[str, str]) -> TaskInstanceKey | None:
"""Build a TaskInstanceKey based on pod annotations"""
log.debug("Creating task key for annotations %s", annotations)
dag_id = annotations['dag_id']
diff --git a/airflow/kubernetes/pod.py b/airflow/kubernetes/pod.py
index b3540a494c..5b946b2e3a 100644
--- a/airflow/kubernetes/pod.py
+++ b/airflow/kubernetes/pod.py
@@ -19,12 +19,14 @@
This module is deprecated.
Please use :mod:`kubernetes.client.models` for `V1ResourceRequirements` and `Port`.
"""
-# flake8: noqa
+from __future__ import annotations
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
+# flake8: noqa
+
with warnings.catch_warnings():
warnings.simplefilter("ignore", RemovedInAirflow3Warning)
from airflow.providers.cncf.kubernetes.backcompat.pod import Port, Resources # noqa: autoflake
diff --git a/airflow/kubernetes/pod_generator.py b/airflow/kubernetes/pod_generator.py
index 17f5e82503..b120f25908 100644
--- a/airflow/kubernetes/pod_generator.py
+++ b/airflow/kubernetes/pod_generator.py
@@ -20,6 +20,8 @@ API and outputs a kubernetes.client.models.V1Pod.
The advantage being that the full Kubernetes API
is supported and no serialization need be written.
"""
+from __future__ import annotations
+
import copy
import datetime
import hashlib
@@ -28,7 +30,6 @@ import re
import uuid
import warnings
from functools import reduce
-from typing import List, Optional, Union
from dateutil import parser
from kubernetes.client import models as k8s
@@ -100,8 +101,8 @@ class PodGenerator:
def __init__(
self,
- pod: Optional[k8s.V1Pod] = None,
- pod_template_file: Optional[str] = None,
+ pod: k8s.V1Pod | None = None,
+ pod_template_file: str | None = None,
extract_xcom: bool = True,
):
if not pod_template_file and not pod:
@@ -147,7 +148,7 @@ class PodGenerator:
return pod_cp
@staticmethod
- def from_obj(obj) -> Optional[Union[dict, k8s.V1Pod]]:
+ def from_obj(obj) -> dict | k8s.V1Pod | None:
"""Converts to pod from obj"""
if obj is None:
return None
@@ -181,7 +182,7 @@ class PodGenerator:
)
@staticmethod
- def from_legacy_obj(obj) -> Optional[k8s.V1Pod]:
+ def from_legacy_obj(obj) -> k8s.V1Pod | None:
"""Converts to pod from obj"""
if obj is None:
return None
@@ -218,7 +219,7 @@ class PodGenerator:
return PodGeneratorDeprecated(**namespaced).gen_pod()
@staticmethod
- def reconcile_pods(base_pod: k8s.V1Pod, client_pod: Optional[k8s.V1Pod]) -> k8s.V1Pod:
+ def reconcile_pods(base_pod: k8s.V1Pod, client_pod: k8s.V1Pod | None) -> k8s.V1Pod:
"""
:param base_pod: has the base attributes which are overwritten if they exist
in the client pod and remain if they do not exist in the client_pod
@@ -262,8 +263,8 @@ class PodGenerator:
@staticmethod
def reconcile_specs(
- base_spec: Optional[k8s.V1PodSpec], client_spec: Optional[k8s.V1PodSpec]
- ) -> Optional[k8s.V1PodSpec]:
+ base_spec: k8s.V1PodSpec | None, client_spec: k8s.V1PodSpec | None
+ ) -> k8s.V1PodSpec | None:
"""
:param base_spec: has the base attributes which are overwritten if they exist
in the client_spec and remain if they do not exist in the client_spec
@@ -286,8 +287,8 @@ class PodGenerator:
@staticmethod
def reconcile_containers(
- base_containers: List[k8s.V1Container], client_containers: List[k8s.V1Container]
- ) -> List[k8s.V1Container]:
+ base_containers: list[k8s.V1Container], client_containers: list[k8s.V1Container]
+ ) -> list[k8s.V1Container]:
"""
:param base_containers: has the base attributes which are overwritten if they exist
in the client_containers and remain if they do not exist in the client_containers
@@ -321,13 +322,13 @@ class PodGenerator:
pod_id: str,
try_number: int,
kube_image: str,
- date: Optional[datetime.datetime],
- args: List[str],
- pod_override_object: Optional[k8s.V1Pod],
+ date: datetime.datetime | None,
+ args: list[str],
+ pod_override_object: k8s.V1Pod | None,
base_worker_pod: k8s.V1Pod,
namespace: str,
scheduler_job_id: str,
- run_id: Optional[str] = None,
+ run_id: str | None = None,
map_index: int = -1,
) -> k8s.V1Pod:
"""
@@ -436,7 +437,7 @@ class PodGenerator:
return api_client._ApiClient__deserialize_model(pod_dict, k8s.V1Pod)
@staticmethod
- def make_unique_pod_id(pod_id: str) -> Optional[str]:
+ def make_unique_pod_id(pod_id: str) -> str | None:
r"""
Kubernetes pod names must consist of one or more lowercase
rfc1035/rfc1123 labels separated by '.' with a maximum length of 253
diff --git a/airflow/kubernetes/pod_generator_deprecated.py b/airflow/kubernetes/pod_generator_deprecated.py
index fcbb2bb402..f9fa278506 100644
--- a/airflow/kubernetes/pod_generator_deprecated.py
+++ b/airflow/kubernetes/pod_generator_deprecated.py
@@ -20,11 +20,12 @@ API and outputs a kubernetes.client.models.V1Pod.
The advantage being that the full Kubernetes API
is supported and no serialization need be written.
"""
+from __future__ import annotations
+
import copy
import hashlib
import re
import uuid
-from typing import Dict, List, Optional, Union
from kubernetes.client import models as k8s
@@ -117,33 +118,33 @@ class PodGenerator:
def __init__(
self,
- image: Optional[str] = None,
- name: Optional[str] = None,
- namespace: Optional[str] = None,
- volume_mounts: Optional[List[Union[k8s.V1VolumeMount, dict]]] = None,
- envs: Optional[Dict[str, str]] = None,
- cmds: Optional[List[str]] = None,
- args: Optional[List[str]] = None,
- labels: Optional[Dict[str, str]] = None,
- node_selectors: Optional[Dict[str, str]] = None,
- ports: Optional[List[Union[k8s.V1ContainerPort, dict]]] = None,
- volumes: Optional[List[Union[k8s.V1Volume, dict]]] = None,
- image_pull_policy: Optional[str] = None,
- restart_policy: Optional[str] = None,
- image_pull_secrets: Optional[str] = None,
- init_containers: Optional[List[k8s.V1Container]] = None,
- service_account_name: Optional[str] = None,
- resources: Optional[Union[k8s.V1ResourceRequirements, dict]] = None,
- annotations: Optional[Dict[str, str]] = None,
- affinity: Optional[dict] = None,
+ image: str | None = None,
+ name: str | None = None,
+ namespace: str | None = None,
+ volume_mounts: list[k8s.V1VolumeMount | dict] | None = None,
+ envs: dict[str, str] | None = None,
+ cmds: list[str] | None = None,
+ args: list[str] | None = None,
+ labels: dict[str, str] | None = None,
+ node_selectors: dict[str, str] | None = None,
+ ports: list[k8s.V1ContainerPort | dict] | None = None,
+ volumes: list[k8s.V1Volume | dict] | None = None,
+ image_pull_policy: str | None = None,
+ restart_policy: str | None = None,
+ image_pull_secrets: str | None = None,
+ init_containers: list[k8s.V1Container] | None = None,
+ service_account_name: str | None = None,
+ resources: k8s.V1ResourceRequirements | dict | None = None,
+ annotations: dict[str, str] | None = None,
+ affinity: dict | None = None,
hostnetwork: bool = False,
- tolerations: Optional[list] = None,
- security_context: Optional[Union[k8s.V1PodSecurityContext, dict]] = None,
- configmaps: Optional[List[str]] = None,
- dnspolicy: Optional[str] = None,
- schedulername: Optional[str] = None,
+ tolerations: list | None = None,
+ security_context: k8s.V1PodSecurityContext | dict | None = None,
+ configmaps: list[str] | None = None,
+ dnspolicy: str | None = None,
+ schedulername: str | None = None,
extract_xcom: bool = False,
- priority_class_name: Optional[str] = None,
+ priority_class_name: str | None = None,
):
self.pod = k8s.V1Pod()
@@ -240,7 +241,7 @@ class PodGenerator:
return pod_cp
@staticmethod
- def from_obj(obj) -> Optional[k8s.V1Pod]:
+ def from_obj(obj) -> k8s.V1Pod | None:
"""Converts to pod from obj"""
if obj is None:
return None
diff --git a/airflow/kubernetes/pod_launcher.py b/airflow/kubernetes/pod_launcher.py
index 0b9cbbe45a..bd52f49653 100644
--- a/airflow/kubernetes/pod_launcher.py
+++ b/airflow/kubernetes/pod_launcher.py
@@ -19,4 +19,6 @@
This module is deprecated.
Please use :mod:`kubernetes.client.models` for V1ResourceRequirements and Port.
"""
+from __future__ import annotations
+
from airflow.kubernetes.pod_launcher_deprecated import PodLauncher, PodStatus # noqa: autoflake
diff --git a/airflow/kubernetes/pod_launcher_deprecated.py b/airflow/kubernetes/pod_launcher_deprecated.py
index acff26b787..d67d14252e 100644
--- a/airflow/kubernetes/pod_launcher_deprecated.py
+++ b/airflow/kubernetes/pod_launcher_deprecated.py
@@ -15,12 +15,13 @@
# specific language governing permissions and limitations
# under the License.
"""Launches PODs"""
+from __future__ import annotations
+
import json
import math
import time
import warnings
from datetime import datetime as dt
-from typing import Optional, Tuple
import pendulum
import tenacity
@@ -69,7 +70,7 @@ class PodLauncher(LoggingMixin):
self,
kube_client: client.CoreV1Api = None,
in_cluster: bool = True,
- cluster_context: Optional[str] = None,
+ cluster_context: str | None = None,
extract_xcom: bool = False,
):
"""
@@ -134,7 +135,7 @@ class PodLauncher(LoggingMixin):
raise AirflowException("Pod took too long to start")
time.sleep(1)
- def monitor_pod(self, pod: V1Pod, get_logs: bool) -> Tuple[State, Optional[str]]:
+ def monitor_pod(self, pod: V1Pod, get_logs: bool) -> tuple[State, str | None]:
"""
Monitors a pod and returns the final state
@@ -175,7 +176,7 @@ class PodLauncher(LoggingMixin):
time.sleep(2)
return self._task_status(self.read_pod(pod)), result
- def parse_log_line(self, line: str) -> Tuple[Optional[str], str]:
+ def parse_log_line(self, line: str) -> tuple[str | None, str]:
"""
Parse K8s log line and returns the final state
@@ -222,9 +223,9 @@ class PodLauncher(LoggingMixin):
def read_pod_logs(
self,
pod: V1Pod,
- tail_lines: Optional[int] = None,
+ tail_lines: int | None = None,
timestamps: bool = False,
- since_seconds: Optional[int] = None,
+ since_seconds: int | None = None,
):
"""Reads log from the POD"""
additional_kwargs = {}
diff --git a/airflow/kubernetes/pod_runtime_info_env.py b/airflow/kubernetes/pod_runtime_info_env.py
index 1c81d4eb13..32e178263b 100644
--- a/airflow/kubernetes/pod_runtime_info_env.py
+++ b/airflow/kubernetes/pod_runtime_info_env.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""This module is deprecated. Please use :mod:`kubernetes.client.models.V1EnvVar`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/kubernetes/secret.py b/airflow/kubernetes/secret.py
index afb30916ff..1b9627c3fd 100644
--- a/airflow/kubernetes/secret.py
+++ b/airflow/kubernetes/secret.py
@@ -15,9 +15,10 @@
# specific language governing permissions and limitations
# under the License.
"""Classes for interacting with Kubernetes API"""
+from __future__ import annotations
+
import copy
import uuid
-from typing import Tuple
from kubernetes.client import models as k8s
@@ -75,7 +76,7 @@ class Secret(K8SModel):
"""Reads from environment to secret"""
return k8s.V1EnvFromSource(secret_ref=k8s.V1SecretEnvSource(name=self.secret))
- def to_volume_secret(self) -> Tuple[k8s.V1Volume, k8s.V1VolumeMount]:
+ def to_volume_secret(self) -> tuple[k8s.V1Volume, k8s.V1VolumeMount]:
"""Converts to volume secret"""
vol_id = f'secretvol{uuid.uuid4()}'
volume = k8s.V1Volume(name=vol_id, secret=k8s.V1SecretVolumeSource(secret_name=self.secret))
diff --git a/airflow/kubernetes/volume.py b/airflow/kubernetes/volume.py
index c48e0a72ce..ecb39e457f 100644
--- a/airflow/kubernetes/volume.py
+++ b/airflow/kubernetes/volume.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""This module is deprecated. Please use :mod:`kubernetes.client.models.V1Volume`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/kubernetes/volume_mount.py b/airflow/kubernetes/volume_mount.py
index 0b9543d1e0..e65351d85f 100644
--- a/airflow/kubernetes/volume_mount.py
+++ b/airflow/kubernetes/volume_mount.py
@@ -16,6 +16,8 @@
# specific language governing permissions and limitations
# under the License.
"""This module is deprecated. Please use :mod:`kubernetes.client.models.V1VolumeMount`."""
+from __future__ import annotations
+
import warnings
from airflow.exceptions import RemovedInAirflow3Warning
diff --git a/airflow/lineage/__init__.py b/airflow/lineage/__init__.py
index bf9d27772d..3c215c9159 100644
--- a/airflow/lineage/__init__.py
+++ b/airflow/lineage/__init__.py
@@ -16,10 +16,12 @@
# specific language governing permissions and limitations
# under the License.
"""Provides lineage support functions"""
+from __future__ import annotations
+
import itertools
import logging
from functools import wraps
-from typing import TYPE_CHECKING, Any, Callable, List, Optional, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast
import attr
@@ -38,7 +40,7 @@ AUTO = "auto"
log = logging.getLogger(__name__)
-def get_backend() -> Optional[LineageBackend]:
+def get_backend() -> LineageBackend | None:
"""Gets the lineage backend if defined in the configs"""
clazz = conf.getimport("lineage", "backend", fallback=None)
@@ -54,7 +56,7 @@ def get_backend() -> Optional[LineageBackend]:
return None
-def _render_object(obj: Any, context: "Context") -> dict:
+def _render_object(obj: Any, context: Context) -> dict:
return context['ti'].task.render_template(obj, context)
@@ -66,7 +68,7 @@ def _deserialize(serialized: dict):
return cls(**BaseSerialization.deserialize(serialized['__var']))
-def _serialize(objs: List[Any], source: str):
+def _serialize(objs: list[Any], source: str):
"""Serialize an attrs-decorated class to JSON"""
from airflow.serialization.serialized_objects import BaseSerialization
diff --git a/airflow/lineage/backend.py b/airflow/lineage/backend.py
index ca072f4341..ffb5d214c3 100644
--- a/airflow/lineage/backend.py
+++ b/airflow/lineage/backend.py
@@ -16,7 +16,9 @@
# specific language governing permissions and limitations
# under the License.
"""Sends lineage metadata to a backend"""
-from typing import TYPE_CHECKING, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
if TYPE_CHECKING:
from airflow.models.baseoperator import BaseOperator
@@ -27,10 +29,10 @@ class LineageBackend:
def send_lineage(
self,
- operator: 'BaseOperator',
- inlets: Optional[list] = None,
- outlets: Optional[list] = None,
- context: Optional[dict] = None,
+ operator: BaseOperator,
+ inlets: list | None = None,
+ outlets: list | None = None,
+ context: dict | None = None,
):
"""
Sends lineage metadata to a backend
diff --git a/airflow/lineage/entities.py b/airflow/lineage/entities.py
index cb0fede80d..eaa630ad1a 100644
--- a/airflow/lineage/entities.py
+++ b/airflow/lineage/entities.py
@@ -15,12 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Defines the base entities that can be used for providing lineage
information.
"""
-from typing import Any, ClassVar, Dict, List, Optional
+from __future__ import annotations
+
+from typing import Any, ClassVar
import attr
@@ -32,7 +33,7 @@ class File:
template_fields: ClassVar = ("url",)
url: str = attr.ib()
- type_hint: Optional[str] = None
+ type_hint: str | None = None
@attr.s(auto_attribs=True, kw_only=True)
@@ -40,8 +41,8 @@ class User:
"""User entity. Identifies a user"""
email: str = attr.ib()
- first_name: Optional[str] = None
- last_name: Optional[str] = None
+ first_name: str | None = None
+ last_name: str | None = None
template_fields: ClassVar = ("email", "first_name", "last_name")
@@ -60,9 +61,9 @@ class Column:
"""Column of a Table"""
name: str = attr.ib()
- description: Optional[str] = None
+ description: str | None = None
data_type: str = attr.ib()
- tags: List[Tag] = []
+ tags: list[Tag] = []
template_fields: ClassVar = ("name", "description", "data_type", "tags")
@@ -72,7 +73,7 @@ class Column:
# `attr.converters.default_if_none(default=False)`
-def default_if_none(arg: Optional[bool]) -> bool:
+def default_if_none(arg: bool | None) -> bool:
return arg or False
@@ -83,12 +84,12 @@ class Table:
database: str = attr.ib()
cluster: str = attr.ib()
name: str = attr.ib()
- tags: List[Tag] = []
- description: Optional[str] = None
- columns: List[Column] = []
- owners: List[User] = []
- extra: Dict[str, Any] = {}
- type_hint: Optional[str] = None
+ tags: list[Tag] = []
+ description: str | None = None
+ columns: list[Column] = []
+ owners: list[User] = []
+ extra: dict[str, Any] = {}
+ type_hint: str | None = None
template_fields: ClassVar = (
"database",
diff --git a/airflow/listeners/__init__.py b/airflow/listeners/__init__.py
index d1df70bda1..87840b50e2 100644
--- a/airflow/listeners/__init__.py
+++ b/airflow/listeners/__init__.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
from pluggy import HookimplMarker
hookimpl = HookimplMarker("airflow")
diff --git a/airflow/listeners/events.py b/airflow/listeners/events.py
index d5af64710a..2b98224a8c 100644
--- a/airflow/listeners/events.py
+++ b/airflow/listeners/events.py
@@ -14,6 +14,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import logging
from sqlalchemy import event
diff --git a/airflow/listeners/listener.py b/airflow/listeners/listener.py
index 3c4d052399..754bc1af0b 100644
--- a/airflow/listeners/listener.py
+++ b/airflow/listeners/listener.py
@@ -15,6 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import logging
from types import ModuleType
from typing import TYPE_CHECKING
@@ -46,7 +48,7 @@ class ListenerManager:
return len(self.pm.get_plugins()) > 0
@property
- def hook(self) -> "_HookRelay":
+ def hook(self) -> _HookRelay:
"""Returns hook, on which plugin methods specified in spec can be called."""
return self.pm.hook
diff --git a/airflow/listeners/spec.py b/airflow/listeners/spec.py
index fbaf63e89a..78de8a5f62 100644
--- a/airflow/listeners/spec.py
+++ b/airflow/listeners/spec.py
@@ -15,7 +15,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-from typing import TYPE_CHECKING, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
from pluggy import HookspecMarker
@@ -30,20 +32,20 @@ hookspec = HookspecMarker("airflow")
@hookspec
def on_task_instance_running(
- previous_state: "TaskInstanceState", task_instance: "TaskInstance", session: Optional["Session"]
+ previous_state: TaskInstanceState, task_instance: TaskInstance, session: Session | None
):
"""Called when task state changes to RUNNING. Previous_state can be State.NONE."""
@hookspec
def on_task_instance_success(
- previous_state: "TaskInstanceState", task_instance: "TaskInstance", session: Optional["Session"]
+ previous_state: TaskInstanceState, task_instance: TaskInstance, session: Session | None
):
"""Called when task state changes to SUCCESS. Previous_state can be State.NONE."""
@hookspec
def on_task_instance_failed(
- previous_state: "TaskInstanceState", task_instance: "TaskInstance", session: Optional["Session"]
+ previous_state: TaskInstanceState, task_instance: TaskInstance, session: Session | None
):
"""Called when task state changes to FAIL. Previous_state can be State.NONE."""
diff --git a/airflow/logging_config.py b/airflow/logging_config.py
index 645e53eb3e..b0b11cb7b7 100644
--- a/airflow/logging_config.py
+++ b/airflow/logging_config.py
@@ -15,7 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
+
import logging
import warnings
from logging.config import dictConfig
diff --git a/airflow/macros/__init__.py b/airflow/macros/__init__.py
index e1f27411e5..2478a1ee80 100644
--- a/airflow/macros/__init__.py
+++ b/airflow/macros/__init__.py
@@ -15,11 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
+
import time # noqa
import uuid # noqa
from datetime import datetime, timedelta
from random import random # noqa
-from typing import Any, Optional
+from typing import Any
import dateutil # noqa
from pendulum import DateTime
@@ -62,7 +64,7 @@ def ds_format(ds: str, input_format: str, output_format: str) -> str:
return datetime.strptime(str(ds), input_format).strftime(output_format)
-def datetime_diff_for_humans(dt: Any, since: Optional[DateTime] = None) -> str:
+def datetime_diff_for_humans(dt: Any, since: DateTime | None = None) -> str:
"""
Return a human-readable/approximate difference between two datetimes, or
one and now.
diff --git a/airflow/macros/hive.py b/airflow/macros/hive.py
index fe5685fcf2..c170217fc8 100644
--- a/airflow/macros/hive.py
+++ b/airflow/macros/hive.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
import datetime
diff --git a/airflow/migrations/db_types.py b/airflow/migrations/db_types.py
index 9b8f3e974f..fb2d6c5edb 100644
--- a/airflow/migrations/db_types.py
+++ b/airflow/migrations/db_types.py
@@ -15,7 +15,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import context
from lazy_object_proxy import Proxy
diff --git a/airflow/migrations/env.py b/airflow/migrations/env.py
index 9faa8821fe..6474f0799c 100644
--- a/airflow/migrations/env.py
+++ b/airflow/migrations/env.py
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from logging.config import fileConfig
diff --git a/airflow/migrations/utils.py b/airflow/migrations/utils.py
index 5737fa9507..295f00ee6a 100644
--- a/airflow/migrations/utils.py
+++ b/airflow/migrations/utils.py
@@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from __future__ import annotations
from collections import defaultdict
from contextlib import contextmanager
diff --git a/airflow/migrations/versions/0001_1_5_0_current_schema.py b/airflow/migrations/versions/0001_1_5_0_current_schema.py
index c700df8c47..73d6b8ed1d 100644
--- a/airflow/migrations/versions/0001_1_5_0_current_schema.py
+++ b/airflow/migrations/versions/0001_1_5_0_current_schema.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""current schema
Revision ID: e3a246e0dc1
@@ -23,6 +22,7 @@ Revises:
Create Date: 2015-08-18 16:35:00.883495
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0002_1_5_0_create_is_encrypted.py b/airflow/migrations/versions/0002_1_5_0_create_is_encrypted.py
index 2d13be3d2e..53d39039ab 100644
--- a/airflow/migrations/versions/0002_1_5_0_create_is_encrypted.py
+++ b/airflow/migrations/versions/0002_1_5_0_create_is_encrypted.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``is_encrypted`` column in ``connection`` table
Revision ID: 1507a7289a2f
@@ -23,6 +22,8 @@ Revises: e3a246e0dc1
Create Date: 2015-08-18 18:57:51.927315
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0003_1_5_0_for_compatibility.py b/airflow/migrations/versions/0003_1_5_0_for_compatibility.py
index cd45e1fa74..fcd9a1fcf1 100644
--- a/airflow/migrations/versions/0003_1_5_0_for_compatibility.py
+++ b/airflow/migrations/versions/0003_1_5_0_for_compatibility.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Maintain history for compatibility with earlier migrations
Revision ID: 13eb55f81627
@@ -23,6 +22,7 @@ Revises: 1507a7289a2f
Create Date: 2015-08-23 05:12:49.732174
"""
+from __future__ import annotations
# revision identifiers, used by Alembic.
revision = '13eb55f81627'
diff --git a/airflow/migrations/versions/0004_1_5_0_more_logging_into_task_isntance.py b/airflow/migrations/versions/0004_1_5_0_more_logging_into_task_isntance.py
index 2eb793adda..c5ba20faf1 100644
--- a/airflow/migrations/versions/0004_1_5_0_more_logging_into_task_isntance.py
+++ b/airflow/migrations/versions/0004_1_5_0_more_logging_into_task_isntance.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``operator`` and ``queued_dttm`` to ``task_instance`` table
Revision ID: 338e90f54d61
@@ -23,6 +22,8 @@ Revises: 13eb55f81627
Create Date: 2015-08-25 06:09:20.460147
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0005_1_5_2_job_id_indices.py b/airflow/migrations/versions/0005_1_5_2_job_id_indices.py
index e6ba4fd226..9977d925bd 100644
--- a/airflow/migrations/versions/0005_1_5_2_job_id_indices.py
+++ b/airflow/migrations/versions/0005_1_5_2_job_id_indices.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add indices in ``job`` table
Revision ID: 52d714495f0
@@ -23,6 +22,8 @@ Revises: 338e90f54d61
Create Date: 2015-10-20 03:17:01.962542
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0006_1_6_0_adding_extra_to_log.py b/airflow/migrations/versions/0006_1_6_0_adding_extra_to_log.py
index 5bc28ad372..87ac0a3c11 100644
--- a/airflow/migrations/versions/0006_1_6_0_adding_extra_to_log.py
+++ b/airflow/migrations/versions/0006_1_6_0_adding_extra_to_log.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Adding ``extra`` column to ``Log`` table
Revision ID: 502898887f84
@@ -23,6 +22,8 @@ Revises: 52d714495f0
Create Date: 2015-11-03 22:50:49.794097
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0007_1_6_0_add_dagrun.py b/airflow/migrations/versions/0007_1_6_0_add_dagrun.py
index 66a65fd3c3..300d62f133 100644
--- a/airflow/migrations/versions/0007_1_6_0_add_dagrun.py
+++ b/airflow/migrations/versions/0007_1_6_0_add_dagrun.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``dag_run`` table
Revision ID: 1b38cef5b76e
@@ -23,6 +22,7 @@ Revises: 52d714495f0
Create Date: 2015-10-27 08:31:48.475140
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0008_1_6_0_task_duration.py b/airflow/migrations/versions/0008_1_6_0_task_duration.py
index 9fa217d1aa..fb39ffe69d 100644
--- a/airflow/migrations/versions/0008_1_6_0_task_duration.py
+++ b/airflow/migrations/versions/0008_1_6_0_task_duration.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Change ``task_instance.task_duration`` type to ``FLOAT``
Revision ID: 2e541a1dcfed
@@ -23,6 +22,7 @@ Revises: 1b38cef5b76e
Create Date: 2015-10-28 20:38:41.266143
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0009_1_6_0_dagrun_config.py b/airflow/migrations/versions/0009_1_6_0_dagrun_config.py
index ae77907137..4c1ced0b9d 100644
--- a/airflow/migrations/versions/0009_1_6_0_dagrun_config.py
+++ b/airflow/migrations/versions/0009_1_6_0_dagrun_config.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``conf`` column in ``dag_run`` table
Revision ID: 40e67319e3a9
diff --git a/airflow/migrations/versions/0010_1_6_2_add_password_column_to_user.py b/airflow/migrations/versions/0010_1_6_2_add_password_column_to_user.py
index d1004ba52a..3578fd1e7b 100644
--- a/airflow/migrations/versions/0010_1_6_2_add_password_column_to_user.py
+++ b/airflow/migrations/versions/0010_1_6_2_add_password_column_to_user.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``password`` column to ``user`` table
Revision ID: 561833c1c74b
@@ -23,6 +22,8 @@ Revises: 40e67319e3a9
Create Date: 2015-11-30 06:51:25.872557
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0011_1_6_2_dagrun_start_end.py b/airflow/migrations/versions/0011_1_6_2_dagrun_start_end.py
index ae471443b0..2e49cb365f 100644
--- a/airflow/migrations/versions/0011_1_6_2_dagrun_start_end.py
+++ b/airflow/migrations/versions/0011_1_6_2_dagrun_start_end.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``start_date`` and ``end_date`` in ``dag_run`` table
Revision ID: 4446e08588
@@ -23,6 +22,7 @@ Revises: 561833c1c74b
Create Date: 2015-12-10 11:26:18.439223
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0012_1_7_0_add_notification_sent_column_to_sla_miss.py b/airflow/migrations/versions/0012_1_7_0_add_notification_sent_column_to_sla_miss.py
index 0e3548257c..f4883bdd3c 100644
--- a/airflow/migrations/versions/0012_1_7_0_add_notification_sent_column_to_sla_miss.py
+++ b/airflow/migrations/versions/0012_1_7_0_add_notification_sent_column_to_sla_miss.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``notification_sent`` column to ``sla_miss`` table
Revision ID: bbc73705a13e
@@ -23,6 +22,8 @@ Revises: 4446e08588
Create Date: 2016-01-14 18:05:54.871682
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0013_1_7_0_add_a_column_to_track_the_encryption_.py b/airflow/migrations/versions/0013_1_7_0_add_a_column_to_track_the_encryption_.py
index 4e0d53e062..f5ea000be6 100644
--- a/airflow/migrations/versions/0013_1_7_0_add_a_column_to_track_the_encryption_.py
+++ b/airflow/migrations/versions/0013_1_7_0_add_a_column_to_track_the_encryption_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add a column to track the encryption state of the 'Extra' field in connection
Revision ID: bba5a7cfc896
@@ -23,6 +22,7 @@ Revises: bbc73705a13e
Create Date: 2016-01-29 15:10:32.656425
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0014_1_7_0_add_is_encrypted_column_to_variable_.py b/airflow/migrations/versions/0014_1_7_0_add_is_encrypted_column_to_variable_.py
index e72260ec20..61e660361b 100644
--- a/airflow/migrations/versions/0014_1_7_0_add_is_encrypted_column_to_variable_.py
+++ b/airflow/migrations/versions/0014_1_7_0_add_is_encrypted_column_to_variable_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``is_encrypted`` column to variable table
Revision ID: 1968acfc09e3
@@ -23,6 +22,8 @@ Revises: bba5a7cfc896
Create Date: 2016-02-02 17:20:55.692295
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0015_1_7_1_rename_user_table.py b/airflow/migrations/versions/0015_1_7_1_rename_user_table.py
index 7259eb5c18..2d736ab6b4 100644
--- a/airflow/migrations/versions/0015_1_7_1_rename_user_table.py
+++ b/airflow/migrations/versions/0015_1_7_1_rename_user_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Rename user table
Revision ID: 2e82aab8ef20
@@ -23,6 +22,8 @@ Revises: 1968acfc09e3
Create Date: 2016-04-02 19:28:15.211915
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0016_1_7_1_add_ti_state_index.py b/airflow/migrations/versions/0016_1_7_1_add_ti_state_index.py
index eae9bfd064..1aa3d79ea8 100644
--- a/airflow/migrations/versions/0016_1_7_1_add_ti_state_index.py
+++ b/airflow/migrations/versions/0016_1_7_1_add_ti_state_index.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add TI state index
Revision ID: 211e584da130
@@ -23,6 +22,8 @@ Revises: 2e82aab8ef20
Create Date: 2016-06-30 10:54:24.323588
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0017_1_7_1_add_task_fails_journal_table.py b/airflow/migrations/versions/0017_1_7_1_add_task_fails_journal_table.py
index 679efb70a1..45eba54dd2 100644
--- a/airflow/migrations/versions/0017_1_7_1_add_task_fails_journal_table.py
+++ b/airflow/migrations/versions/0017_1_7_1_add_task_fails_journal_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``task_fail`` table
Revision ID: 64de9cddf6c9
@@ -23,6 +22,8 @@ Revises: 211e584da130
Create Date: 2016-08-03 14:02:59.203021
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0018_1_7_1_add_dag_stats_table.py b/airflow/migrations/versions/0018_1_7_1_add_dag_stats_table.py
index 9cb37d2a41..57229c624f 100644
--- a/airflow/migrations/versions/0018_1_7_1_add_dag_stats_table.py
+++ b/airflow/migrations/versions/0018_1_7_1_add_dag_stats_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``dag_stats`` table
Revision ID: f2ca10b85618
@@ -23,6 +22,8 @@ Revises: 64de9cddf6c9
Create Date: 2016-07-20 15:08:28.247537
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0019_1_7_1_add_fractional_seconds_to_mysql_tables.py b/airflow/migrations/versions/0019_1_7_1_add_fractional_seconds_to_mysql_tables.py
index 9cee531e74..64173c94ac 100644
--- a/airflow/migrations/versions/0019_1_7_1_add_fractional_seconds_to_mysql_tables.py
+++ b/airflow/migrations/versions/0019_1_7_1_add_fractional_seconds_to_mysql_tables.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add fractional seconds to MySQL tables
Revision ID: 4addfa1236f1
@@ -23,6 +22,7 @@ Revises: f2ca10b85618
Create Date: 2016-09-11 13:39:18.592072
"""
+from __future__ import annotations
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0020_1_7_1_xcom_dag_task_indices.py b/airflow/migrations/versions/0020_1_7_1_xcom_dag_task_indices.py
index ea2aba8e32..fd82a9bd0e 100644
--- a/airflow/migrations/versions/0020_1_7_1_xcom_dag_task_indices.py
+++ b/airflow/migrations/versions/0020_1_7_1_xcom_dag_task_indices.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add indices on ``xcom`` table
Revision ID: 8504051e801b
@@ -23,6 +22,7 @@ Revises: 4addfa1236f1
Create Date: 2016-11-29 08:13:03.253312
"""
+from __future__ import annotations
from alembic import op
diff --git a/airflow/migrations/versions/0021_1_7_1_add_pid_field_to_taskinstance.py b/airflow/migrations/versions/0021_1_7_1_add_pid_field_to_taskinstance.py
index cea1d1010a..945159878e 100644
--- a/airflow/migrations/versions/0021_1_7_1_add_pid_field_to_taskinstance.py
+++ b/airflow/migrations/versions/0021_1_7_1_add_pid_field_to_taskinstance.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``pid`` field to ``TaskInstance``
Revision ID: 5e7d17757c7a
@@ -23,6 +22,7 @@ Revises: 8504051e801b
Create Date: 2016-12-07 15:51:37.119478
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0022_1_7_1_add_dag_id_state_index_on_dag_run_table.py b/airflow/migrations/versions/0022_1_7_1_add_dag_id_state_index_on_dag_run_table.py
index a7acdb2e7b..8d67b76acc 100644
--- a/airflow/migrations/versions/0022_1_7_1_add_dag_id_state_index_on_dag_run_table.py
+++ b/airflow/migrations/versions/0022_1_7_1_add_dag_id_state_index_on_dag_run_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``dag_id``/``state`` index on ``dag_run`` table
Revision ID: 127d2bf2dfa7
@@ -23,6 +22,8 @@ Revises: 5e7d17757c7a
Create Date: 2017-01-25 11:43:51.635667
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0023_1_8_2_add_max_tries_column_to_task_instance.py b/airflow/migrations/versions/0023_1_8_2_add_max_tries_column_to_task_instance.py
index 7685b77afd..3e9cee0a3b 100644
--- a/airflow/migrations/versions/0023_1_8_2_add_max_tries_column_to_task_instance.py
+++ b/airflow/migrations/versions/0023_1_8_2_add_max_tries_column_to_task_instance.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``max_tries`` column to ``task_instance``
Revision ID: cc1e65623dc7
@@ -22,6 +21,7 @@ Revises: 127d2bf2dfa7
Create Date: 2017-06-19 16:53:12.851141
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0024_1_8_2_make_xcom_value_column_a_large_binary.py b/airflow/migrations/versions/0024_1_8_2_make_xcom_value_column_a_large_binary.py
index 00a21bf543..57dd514923 100644
--- a/airflow/migrations/versions/0024_1_8_2_make_xcom_value_column_a_large_binary.py
+++ b/airflow/migrations/versions/0024_1_8_2_make_xcom_value_column_a_large_binary.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Make xcom value column a large binary
Revision ID: bdaa763e6c56
@@ -23,6 +22,8 @@ Revises: cc1e65623dc7
Create Date: 2017-08-14 16:06:31.568971
"""
+from __future__ import annotations
+
import dill
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0025_1_8_2_add_ti_job_id_index.py b/airflow/migrations/versions/0025_1_8_2_add_ti_job_id_index.py
index 364e57d503..8e3888b707 100644
--- a/airflow/migrations/versions/0025_1_8_2_add_ti_job_id_index.py
+++ b/airflow/migrations/versions/0025_1_8_2_add_ti_job_id_index.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Create index on ``job_id`` column in ``task_instance`` table
Revision ID: 947454bf1dff
@@ -23,6 +22,8 @@ Revises: bdaa763e6c56
Create Date: 2017-08-15 15:12:13.845074
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0026_1_8_2_increase_text_size_for_mysql.py b/airflow/migrations/versions/0026_1_8_2_increase_text_size_for_mysql.py
index 41450aec0f..d34f44a6a7 100644
--- a/airflow/migrations/versions/0026_1_8_2_increase_text_size_for_mysql.py
+++ b/airflow/migrations/versions/0026_1_8_2_increase_text_size_for_mysql.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Increase text size for MySQL (not relevant for other DBs' text types)
Revision ID: d2ae31099d61
@@ -23,6 +22,8 @@ Revises: 947454bf1dff
Create Date: 2017-08-18 17:07:16.686130
"""
+from __future__ import annotations
+
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0027_1_10_0_add_time_zone_awareness.py b/airflow/migrations/versions/0027_1_10_0_add_time_zone_awareness.py
index 8945ff06c9..23e1779bad 100644
--- a/airflow/migrations/versions/0027_1_10_0_add_time_zone_awareness.py
+++ b/airflow/migrations/versions/0027_1_10_0_add_time_zone_awareness.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add time zone awareness
Revision ID: 0e2a74e0fc9f
@@ -22,6 +21,7 @@ Revises: d2ae31099d61
Create Date: 2017-11-10 22:22:31.326152
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0028_1_10_0_add_kubernetes_resource_checkpointing.py b/airflow/migrations/versions/0028_1_10_0_add_kubernetes_resource_checkpointing.py
index c37831ac05..0a345aa91b 100644
--- a/airflow/migrations/versions/0028_1_10_0_add_kubernetes_resource_checkpointing.py
+++ b/airflow/migrations/versions/0028_1_10_0_add_kubernetes_resource_checkpointing.py
@@ -14,8 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
"""Add Kubernetes resource check-pointing
Revision ID: 33ae817a1ff4
@@ -23,6 +21,8 @@ Revises: 947454bf1dff
Create Date: 2017-09-11 15:26:47.598494
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0029_1_10_0_add_executor_config_to_task_instance.py b/airflow/migrations/versions/0029_1_10_0_add_executor_config_to_task_instance.py
index 67bda056ec..8d5ee9fa30 100644
--- a/airflow/migrations/versions/0029_1_10_0_add_executor_config_to_task_instance.py
+++ b/airflow/migrations/versions/0029_1_10_0_add_executor_config_to_task_instance.py
@@ -14,8 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
"""Add ``executor_config`` column to ``task_instance`` table
Revision ID: 33ae817a1ff4
@@ -23,6 +21,7 @@ Revises: 947454bf1dff
Create Date: 2017-09-11 15:26:47.598494
"""
+from __future__ import annotations
import dill
import sqlalchemy as sa
diff --git a/airflow/migrations/versions/0030_1_10_0_add_kubernetes_scheduler_uniqueness.py b/airflow/migrations/versions/0030_1_10_0_add_kubernetes_scheduler_uniqueness.py
index 4dd1864580..c19a3d7da1 100644
--- a/airflow/migrations/versions/0030_1_10_0_add_kubernetes_scheduler_uniqueness.py
+++ b/airflow/migrations/versions/0030_1_10_0_add_kubernetes_scheduler_uniqueness.py
@@ -14,8 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
"""Add kubernetes scheduler uniqueness
Revision ID: 86770d1215c0
@@ -23,6 +21,8 @@ Revises: 27c6a30d7c24
Create Date: 2018-04-03 15:31:20.814328
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0031_1_10_0_merge_heads.py b/airflow/migrations/versions/0031_1_10_0_merge_heads.py
index 33691b7b04..7edde0a76c 100644
--- a/airflow/migrations/versions/0031_1_10_0_merge_heads.py
+++ b/airflow/migrations/versions/0031_1_10_0_merge_heads.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Merge migrations Heads
Revision ID: 05f30312d566
@@ -22,6 +21,7 @@ Revises: 86770d1215c0, 0e2a74e0fc9f
Create Date: 2018-06-17 10:47:23.339972
"""
+from __future__ import annotations
# revision identifiers, used by Alembic.
revision = '05f30312d566'
diff --git a/airflow/migrations/versions/0032_1_10_0_fix_mysql_not_null_constraint.py b/airflow/migrations/versions/0032_1_10_0_fix_mysql_not_null_constraint.py
index 765d43762e..8891911d75 100644
--- a/airflow/migrations/versions/0032_1_10_0_fix_mysql_not_null_constraint.py
+++ b/airflow/migrations/versions/0032_1_10_0_fix_mysql_not_null_constraint.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Fix MySQL not null constraint
Revision ID: f23433877c24
@@ -22,6 +21,8 @@ Revises: 05f30312d566
Create Date: 2018-06-17 10:16:31.412131
"""
+from __future__ import annotations
+
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0033_1_10_0_fix_sqlite_foreign_key.py b/airflow/migrations/versions/0033_1_10_0_fix_sqlite_foreign_key.py
index abf289572f..f48439742e 100644
--- a/airflow/migrations/versions/0033_1_10_0_fix_sqlite_foreign_key.py
+++ b/airflow/migrations/versions/0033_1_10_0_fix_sqlite_foreign_key.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Fix Sqlite foreign key
Revision ID: 856955da8476
@@ -22,6 +21,7 @@ Revises: f23433877c24
Create Date: 2018-06-17 15:54:53.844230
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0034_1_10_0_index_taskfail.py b/airflow/migrations/versions/0034_1_10_0_index_taskfail.py
index 1323e77f7f..131dd3c724 100644
--- a/airflow/migrations/versions/0034_1_10_0_index_taskfail.py
+++ b/airflow/migrations/versions/0034_1_10_0_index_taskfail.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Create index on ``task_fail`` table
Revision ID: 9635ae0956e7
@@ -22,6 +21,8 @@ Revises: 856955da8476
Create Date: 2018-06-17 21:40:01.963540
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0035_1_10_2_add_idx_log_dag.py b/airflow/migrations/versions/0035_1_10_2_add_idx_log_dag.py
index 76fcfd375b..b0d38603ab 100644
--- a/airflow/migrations/versions/0035_1_10_2_add_idx_log_dag.py
+++ b/airflow/migrations/versions/0035_1_10_2_add_idx_log_dag.py
@@ -22,6 +22,8 @@ Revises: 9635ae0956e7
Create Date: 2018-08-07 06:41:41.028249
"""
+from __future__ import annotations
+
from alembic import op
# revision identifiers, used by Alembic.
diff --git a/airflow/migrations/versions/0036_1_10_2_add_index_to_taskinstance.py b/airflow/migrations/versions/0036_1_10_2_add_index_to_taskinstance.py
index a52663dcc9..4c3a5d0265 100644
--- a/airflow/migrations/versions/0036_1_10_2_add_index_to_taskinstance.py
+++ b/airflow/migrations/versions/0036_1_10_2_add_index_to_taskinstance.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add index to ``task_instance`` table
Revision ID: bf00311e1990
@@ -23,6 +22,7 @@ Revises: dd25f486b8ea
Create Date: 2018-09-12 09:53:52.007433
"""
+from __future__ import annotations
from alembic import op
diff --git a/airflow/migrations/versions/0037_1_10_2_add_task_reschedule_table.py b/airflow/migrations/versions/0037_1_10_2_add_task_reschedule_table.py
index a26204fff1..8baf114f00 100644
--- a/airflow/migrations/versions/0037_1_10_2_add_task_reschedule_table.py
+++ b/airflow/migrations/versions/0037_1_10_2_add_task_reschedule_table.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``task_reschedule`` table
Revision ID: 0a2a5b66e19d
@@ -22,6 +21,8 @@ Revises: 9635ae0956e7
Create Date: 2018-06-17 22:50:00.053620
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0038_1_10_2_add_sm_dag_index.py b/airflow/migrations/versions/0038_1_10_2_add_sm_dag_index.py
index fd2403f89c..87893d3af5 100644
--- a/airflow/migrations/versions/0038_1_10_2_add_sm_dag_index.py
+++ b/airflow/migrations/versions/0038_1_10_2_add_sm_dag_index.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Merge migrations Heads.
Revision ID: 03bc53e68815
@@ -22,6 +21,7 @@ Revises: 0a2a5b66e19d, bf00311e1990
Create Date: 2018-11-24 20:21:46.605414
"""
+from __future__ import annotations
from alembic import op
diff --git a/airflow/migrations/versions/0039_1_10_2_add_superuser_field.py b/airflow/migrations/versions/0039_1_10_2_add_superuser_field.py
index 11ea2938de..2a5524e195 100644
--- a/airflow/migrations/versions/0039_1_10_2_add_superuser_field.py
+++ b/airflow/migrations/versions/0039_1_10_2_add_superuser_field.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add superuser field
Revision ID: 41f5f12752f8
@@ -22,6 +21,7 @@ Revises: 03bc53e68815
Create Date: 2018-12-04 15:50:04.456875
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0040_1_10_3_add_fields_to_dag.py b/airflow/migrations/versions/0040_1_10_3_add_fields_to_dag.py
index a1eb00f14f..13aaf27005 100644
--- a/airflow/migrations/versions/0040_1_10_3_add_fields_to_dag.py
+++ b/airflow/migrations/versions/0040_1_10_3_add_fields_to_dag.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``description`` and ``default_view`` column to ``dag`` table
Revision ID: c8ffec048a3b
@@ -23,6 +22,7 @@ Revises: 41f5f12752f8
Create Date: 2018-12-23 21:55:46.463634
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0041_1_10_3_add_schedule_interval_to_dag.py b/airflow/migrations/versions/0041_1_10_3_add_schedule_interval_to_dag.py
index 42e0a5db92..93188e2a4c 100644
--- a/airflow/migrations/versions/0041_1_10_3_add_schedule_interval_to_dag.py
+++ b/airflow/migrations/versions/0041_1_10_3_add_schedule_interval_to_dag.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add schedule interval to dag
Revision ID: dd4ecb8fbee3
@@ -23,6 +22,7 @@ Revises: c8ffec048a3b
Create Date: 2018-12-27 18:39:25.748032
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0042_1_10_3_task_reschedule_fk_on_cascade_delete.py b/airflow/migrations/versions/0042_1_10_3_task_reschedule_fk_on_cascade_delete.py
index 9a535cbc99..71c12204e3 100644
--- a/airflow/migrations/versions/0042_1_10_3_task_reschedule_fk_on_cascade_delete.py
+++ b/airflow/migrations/versions/0042_1_10_3_task_reschedule_fk_on_cascade_delete.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""task reschedule foreign key on cascade delete
Revision ID: 939bb1e647c8
@@ -22,6 +21,7 @@ Revises: dd4ecb8fbee3
Create Date: 2019-02-04 20:21:50.669751
"""
+from __future__ import annotations
from alembic import op
diff --git a/airflow/migrations/versions/0043_1_10_4_make_taskinstance_pool_not_nullable.py b/airflow/migrations/versions/0043_1_10_4_make_taskinstance_pool_not_nullable.py
index 53261af4a8..dcfd2acb19 100644
--- a/airflow/migrations/versions/0043_1_10_4_make_taskinstance_pool_not_nullable.py
+++ b/airflow/migrations/versions/0043_1_10_4_make_taskinstance_pool_not_nullable.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Make ``TaskInstance.pool`` not nullable
Revision ID: 6e96a59344a4
@@ -23,6 +22,7 @@ Revises: 939bb1e647c8
Create Date: 2019-06-13 21:51:32.878437
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0044_1_10_7_add_serialized_dag_table.py b/airflow/migrations/versions/0044_1_10_7_add_serialized_dag_table.py
index 38179f2a70..e1641b7661 100644
--- a/airflow/migrations/versions/0044_1_10_7_add_serialized_dag_table.py
+++ b/airflow/migrations/versions/0044_1_10_7_add_serialized_dag_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``serialized_dag`` table
Revision ID: d38e04c12aa2
@@ -23,6 +22,8 @@ Revises: 6e96a59344a4
Create Date: 2019-08-01 14:39:35.616417
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0045_1_10_7_add_root_dag_id_to_dag.py b/airflow/migrations/versions/0045_1_10_7_add_root_dag_id_to_dag.py
index f879450369..571338cf85 100644
--- a/airflow/migrations/versions/0045_1_10_7_add_root_dag_id_to_dag.py
+++ b/airflow/migrations/versions/0045_1_10_7_add_root_dag_id_to_dag.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``root_dag_id`` to ``DAG``
Revision ID: b3b105409875
@@ -23,6 +22,7 @@ Revises: d38e04c12aa2
Create Date: 2019-09-28 23:20:01.744775
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0046_1_10_5_change_datetime_to_datetime2_6_on_mssql_.py b/airflow/migrations/versions/0046_1_10_5_change_datetime_to_datetime2_6_on_mssql_.py
index 9b0773fc8d..ac056f43a4 100644
--- a/airflow/migrations/versions/0046_1_10_5_change_datetime_to_datetime2_6_on_mssql_.py
+++ b/airflow/migrations/versions/0046_1_10_5_change_datetime_to_datetime2_6_on_mssql_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""change datetime to datetime2(6) on MSSQL tables.
Revision ID: 74effc47d867
@@ -23,6 +22,7 @@ Revises: 6e96a59344a4
Create Date: 2019-08-01 15:19:57.585620
"""
+from __future__ import annotations
from collections import defaultdict
diff --git a/airflow/migrations/versions/0047_1_10_4_increase_queue_name_size_limit.py b/airflow/migrations/versions/0047_1_10_4_increase_queue_name_size_limit.py
index 50dbc43cae..9bbec39828 100644
--- a/airflow/migrations/versions/0047_1_10_4_increase_queue_name_size_limit.py
+++ b/airflow/migrations/versions/0047_1_10_4_increase_queue_name_size_limit.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Increase queue name size limit
Revision ID: 004c1210f153
@@ -23,6 +22,7 @@ Revises: 939bb1e647c8
Create Date: 2019-06-07 07:46:04.262275
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0048_1_10_3_remove_dag_stat_table.py b/airflow/migrations/versions/0048_1_10_3_remove_dag_stat_table.py
index 454482e718..4b3f8e470c 100644
--- a/airflow/migrations/versions/0048_1_10_3_remove_dag_stat_table.py
+++ b/airflow/migrations/versions/0048_1_10_3_remove_dag_stat_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Remove ``dag_stat`` table
Revision ID: a56c9515abdc
@@ -23,6 +22,7 @@ Revises: c8ffec048a3b
Create Date: 2018-12-27 10:27:59.715872
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0049_1_10_7_merge_heads.py b/airflow/migrations/versions/0049_1_10_7_merge_heads.py
index 1f589e993f..122427e323 100644
--- a/airflow/migrations/versions/0049_1_10_7_merge_heads.py
+++ b/airflow/migrations/versions/0049_1_10_7_merge_heads.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Straighten out the migrations
Revision ID: 08364691d074
@@ -23,6 +22,7 @@ Revises: a56c9515abdc, 004c1210f153, 74effc47d867, b3b105409875
Create Date: 2019-11-19 22:05:11.752222
"""
+from __future__ import annotations
# revision identifiers, used by Alembic.
revision = '08364691d074'
diff --git a/airflow/migrations/versions/0050_1_10_7_increase_length_for_connection_password.py b/airflow/migrations/versions/0050_1_10_7_increase_length_for_connection_password.py
index 955e1a6686..d2c04550e8 100644
--- a/airflow/migrations/versions/0050_1_10_7_increase_length_for_connection_password.py
+++ b/airflow/migrations/versions/0050_1_10_7_increase_length_for_connection_password.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Increase length for connection password
Revision ID: fe461863935f
@@ -23,6 +22,7 @@ Revises: 08364691d074
Create Date: 2019-12-08 09:47:09.033009
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0051_1_10_8_add_dagtags_table.py b/airflow/migrations/versions/0051_1_10_8_add_dagtags_table.py
index a7ae10860e..210a26cf4f 100644
--- a/airflow/migrations/versions/0051_1_10_8_add_dagtags_table.py
+++ b/airflow/migrations/versions/0051_1_10_8_add_dagtags_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``DagTags`` table
Revision ID: 7939bcff74ba
@@ -23,6 +22,7 @@ Revises: fe461863935f
Create Date: 2020-01-07 19:39:01.247442
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0052_1_10_10_add_pool_slots_field_to_task_instance.py b/airflow/migrations/versions/0052_1_10_10_add_pool_slots_field_to_task_instance.py
index 26bff68ba2..562e80c27b 100644
--- a/airflow/migrations/versions/0052_1_10_10_add_pool_slots_field_to_task_instance.py
+++ b/airflow/migrations/versions/0052_1_10_10_add_pool_slots_field_to_task_instance.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``pool_slots`` field to ``task_instance``
Revision ID: a4c2fd67d16b
@@ -23,6 +22,7 @@ Revises: 7939bcff74ba
Create Date: 2020-01-14 03:35:01.161519
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0053_1_10_10_add_rendered_task_instance_fields_table.py b/airflow/migrations/versions/0053_1_10_10_add_rendered_task_instance_fields_table.py
index 2027dd4745..3b836495be 100644
--- a/airflow/migrations/versions/0053_1_10_10_add_rendered_task_instance_fields_table.py
+++ b/airflow/migrations/versions/0053_1_10_10_add_rendered_task_instance_fields_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``RenderedTaskInstanceFields`` table
Revision ID: 852ae6c715af
@@ -23,6 +22,7 @@ Revises: a4c2fd67d16b
Create Date: 2020-03-10 22:19:18.034961
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0054_1_10_10_add_dag_code_table.py b/airflow/migrations/versions/0054_1_10_10_add_dag_code_table.py
index 523a1a0f8c..b9d8bb219b 100644
--- a/airflow/migrations/versions/0054_1_10_10_add_dag_code_table.py
+++ b/airflow/migrations/versions/0054_1_10_10_add_dag_code_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``dag_code`` table
Revision ID: 952da73b5eff
@@ -23,6 +22,7 @@ Revises: 852ae6c715af
Create Date: 2020-03-12 12:39:01.797462
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0055_1_10_11_add_precision_to_execution_date_in_mysql.py b/airflow/migrations/versions/0055_1_10_11_add_precision_to_execution_date_in_mysql.py
index 873ba148d0..aeaa13c5f1 100644
--- a/airflow/migrations/versions/0055_1_10_11_add_precision_to_execution_date_in_mysql.py
+++ b/airflow/migrations/versions/0055_1_10_11_add_precision_to_execution_date_in_mysql.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add Precision to ``execution_date`` in ``RenderedTaskInstanceFields`` table
Revision ID: a66efa278eea
@@ -23,6 +22,7 @@ Revises: 952da73b5eff
Create Date: 2020-06-16 21:44:02.883132
"""
+from __future__ import annotations
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0056_1_10_12_add_dag_hash_column_to_serialized_dag_.py b/airflow/migrations/versions/0056_1_10_12_add_dag_hash_column_to_serialized_dag_.py
index 5113e2f28c..e77223ebd7 100644
--- a/airflow/migrations/versions/0056_1_10_12_add_dag_hash_column_to_serialized_dag_.py
+++ b/airflow/migrations/versions/0056_1_10_12_add_dag_hash_column_to_serialized_dag_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``dag_hash`` Column to ``serialized_dag`` table
Revision ID: da3f683c3a5a
@@ -23,6 +22,7 @@ Revises: a66efa278eea
Create Date: 2020-08-07 20:52:09.178296
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0057_1_10_13_add_fab_tables.py b/airflow/migrations/versions/0057_1_10_13_add_fab_tables.py
index bd3fe44e9e..6cb76ea70a 100644
--- a/airflow/migrations/versions/0057_1_10_13_add_fab_tables.py
+++ b/airflow/migrations/versions/0057_1_10_13_add_fab_tables.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Create FAB Tables
Revision ID: 92c57b58940d
@@ -23,6 +22,7 @@ Revises: da3f683c3a5a
Create Date: 2020-11-13 19:27:10.161814
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0058_1_10_13_increase_length_of_fab_ab_view_menu_.py b/airflow/migrations/versions/0058_1_10_13_increase_length_of_fab_ab_view_menu_.py
index 4378c8bd0c..75f7c8835a 100644
--- a/airflow/migrations/versions/0058_1_10_13_increase_length_of_fab_ab_view_menu_.py
+++ b/airflow/migrations/versions/0058_1_10_13_increase_length_of_fab_ab_view_menu_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Increase length of ``Flask-AppBuilder`` ``ab_view_menu.name`` column
Revision ID: 03afc6b6f902
@@ -23,6 +22,7 @@ Revises: 92c57b58940d
Create Date: 2020-11-13 22:21:41.619565
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0059_2_0_0_drop_user_and_chart.py b/airflow/migrations/versions/0059_2_0_0_drop_user_and_chart.py
index deb2778661..892078c845 100644
--- a/airflow/migrations/versions/0059_2_0_0_drop_user_and_chart.py
+++ b/airflow/migrations/versions/0059_2_0_0_drop_user_and_chart.py
@@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Drop ``user`` and ``chart`` table
Revision ID: cf5dc11e79ad
@@ -22,6 +21,8 @@ Revises: 03afc6b6f902
Create Date: 2019-01-24 15:30:35.834740
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mysql
diff --git a/airflow/migrations/versions/0060_2_0_0_remove_id_column_from_xcom.py b/airflow/migrations/versions/0060_2_0_0_remove_id_column_from_xcom.py
index a588af5c53..133f48ba10 100644
--- a/airflow/migrations/versions/0060_2_0_0_remove_id_column_from_xcom.py
+++ b/airflow/migrations/versions/0060_2_0_0_remove_id_column_from_xcom.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Remove id column from xcom
Revision ID: bbf4a7ad0465
@@ -23,6 +22,7 @@ Revises: cf5dc11e79ad
Create Date: 2019-10-29 13:53:09.445943
"""
+from __future__ import annotations
from collections import defaultdict
diff --git a/airflow/migrations/versions/0061_2_0_0_increase_length_of_pool_name.py b/airflow/migrations/versions/0061_2_0_0_increase_length_of_pool_name.py
index 1174c9be6b..884690cfb1 100644
--- a/airflow/migrations/versions/0061_2_0_0_increase_length_of_pool_name.py
+++ b/airflow/migrations/versions/0061_2_0_0_increase_length_of_pool_name.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Increase length of pool name
Revision ID: b25a55525161
@@ -23,6 +22,7 @@ Revises: bbf4a7ad0465
Create Date: 2020-03-09 08:48:14.534700
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0062_2_0_0_add_dagrun_run_type.py b/airflow/migrations/versions/0062_2_0_0_add_dagrun_run_type.py
index ce3d090742..b3c1f8af19 100644
--- a/airflow/migrations/versions/0062_2_0_0_add_dagrun_run_type.py
+++ b/airflow/migrations/versions/0062_2_0_0_add_dagrun_run_type.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""
Add ``run_type`` column in ``dag_run`` table
@@ -24,6 +23,7 @@ Revises: b25a55525161
Create Date: 2020-04-08 13:35:25.671327
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0063_2_0_0_set_conn_type_as_non_nullable.py b/airflow/migrations/versions/0063_2_0_0_set_conn_type_as_non_nullable.py
index 1df5bfbfe3..09d6ce4311 100644
--- a/airflow/migrations/versions/0063_2_0_0_set_conn_type_as_non_nullable.py
+++ b/airflow/migrations/versions/0063_2_0_0_set_conn_type_as_non_nullable.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Set ``conn_type`` as non-nullable
Revision ID: 8f966b9c467a
@@ -23,6 +22,7 @@ Revises: 3c20cacc0044
Create Date: 2020-06-08 22:36:34.534121
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0064_2_0_0_add_unique_constraint_to_conn_id.py b/airflow/migrations/versions/0064_2_0_0_add_unique_constraint_to_conn_id.py
index e3172ddf34..eca02af502 100644
--- a/airflow/migrations/versions/0064_2_0_0_add_unique_constraint_to_conn_id.py
+++ b/airflow/migrations/versions/0064_2_0_0_add_unique_constraint_to_conn_id.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add unique constraint to ``conn_id``
Revision ID: 8d48763f6d53
@@ -23,6 +22,7 @@ Revises: 8f966b9c467a
Create Date: 2020-05-03 16:55:01.834231
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0065_2_0_0_update_schema_for_smart_sensor.py b/airflow/migrations/versions/0065_2_0_0_update_schema_for_smart_sensor.py
index 228361dc58..1b88f4cdae 100644
--- a/airflow/migrations/versions/0065_2_0_0_update_schema_for_smart_sensor.py
+++ b/airflow/migrations/versions/0065_2_0_0_update_schema_for_smart_sensor.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``sensor_instance`` table
Revision ID: e38be357a868
@@ -23,6 +22,8 @@ Revises: 8d48763f6d53
Create Date: 2019-06-07 04:03:17.003939
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
from sqlalchemy import func
diff --git a/airflow/migrations/versions/0066_2_0_0_add_queued_by_job_id_to_ti.py b/airflow/migrations/versions/0066_2_0_0_add_queued_by_job_id_to_ti.py
index 90247d02ef..e90a7017b5 100644
--- a/airflow/migrations/versions/0066_2_0_0_add_queued_by_job_id_to_ti.py
+++ b/airflow/migrations/versions/0066_2_0_0_add_queued_by_job_id_to_ti.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add queued by Job ID to TI
Revision ID: b247b1e3d1ed
@@ -23,6 +22,7 @@ Revises: e38be357a868
Create Date: 2020-09-04 11:53:00.978882
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0067_2_0_0_add_external_executor_id_to_ti.py b/airflow/migrations/versions/0067_2_0_0_add_external_executor_id_to_ti.py
index a9b6030caf..0c41a7eb32 100644
--- a/airflow/migrations/versions/0067_2_0_0_add_external_executor_id_to_ti.py
+++ b/airflow/migrations/versions/0067_2_0_0_add_external_executor_id_to_ti.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add external executor ID to TI
Revision ID: e1a11ece99cc
@@ -23,6 +22,7 @@ Revises: b247b1e3d1ed
Create Date: 2020-09-12 08:23:45.698865
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0068_2_0_0_drop_kuberesourceversion_and_.py b/airflow/migrations/versions/0068_2_0_0_drop_kuberesourceversion_and_.py
index d2449aa48d..0239b467c5 100644
--- a/airflow/migrations/versions/0068_2_0_0_drop_kuberesourceversion_and_.py
+++ b/airflow/migrations/versions/0068_2_0_0_drop_kuberesourceversion_and_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Drop ``KubeResourceVersion`` and ``KubeWorkerId``
Revision ID: bef4f3d11e8b
@@ -23,6 +22,7 @@ Revises: e1a11ece99cc
Create Date: 2020-09-22 18:45:28.011654
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0069_2_0_0_add_scheduling_decision_to_dagrun_and_.py b/airflow/migrations/versions/0069_2_0_0_add_scheduling_decision_to_dagrun_and_.py
index 66f25cf29f..99a97e2105 100644
--- a/airflow/migrations/versions/0069_2_0_0_add_scheduling_decision_to_dagrun_and_.py
+++ b/airflow/migrations/versions/0069_2_0_0_add_scheduling_decision_to_dagrun_and_.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``scheduling_decision`` to ``DagRun`` and ``DAG``
Revision ID: 98271e7606e2
@@ -23,6 +22,7 @@ Revises: bef4f3d11e8b
Create Date: 2020-10-01 12:13:32.968148
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0070_2_0_0_fix_mssql_exec_date_rendered_task_instance.py b/airflow/migrations/versions/0070_2_0_0_fix_mssql_exec_date_rendered_task_instance.py
index dd14290729..a12a0a6516 100644
--- a/airflow/migrations/versions/0070_2_0_0_fix_mssql_exec_date_rendered_task_instance.py
+++ b/airflow/migrations/versions/0070_2_0_0_fix_mssql_exec_date_rendered_task_instance.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""fix_mssql_exec_date_rendered_task_instance_fields_for_MSSQL
Revision ID: 52d53670a240
@@ -23,6 +22,8 @@ Revises: 98271e7606e2
Create Date: 2020-10-13 15:13:24.911486
"""
+from __future__ import annotations
+
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import mssql
diff --git a/airflow/migrations/versions/0071_2_0_0_add_job_id_to_dagrun_table.py b/airflow/migrations/versions/0071_2_0_0_add_job_id_to_dagrun_table.py
index c42308393f..79ebbd0ac4 100644
--- a/airflow/migrations/versions/0071_2_0_0_add_job_id_to_dagrun_table.py
+++ b/airflow/migrations/versions/0071_2_0_0_add_job_id_to_dagrun_table.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add ``creating_job_id`` to ``DagRun`` table
Revision ID: 364159666cbd
@@ -23,6 +22,7 @@ Revises: 52d53670a240
Create Date: 2020-10-10 09:08:07.332456
"""
+from __future__ import annotations
import sqlalchemy as sa
from alembic import op
diff --git a/airflow/migrations/versions/0072_2_0_0_add_k8s_yaml_to_rendered_templates.py b/airflow/migrations/versions/0072_2_0_0_add_k8s_yaml_to_rendered_templates.py
index 675df14259..6c431de336 100644
--- a/airflow/migrations/versions/0072_2_0_0_add_k8s_yaml_to_rendered_templates.py
+++ b/airflow/migrations/versions/0072_2_0_0_add_k8s_yaml_to_rendered_templates.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""add-k8s-yaml-to-rendered-templates
Revision ID: 45ba3f1493b9
@@ -23,6 +22,7 @@ Revises: 364159666cbd
Create Date: 2020-10-23 23:01:52.471442
"""
+from __future__ import annotations
import sqlalchemy_jsonfield
from alembic import op
diff --git a/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py b/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py
index 2538a12a89..a1705626e7 100644
--- a/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py
+++ b/airflow/migrations/versions/0073_2_0_0_prefix_dag_permissions.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Prefix DAG permissions.
Revision ID: 849da589634d
@@ -23,6 +22,7 @@ Revises: 45ba3f1493b9
Create Date: 2020-10-01 17:25:10.006322
"""
+from __future__ import annotations
from flask_appbuilder import SQLA
diff --git a/airflow/migrations/versions/0074_2_0_0_resource_based_permissions.py b/airflow/migrations/versions/0074_2_0_0_resource_based_permissions.py
index 6942be25cf..296d07747f 100644
--- a/airflow/migrations/versions/0074_2_0_0_resource_based_permissions.py
+++ b/airflow/migrations/versions/0074_2_0_0_resource_based_permissions.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Resource based permissions.
Revision ID: 2c6edca13270
@@ -23,6 +22,8 @@ Revises: 849da589634d
Create Date: 2020-10-21 00:18:52.529438
"""
+from __future__ import annotations
+
import logging
from airflow.security import permissions
diff --git a/airflow/migrations/versions/0075_2_0_0_add_description_field_to_connection.py b/airflow/migrations/versions/0075_2_0_0_add_description_field_to_connection.py
index 4c3f5835dc..b4275cdda3 100644
--- a/airflow/migrations/versions/0075_2_0_0_add_description_field_to_connection.py
+++ b/airflow/migrations/versions/0075_2_0_0_add_description_field_to_connection.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
"""Add description field to ``connection`` table
Revision ID: 61ec73d9401f
@@ -23,6 +22,7 @@ Revises: 2c6edca13270
Create Date: 2020-09-10 14:56:30.279248
"""
+from __future__ import annotations
import sqlalchemy as sa
... 25074 lines suppressed ...