You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/08/28 12:47:32 UTC
[airflow] branch openlineage-system-tests updated (29143bd372 -> 25a95e132f)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch openlineage-system-tests
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 29143bd372 system tests: implement operator, variable transport
add 85aea74b64 Fix AWS Batch waiter failure state (#33656)
add 5544c8b20b Addd triggers and transfers to provider_info.schema (#33654)
add 5f504e9a17 Upgrade botocore/aiobotocore minimum requirements (#33649)
add 4bdf908041 Fix 2.7.0 db migration job errors (#33652)
add 2f8caf91ab Fix broken link to the "UPDATING.md" file (#33583)
add 92474db6a5 Refactor: Better percentage formatting (#33595)
add 2ae1c10bfa add support for Yandex Dataproc cluster labels (#29811)
add dd218dde8a Improve provider verification pre-commit (#33640)
add 4a7cd606cc Bump azure-kusto-data>=4.1.0 (#33598)
add b1cdab39f8 Add back `get_url_for_login` in security manager (#33660)
add 5dfbbbbf5a Referencing correct hooks for Apache Pinot (#33601)
add 4037d79b5c Bring back wrong "hook-class-names" check (#33662)
add 94f70d8184 Replace strftime with f-strings where nicer (#33455)
add c88ff1d7c2 Bump apache-airflow-providers-snowflake due to breaking changes (#33615)
add 12ee795bbf AWS Connection Docs: Remove deprecation information due to undeprecate wrong `aws_conn_id` fallback behaviour (#33636)
add 75ce4d1923 Add istio test, use curl /quitquitquit to exit sidecar, and some othe… (#33306)
add b1a3b42880 Make auth managers provide their own airflow CLI commands (#33481)
add 53a8973952 Fix bug in task logs when using AWS CloudWatch. Do not set `start_time` (#33673)
add 2dbb963324 Refactor: Improve detection of duplicates and list sorting (#33675)
add d06c14f527 Set snowflake_conn_id on Snowflake Operators to avoid error (#33681)
add 79b8cfc0fa Sort data before groupby in TIS duration calculation (#33535)
add 6130993d78 Fix KubernetesPodOperator duplicating logs when interrupted (#33500)
add 9dac4878c7 Add DefaultAzureCredential auth for ADX service (#33627)
add 4154cc04ce Do not create lists we don't need (#33519)
add e0c40c5369 Refactor: Use tmp_path in test_oracle_to_azure_data_lake (#33633)
add 1953648fda Refactor: lists and paths in dev (#33626)
add 85acbb4ae9 Refactor: Remove useless str() calls (#33629)
add dc47c460dc Update docs in databricks.py - we use 2.1 now (#32340)
add ed6a4fd116 Remove deprecated parts from Slack provider (#33557)
add 08188f80f4 Bring back hive support for Python 3.11 (#32607)
add 96efcfb830 Further improvements for provider verification (#33670)
add ec776de518 Use str.splitlines() to split lines in breeze (#33594)
add 2d8625253f Remove non-public interface usage in EcsRunTaskOperator (#29447)
add 626d3daa9b Add OpenLineage support for Trino. (#32910)
add faa50cbe2f feat(providers/microsoft): add DefaultAzureCredential support to AzureContainerInstanceHook (#33467)
add 47682042a4 Add back link to the ASF blog about severity to the policy (#33691)
add 9fa782f622 Set context inside templates (#33645)
add df7c170e55 Avoid WSL2 ones when finding a context for Breeze (#33547)
add 6a7812b7c8 Revert "Avoid WSL2 ones when finding a context for Breeze (#33547)" (#33695)
add f971ba2f2f SECURITY_MANAGER_CLASS should be a referrence to class, not a string (#33690)
add c477031039 Inspect container state rather than last_state when deciding whether to skip (#33702)
add 36b06a7bb4 Ability to inject extra containers into pgbouncer (#33686)
add 83efcaa835 always push ECS task ARN to xcom in `EcsRunTaskOperator` (#33703)
add 370348a396 Treat dag-defined access_control as authoritative if defined (#33632)
add e8ba57968d Add limit 1 if required first value from query result (#33672)
add 9b8a093e07 Use tmp_path in amazon s3 test (#33705)
add 633217c842 Simplify conditions on len() in providers/amazon (#33565)
add ae56caea92 Use tmp_path in cli/command tests (#33709)
add 0e00564328 Use str.splitlines() to split lines (#33592)
add 4bd0b56432 Refactor: less DRY in test_mapped_task_instance_endpoint (#33710)
add 66d0222947 Add CloudRunHook and operators (#33067)
add ba0bab0114 Refactor: Use random.choices (#33631)
add 3f984edd00 Refactor: Use Path.unlink(missing_ok=True) (#33666)
add 5c35786ca2 Fix cleaning zombie RESTARTING tasks (#33706)
add 0931abdb25 Add AzureBatchOperator example (#33716)
add 3bb1c07861 Refactor SqlAlchemy session.execute() calls to 2.0 style in case of plain text SQL queries (#32857)
add 38f2737153 Fix Azure Batch Hook instantation (#33731)
add 95ece9feef Move Azure examples into system tests (#33727)
add 8aa2229182 Refactor: path.rglob(…) does not need ** (#33669)
add 94c8291629 Refactor: Use f-strings (#33734)
add 7a493ea949 test(providers/microsoft): add test case for AzureIdentityCredentialAdapter.signed_session (#33687)
add 55ff4cf935 fix(providers/azure): remove json.dumps when querying AzureCosmosDBHook (#33653)
add 947b504d5b Add DefaultAzureCredential support to AzureBatchHook (#33469)
add d91c481ac9 Respect "soft_fail" argument when "poke" is called (#33401)
add eb93f67b2b Refactor: tmp_path in tests/dag_processing (#33740)
add a5aa16184d Add documentation explaining template_ext (and how to override it) (#33735)
add a16f8613a8 Chart: update gitsync (#33748)
add c077d19060 Prepare docs for Aug 2023 3rd wave of Providers (#33730)
add faf32539d6 Optimise Airflow DB backend usage in Azure Provider (#33750)
add c90eec9365 Use f-string instead of in Airflow providers (#33752)
add 60e6847c18 Use a single statement with multiple contexts instead of nested statements in core (#33769)
add 03529d524f Add DefaultAzureCredential support to cosmos (#33436)
add bcb97d05ad Fix partial doc reference error due to missing space (#33770)
add 5a1889a7e7 Merge multiple isintance calls for the same object in a single call (#33767)
add cfa5477aa4 Replace single quotes by double quotes in all Airflow modules (#33766)
add a44f261fcd Replace type func by isinstance in fab_security manager (#33760)
add e16e9f3439 Replace type func by isinstance in DockerOperator (#33759)
add a1d4a20548 E731: replace lambda by a def method in Airflow core (#33758)
add 2a00f18ebb E731: replace lambda by a def method in Airflow providers (#33757)
add 3d01919a30 Avoid blocking event loop when using DataFusionAsyncHook by replacing sleep by asyncio.sleep (#33756)
add 272b40aaad Use f-string instead of in Airflow core (#33753)
add 784e0ef04b Refactor: Simplify a few loops (#33736)
add 4e545c8190 remove unnecessary map and rewrite it using list in Airflow core (#33764)
add e4ae83d488 Use startswith once with a tuple in Hive hook (#33765)
add 353b1482d3 remove unnecessary and rewrite it using list in providers (#33763)
add 3d575fed54 Allow passing fully_qualified_namespace and credential to initialize Azure Service Bus Client (#33493)
add 1e81ed1999 Use literal dict instead of calling dict() in Airflow core (#33762)
add b11525702c Use literal dict instead of calling dict() in providers (#33761)
add 984ba22e6e Limit hive provider check for Python 3.11 temporarily (#33774)
add 4bae275edd Remove duplicated message commit in Azure MessageHook (#33776)
add 6d182beec6 Use a single statement with multiple contexts instead of nested statements in providers (#33768)
add 9ce76e321f Respect "soft_fail" for core async sensors (#33403)
add b1f2a1693c Convert hard-coded allowlist error code to be argument of HttpSensor (#33717)
add 42bc8fcb6b Update pod.py (#33779)
add 3ba994d8f4 Limit Redis dependencies to match celery limits (#33773)
add a746defd6b Fix version heads map pre_commit rule (#33749)
add 2b1a1944aa Remove "eager upgrade" from PROD image completely (#33784)
add badcf24e17 Fix an edge case after refactoring back-reference breeze command (#33790)
add c96ae2b277 Run back-reference breeze command as part of our CI builds (#33791)
add 8227db3436 Vastly speed up provider documentation publishing (#33792)
add f9276f0597 Use PyPI constraints for PROD image in non-main branch (#33789)
add b860a6adde Fix test logic for test_reset_orphaned_tasks_non_running_dagruns (#33741)
add 00d54c04c4 Refactor: Use tmp_path in breeze/tests (#33739)
add 660386b5d9 Refactor: Use inplace .sort() (#33743)
add 038216e8e1 Chart: Update postgres subchart (#33747)
add 02af225e7b Refactor: Use tmp_path in tests/plugins (#33738)
add 64948fa782 Fix UI DAG counts including deleted DAGs (#33778)
add 452a978337 Improve modules import in cncf.kubernetes probvider by move some of them into a type-checking block (#33781)
add 6802d4136f Improve modules import in google provider by move some of them into a type-checking block (#33783)
add 9d8c77e447 Improve modules import in Airflow providers by some of them into a type-checking block (#33754)
add fe05e1a53d Improve importing the module in Airflow api_connexion package (#33798)
add 326acdeb96 Improve importing the module in Airflow API package (#33797)
add 8ebb70091b Improve importing the module in Airflow operators package (#33800)
add 87b08ad084 Improve importing the module in Airflow models package (#33799)
add 4f3d2841f6 Minor grammatical fix (#33802)
add 97091b1784 Improve importing the module in Airflow sensors package (#33801)
add 58d8577f35 Improve importing the module in Airflow utils package (#33803)
add a23f3181fe Improve importing the module in Airflow decorators package (#33804)
add cede385e93 Always use `Literal` from `typing_extensions` (#33794)
add 5da4afe908 Improve importing the modules in Airflow jobs package (#33805)
add f4c522f120 Improve importing the modules in Airflow auth, api_internal and callbacks packages (#33807)
add dfa70b9423 Fix package name in exception message for hdfs provider (#33813)
add bb3adb4b90 Improve importing the modules in Airflow cli package (#33806)
add d96f931b99 Use correct headings in HDFS Provider changelog (#33809)
add b470c6bdcc Improve importing the modules in Airflow www package (#33810)
add aa54a22185 Improve importing the modules in Airflow macros, metrics, notifications and secrets packages (#33811)
add 0c8295ce91 Improve importing the modules in Airflow serialization, task, template and ti_deps packages (#33812)
add 2663b177a6 Generate Python API docs for Google ADS (#33814)
add 379bd80573 feat: support K8S 1.28 (#33816)
add 015fef31bb Improve importing the modules in Airflow dag_processing, datasets and example_dags packages (#33808)
add 76c2ade2c6 feat(providers/microsoft): add DefaultAzureCredential to data_lake (#33433)
add 7faa727951 Refactor: remove unused state - SHUTDOWN (#33746)
add 9d5327806f Bump azure-mgmt-containerinstance>=7.0.0,<9.0.0 (#33696)
new 25a95e132f system tests: implement operator, variable transport
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (29143bd372)
\
N -- N -- N refs/heads/openlineage-system-tests (25a95e132f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/SECURITY.md | 15 +-
.github/actions/build-ci-images/action.yml | 14 +-
.github/actions/build-prod-images/action.yml | 21 +-
.github/workflows/ci.yml | 10 +-
.pre-commit-config.yaml | 1 +
BREEZE.rst | 3 +
Dockerfile | 52 +-
Dockerfile.ci | 9 +-
README.md | 18 +-
airflow/api/auth/backend/basic_auth.py | 6 +-
airflow/api/client/__init__.py | 5 +-
airflow/api/common/delete_dag.py | 5 +-
airflow/api/common/experimental/__init__.py | 9 +-
airflow/api/common/experimental/get_code.py | 2 +-
.../api/common/experimental/get_dag_run_state.py | 5 +-
airflow/api/common/experimental/get_lineage.py | 10 +-
airflow/api/common/experimental/get_task.py | 6 +-
.../api/common/experimental/get_task_instance.py | 8 +-
airflow/api/common/experimental/pool.py | 6 +-
airflow/api/common/mark_tasks.py | 13 +-
airflow/api/common/trigger_dag.py | 5 +-
.../api_connexion/endpoints/connection_endpoint.py | 8 +-
airflow/api_connexion/endpoints/dag_endpoint.py | 11 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 10 +-
.../api_connexion/endpoints/dag_source_endpoint.py | 2 +-
.../endpoints/dag_warning_endpoint.py | 9 +-
.../api_connexion/endpoints/dataset_endpoint.py | 10 +-
.../api_connexion/endpoints/event_log_endpoint.py | 9 +-
.../api_connexion/endpoints/extra_link_endpoint.py | 13 +-
airflow/api_connexion/endpoints/health_endpoint.py | 6 +-
.../endpoints/import_error_endpoint.py | 9 +-
airflow/api_connexion/endpoints/log_endpoint.py | 10 +-
airflow/api_connexion/endpoints/plugin_endpoint.py | 6 +-
airflow/api_connexion/endpoints/pool_endpoint.py | 8 +-
.../api_connexion/endpoints/provider_endpoint.py | 9 +-
.../endpoints/role_and_permission_endpoint.py | 7 +-
airflow/api_connexion/endpoints/task_endpoint.py | 7 +-
.../endpoints/task_instance_endpoint.py | 12 +-
airflow/api_connexion/endpoints/user_endpoint.py | 8 +-
.../api_connexion/endpoints/variable_endpoint.py | 8 +-
.../api_connexion/endpoints/version_endpoint.py | 6 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 8 +-
airflow/api_connexion/exceptions.py | 6 +-
airflow/api_connexion/parameters.py | 9 +-
airflow/api_connexion/schemas/dag_schema.py | 8 +-
.../api_connexion/schemas/task_instance_schema.py | 7 +-
airflow/api_connexion/schemas/task_schema.py | 6 +-
airflow/api_internal/endpoints/rpc_api_endpoint.py | 6 +-
airflow/auth/managers/base_auth_manager.py | 11 +-
.../managers/fab/cli_commands}/__init__.py | 0
.../auth/managers/fab/cli_commands/definition.py | 220 +++
.../managers/fab/cli_commands}/role_command.py | 17 +-
.../fab/cli_commands}/sync_perm_command.py | 2 +-
.../managers/fab/cli_commands}/user_command.py | 17 +-
.../managers/fab/cli_commands/utils.py} | 7 +-
airflow/auth/managers/fab/fab_auth_manager.py | 54 +-
.../managers/fab/security_manager/modules/db.py | 46 +-
.../auth/managers/fab/security_manager/override.py | 6 +-
airflow/callbacks/base_callback_sink.py | 5 +-
airflow/callbacks/database_callback_sink.py | 8 +-
airflow/callbacks/pipe_callback_sink.py | 9 +-
airflow/cli/cli_config.py | 213 +--
airflow/cli/cli_parser.py | 22 +-
airflow/cli/commands/cheat_sheet_command.py | 7 +-
airflow/cli/commands/connection_command.py | 25 +-
airflow/cli/commands/dag_command.py | 10 +-
airflow/cli/commands/db_command.py | 6 +-
airflow/cli/commands/jobs_command.py | 6 +-
airflow/cli/commands/standalone_command.py | 16 +-
airflow/cli/commands/task_command.py | 14 +-
airflow/cli/commands/webserver_command.py | 10 +-
airflow/cli/simple_table.py | 6 +-
airflow/cli/utils.py | 5 +-
airflow/contrib/hooks/__init__.py | 440 +++---
airflow/contrib/operators/__init__.py | 1588 ++++++++++----------
airflow/contrib/secrets/__init__.py | 28 +-
airflow/contrib/sensors/__init__.py | 176 +--
airflow/contrib/task_runner/__init__.py | 4 +-
airflow/contrib/utils/__init__.py | 40 +-
airflow/contrib/utils/log/__init__.py | 6 +-
airflow/dag_processing/manager.py | 13 +-
airflow/dag_processing/processor.py | 18 +-
airflow/datasets/manager.py | 5 +-
airflow/decorators/base.py | 26 +-
airflow/decorators/branch_python.py | 7 +-
airflow/decorators/external_python.py | 7 +-
airflow/decorators/python.py | 7 +-
airflow/decorators/python_virtualenv.py | 7 +-
airflow/decorators/sensor.py | 7 +-
airflow/decorators/setup_teardown.py | 7 +-
airflow/decorators/short_circuit.py | 7 +-
airflow/decorators/task_group.py | 6 +-
airflow/example_dags/example_bash_operator.py | 2 +-
airflow/example_dags/example_dag_decorator.py | 6 +-
airflow/example_dags/example_params_trigger_ui.py | 9 +-
airflow/example_dags/example_params_ui_tutorial.py | 9 +-
.../example_short_circuit_decorator.py | 4 +-
.../example_dags/example_short_circuit_operator.py | 4 +-
airflow/example_dags/example_skip_dag.py | 6 +-
airflow/example_dags/plugins/workday.py | 6 +-
airflow/executors/base_executor.py | 2 +-
airflow/executors/debug_executor.py | 2 +-
airflow/executors/local_executor.py | 2 +-
airflow/executors/sequential_executor.py | 2 +-
airflow/jobs/backfill_job_runner.py | 12 +-
airflow/jobs/dag_processor_job_runner.py | 6 +-
airflow/jobs/job.py | 11 +-
airflow/jobs/local_task_job_runner.py | 16 +-
airflow/jobs/scheduler_job_runner.py | 37 +-
airflow/jobs/triggerer_job_runner.py | 11 +-
airflow/macros/__init__.py | 6 +-
airflow/metrics/base_stats_logger.py | 7 +-
airflow/metrics/datadog_logger.py | 8 +-
airflow/metrics/otel_logger.py | 13 +-
airflow/metrics/statsd_logger.py | 8 +-
airflow/models/baseoperator.py | 19 +-
airflow/models/dag.py | 11 +-
airflow/models/dagbag.py | 6 +-
airflow/models/dagcode.py | 6 +-
airflow/models/dagrun.py | 28 +-
airflow/models/dagwarning.py | 5 +-
airflow/models/db_callback_request.py | 5 +-
airflow/models/expandinput.py | 4 +-
airflow/models/mappedoperator.py | 32 +-
airflow/models/operator.py | 8 +-
airflow/models/param.py | 2 +-
airflow/models/pool.py | 6 +-
airflow/models/renderedtifields.py | 6 +-
airflow/models/serialized_dag.py | 15 +-
airflow/models/skipmixin.py | 2 +-
airflow/models/taskinstance.py | 29 +-
airflow/models/taskmixin.py | 9 +-
airflow/models/taskreschedule.py | 7 +-
airflow/models/trigger.py | 10 +-
airflow/models/variable.py | 7 +-
airflow/models/xcom.py | 9 +-
airflow/models/xcom_arg.py | 12 +-
airflow/notifications/basenotifier.py | 7 +-
airflow/operators/__init__.py | 252 ++--
airflow/operators/bash.py | 6 +-
airflow/operators/branch.py | 6 +-
airflow/operators/datetime.py | 6 +-
airflow/operators/email.py | 6 +-
airflow/operators/empty.py | 6 +-
airflow/operators/generic_transfer.py | 6 +-
airflow/operators/latest_only.py | 2 +-
airflow/operators/python.py | 20 +-
airflow/operators/smooth.py | 6 +-
airflow/operators/subdag.py | 14 +-
airflow/operators/trigger_dagrun.py | 2 +-
airflow/operators/weekday.py | 6 +-
airflow/policies.py | 2 +-
airflow/provider_info.schema.json | 54 +
.../providers/airbyte/.latest-doc-only-change.txt | 2 +-
airflow/providers/alibaba/CHANGELOG.rst | 13 +
airflow/providers/alibaba/__init__.py | 2 +-
airflow/providers/alibaba/provider.yaml | 1 +
airflow/providers/amazon/CHANGELOG.rst | 42 +
airflow/providers/amazon/__init__.py | 2 +-
airflow/providers/amazon/aws/hooks/dynamodb.py | 2 +-
airflow/providers/amazon/aws/hooks/emr.py | 2 +-
.../providers/amazon/aws/hooks/redshift_data.py | 2 +-
airflow/providers/amazon/aws/hooks/s3.py | 2 +-
airflow/providers/amazon/aws/hooks/sagemaker.py | 15 +-
.../amazon/aws/log/cloudwatch_task_handler.py | 6 +-
airflow/providers/amazon/aws/operators/ecs.py | 71 +-
.../providers/amazon/aws/operators/sagemaker.py | 2 +-
airflow/providers/amazon/aws/sensors/batch.py | 4 +-
airflow/providers/amazon/aws/sensors/s3.py | 4 +-
airflow/providers/amazon/aws/sensors/sqs.py | 11 +-
.../providers/amazon/aws/transfers/sql_to_s3.py | 4 +-
airflow/providers/amazon/aws/triggers/sqs.py | 8 +-
airflow/providers/amazon/aws/utils/identifiers.py | 51 +
airflow/providers/amazon/aws/utils/sqs.py | 3 +-
airflow/providers/amazon/aws/waiters/batch.json | 6 +-
airflow/providers/amazon/provider.yaml | 21 +-
airflow/providers/apache/beam/CHANGELOG.rst | 18 +
airflow/providers/apache/beam/__init__.py | 2 +-
airflow/providers/apache/beam/hooks/beam.py | 6 +-
airflow/providers/apache/beam/provider.yaml | 1 +
.../apache/cassandra/.latest-doc-only-change.txt | 2 +-
.../apache/drill/.latest-doc-only-change.txt | 2 +-
airflow/providers/apache/drill/hooks/drill.py | 6 +-
.../apache/druid/.latest-doc-only-change.txt | 2 +-
airflow/providers/apache/flink/CHANGELOG.rst | 15 +
airflow/providers/apache/flink/__init__.py | 6 +-
.../apache/flink/operators/flink_kubernetes.py | 4 +-
airflow/providers/apache/flink/provider.yaml | 1 +
.../apache/hdfs/.latest-doc-only-change.txt | 2 +-
airflow/providers/apache/hdfs/CHANGELOG.rst | 4 +-
airflow/providers/apache/hdfs/hooks/hdfs.py | 6 +-
airflow/providers/apache/hdfs/sensors/hdfs.py | 4 +-
airflow/providers/apache/hive/CHANGELOG.rst | 28 +
airflow/providers/apache/hive/__init__.py | 2 +-
airflow/providers/apache/hive/hooks/hive.py | 131 +-
airflow/providers/apache/hive/macros/hive.py | 13 +-
airflow/providers/apache/hive/provider.yaml | 12 +-
.../apache/hive/sensors/metastore_partition.py | 6 +-
.../apache/hive/transfers/mysql_to_hive.py | 29 +-
.../providers/apache/hive/transfers/s3_to_hive.py | 8 +-
airflow/providers/apache/impala/hooks/impala.py | 6 +-
.../apache/kafka/.latest-doc-only-change.txt | 1 +
.../apache/kylin/.latest-doc-only-change.txt | 2 +-
airflow/providers/apache/livy/CHANGELOG.rst | 14 +
airflow/providers/apache/livy/__init__.py | 6 +-
airflow/providers/apache/livy/hooks/livy.py | 6 +-
airflow/providers/apache/livy/provider.yaml | 1 +
airflow/providers/apache/livy/triggers/livy.py | 2 +-
.../apache/pig/.latest-doc-only-change.txt | 2 +-
airflow/providers/apache/pig/hooks/pig.py | 65 +-
airflow/providers/apache/pinot/CHANGELOG.rst | 12 +
airflow/providers/apache/pinot/__init__.py | 2 +-
airflow/providers/apache/pinot/hooks/pinot.py | 13 +-
airflow/providers/apache/pinot/provider.yaml | 5 +-
airflow/providers/apache/spark/CHANGELOG.rst | 12 +
airflow/providers/apache/spark/__init__.py | 2 +-
airflow/providers/apache/spark/provider.yaml | 1 +
.../apache/sqoop/.latest-doc-only-change.txt | 2 +-
.../providers/apprise/.latest-doc-only-change.txt | 1 +
airflow/providers/apprise/notifications/apprise.py | 6 +-
airflow/providers/arangodb/CHANGELOG.rst | 18 +
airflow/providers/arangodb/__init__.py | 6 +-
airflow/providers/arangodb/hooks/arangodb.py | 2 +-
airflow/providers/arangodb/provider.yaml | 1 +
.../providers/asana/.latest-doc-only-change.txt | 2 +-
.../atlassian/jira/.latest-doc-only-change.txt | 2 +-
airflow/providers/celery/CHANGELOG.rst | 14 +
airflow/providers/celery/__init__.py | 2 +-
.../providers/celery/executors/celery_executor.py | 5 +-
.../celery/executors/celery_executor_utils.py | 5 +-
.../celery/executors/celery_kubernetes_executor.py | 4 +-
airflow/providers/celery/provider.yaml | 1 +
.../providers/cloudant/.latest-doc-only-change.txt | 2 +-
airflow/providers/cncf/kubernetes/CHANGELOG.rst | 36 +
airflow/providers/cncf/kubernetes/__init__.py | 2 +-
.../kubernetes/executors/kubernetes_executor.py | 11 +-
.../executors/kubernetes_executor_utils.py | 3 +-
.../executors/local_kubernetes_executor.py | 6 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 4 +-
airflow/providers/cncf/kubernetes/k8s_model.py | 4 +-
airflow/providers/cncf/kubernetes/operators/pod.py | 98 +-
.../cncf/kubernetes/operators/resource.py | 5 +-
airflow/providers/cncf/kubernetes/pod_generator.py | 5 +-
.../cncf/kubernetes/pod_launcher_deprecated.py | 11 +-
airflow/providers/cncf/kubernetes/provider.yaml | 1 +
.../cncf/kubernetes/template_rendering.py | 6 +-
airflow/providers/cncf/kubernetes/triggers/pod.py | 7 +-
.../providers/cncf/kubernetes/utils/delete_from.py | 5 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 47 +-
airflow/providers/common/sql/CHANGELOG.rst | 13 +
airflow/providers/common/sql/__init__.py | 2 +-
airflow/providers/common/sql/operators/sql.py | 3 +-
airflow/providers/common/sql/provider.yaml | 1 +
airflow/providers/daskexecutor/CHANGELOG.rst | 9 +
airflow/providers/daskexecutor/__init__.py | 2 +-
airflow/providers/daskexecutor/provider.yaml | 1 +
airflow/providers/databricks/CHANGELOG.rst | 25 +
airflow/providers/databricks/__init__.py | 2 +-
airflow/providers/databricks/hooks/databricks.py | 12 +-
.../providers/databricks/hooks/databricks_base.py | 18 +-
.../providers/databricks/hooks/databricks_sql.py | 16 +-
.../providers/databricks/operators/databricks.py | 3 +-
.../databricks/operators/databricks_repos.py | 7 +-
.../databricks/operators/databricks_sql.py | 3 +-
airflow/providers/databricks/provider.yaml | 1 +
airflow/providers/datadog/CHANGELOG.rst | 17 +
airflow/providers/datadog/__init__.py | 6 +-
airflow/providers/datadog/provider.yaml | 1 +
airflow/providers/dbt/cloud/CHANGELOG.rst | 9 +
airflow/providers/dbt/cloud/__init__.py | 6 +-
airflow/providers/dbt/cloud/hooks/dbt.py | 23 +-
airflow/providers/dbt/cloud/provider.yaml | 1 +
.../providers/dingding/.latest-doc-only-change.txt | 2 +-
.../providers/discord/.latest-doc-only-change.txt | 2 +-
airflow/providers/docker/CHANGELOG.rst | 10 +
airflow/providers/docker/__init__.py | 2 +-
airflow/providers/docker/decorators/docker.py | 18 +-
airflow/providers/docker/operators/docker.py | 4 +-
airflow/providers/docker/provider.yaml | 1 +
airflow/providers/elasticsearch/CHANGELOG.rst | 22 +
airflow/providers/elasticsearch/__init__.py | 6 +-
.../providers/elasticsearch/hooks/elasticsearch.py | 24 +-
.../elasticsearch/log/es_json_formatter.py | 6 +-
airflow/providers/elasticsearch/log/es_response.py | 2 +-
.../providers/elasticsearch/log/es_task_handler.py | 7 +-
airflow/providers/elasticsearch/provider.yaml | 1 +
airflow/providers/exasol/CHANGELOG.rst | 8 +
airflow/providers/exasol/__init__.py | 2 +-
airflow/providers/exasol/hooks/exasol.py | 22 +-
airflow/providers/exasol/provider.yaml | 1 +
.../providers/facebook/.latest-doc-only-change.txt | 2 +-
airflow/providers/facebook/ads/hooks/ads.py | 6 +-
airflow/providers/ftp/CHANGELOG.rst | 12 +
airflow/providers/ftp/__init__.py | 2 +-
airflow/providers/ftp/provider.yaml | 1 +
airflow/providers/ftp/sensors/ftp.py | 4 +-
airflow/providers/github/CHANGELOG.rst | 17 +
airflow/providers/github/__init__.py | 6 +-
airflow/providers/github/operators/github.py | 4 +-
airflow/providers/github/provider.yaml | 1 +
airflow/providers/github/sensors/github.py | 4 +-
airflow/providers/google/CHANGELOG.rst | 16 +-
airflow/providers/google/__init__.py | 2 +-
airflow/providers/google/ads/.gitignore | 3 +
.../ads/_vendor/googleads/interceptors/__init__.py | 16 -
.../google/ads/_vendor/googleads/v12/__init__.py | 16 -
.../ads/_vendor/googleads/v12/common/__init__.py | 16 -
.../_vendor/googleads/v12/common/types/__init__.py | 16 -
.../ads/_vendor/googleads/v12/enums/__init__.py | 16 -
.../_vendor/googleads/v12/enums/types/__init__.py | 16 -
.../ads/_vendor/googleads/v12/errors/__init__.py | 16 -
.../_vendor/googleads/v12/errors/types/__init__.py | 16 -
.../_vendor/googleads/v12/resources/__init__.py | 16 -
.../googleads/v12/resources/types/__init__.py | 16 -
.../ads/_vendor/googleads/v12/services/__init__.py | 16 -
.../googleads/v12/services/services/__init__.py | 16 -
.../services/services/customer_service/__init__.py | 16 -
.../customer_service/transports/__init__.py | 16 -
.../services/google_ads_service/__init__.py | 16 -
.../google_ads_service/transports/__init__.py | 16 -
.../googleads/v12/services/types/__init__.py | 16 -
airflow/providers/google/ads/hooks/ads.py | 16 +-
.../_internal_client/secret_manager_client.py | 5 +-
.../cloud/example_dags/example_cloud_sql_query.py | 52 +-
airflow/providers/google/cloud/hooks/automl.py | 20 +-
airflow/providers/google/cloud/hooks/bigquery.py | 29 +-
.../providers/google/cloud/hooks/bigquery_dts.py | 8 +-
airflow/providers/google/cloud/hooks/bigtable.py | 19 +-
.../providers/google/cloud/hooks/cloud_batch.py | 8 +-
.../providers/google/cloud/hooks/cloud_build.py | 10 +-
.../providers/google/cloud/hooks/cloud_composer.py | 24 +-
.../google/cloud/hooks/cloud_memorystore.py | 8 +-
.../cloud/hooks/{cloud_batch.py => cloud_run.py} | 154 +-
airflow/providers/google/cloud/hooks/cloud_sql.py | 8 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 14 +-
airflow/providers/google/cloud/hooks/compute.py | 8 +-
.../providers/google/cloud/hooks/compute_ssh.py | 2 +-
.../providers/google/cloud/hooks/datacatalog.py | 8 +-
airflow/providers/google/cloud/hooks/dataflow.py | 12 +-
airflow/providers/google/cloud/hooks/dataform.py | 6 +-
airflow/providers/google/cloud/hooks/datafusion.py | 3 +-
airflow/providers/google/cloud/hooks/dataplex.py | 10 +-
airflow/providers/google/cloud/hooks/dataproc.py | 16 +-
.../google/cloud/hooks/dataproc_metastore.py | 14 +-
airflow/providers/google/cloud/hooks/dlp.py | 276 ++--
airflow/providers/google/cloud/hooks/gcs.py | 18 +-
airflow/providers/google/cloud/hooks/kms.py | 6 +-
.../google/cloud/hooks/kubernetes_engine.py | 68 +-
airflow/providers/google/cloud/hooks/looker.py | 5 +-
airflow/providers/google/cloud/hooks/mlengine.py | 12 +-
.../google/cloud/hooks/natural_language.py | 6 +-
airflow/providers/google/cloud/hooks/os_login.py | 16 +-
airflow/providers/google/cloud/hooks/pubsub.py | 24 +-
airflow/providers/google/cloud/hooks/spanner.py | 12 +-
.../providers/google/cloud/hooks/speech_to_text.py | 6 +-
.../providers/google/cloud/hooks/stackdriver.py | 6 +-
airflow/providers/google/cloud/hooks/tasks.py | 8 +-
.../providers/google/cloud/hooks/text_to_speech.py | 6 +-
.../google/cloud/hooks/vertex_ai/auto_ml.py | 12 +-
.../cloud/hooks/vertex_ai/batch_prediction_job.py | 10 +-
.../google/cloud/hooks/vertex_ai/custom_job.py | 20 +-
.../google/cloud/hooks/vertex_ai/dataset.py | 22 +-
.../cloud/hooks/vertex_ai/endpoint_service.py | 14 +-
.../hooks/vertex_ai/hyperparameter_tuning_job.py | 10 +-
.../google/cloud/hooks/vertex_ai/model_service.py | 12 +-
.../google/cloud/hooks/video_intelligence.py | 8 +-
airflow/providers/google/cloud/hooks/vision.py | 8 +-
airflow/providers/google/cloud/hooks/workflows.py | 14 +-
airflow/providers/google/cloud/links/bigquery.py | 2 +-
.../providers/google/cloud/links/bigquery_dts.py | 2 +-
.../google/cloud/links/cloud_functions.py | 2 +-
.../google/cloud/links/cloud_memorystore.py | 2 +-
airflow/providers/google/cloud/links/cloud_sql.py | 2 +-
.../providers/google/cloud/links/cloud_tasks.py | 2 +-
airflow/providers/google/cloud/links/compute.py | 2 +-
.../providers/google/cloud/links/datacatalog.py | 2 +-
airflow/providers/google/cloud/links/dataflow.py | 2 +-
airflow/providers/google/cloud/links/dataform.py | 2 +-
airflow/providers/google/cloud/links/pubsub.py | 2 +-
airflow/providers/google/cloud/links/spanner.py | 2 +-
.../providers/google/cloud/links/stackdriver.py | 2 +-
airflow/providers/google/cloud/links/workflows.py | 2 +-
.../providers/google/cloud/log/gcs_task_handler.py | 2 +-
.../google/cloud/log/stackdriver_task_handler.py | 12 +-
airflow/providers/google/cloud/operators/automl.py | 3 +-
.../providers/google/cloud/operators/bigquery.py | 2 +-
.../google/cloud/operators/bigquery_dts.py | 3 +-
.../providers/google/cloud/operators/bigtable.py | 8 +-
.../google/cloud/operators/cloud_batch.py | 9 +-
.../google/cloud/operators/cloud_build.py | 3 +-
.../google/cloud/operators/cloud_composer.py | 5 +-
.../google/cloud/operators/cloud_memorystore.py | 5 +-
.../operators/{cloud_batch.py => cloud_run.py} | 287 ++--
.../providers/google/cloud/operators/cloud_sql.py | 309 ++--
.../providers/google/cloud/operators/compute.py | 75 +-
.../google/cloud/operators/datacatalog.py | 5 +-
.../providers/google/cloud/operators/dataflow.py | 4 +-
.../providers/google/cloud/operators/dataform.py | 3 +-
.../providers/google/cloud/operators/datafusion.py | 2 +-
.../providers/google/cloud/operators/dataplex.py | 3 +-
.../providers/google/cloud/operators/dataproc.py | 10 +-
.../google/cloud/operators/dataproc_metastore.py | 3 +-
airflow/providers/google/cloud/operators/dlp.py | 5 +-
.../providers/google/cloud/operators/functions.py | 92 +-
.../google/cloud/operators/kubernetes_engine.py | 3 +-
.../providers/google/cloud/operators/mlengine.py | 4 +-
.../google/cloud/operators/natural_language.py | 5 +-
airflow/providers/google/cloud/operators/pubsub.py | 3 +-
.../providers/google/cloud/operators/spanner.py | 2 +-
.../google/cloud/operators/speech_to_text.py | 5 +-
.../google/cloud/operators/stackdriver.py | 3 +-
airflow/providers/google/cloud/operators/tasks.py | 5 +-
.../google/cloud/operators/text_to_speech.py | 5 +-
.../google/cloud/operators/translate_speech.py | 3 +-
.../google/cloud/operators/vertex_ai/auto_ml.py | 3 +-
.../operators/vertex_ai/batch_prediction_job.py | 5 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 3 +-
.../google/cloud/operators/vertex_ai/dataset.py | 5 +-
.../cloud/operators/vertex_ai/endpoint_service.py | 5 +-
.../vertex_ai/hyperparameter_tuning_job.py | 5 +-
.../cloud/operators/vertex_ai/model_service.py | 3 +-
.../google/cloud/operators/video_intelligence.py | 3 +-
airflow/providers/google/cloud/operators/vision.py | 5 +-
.../providers/google/cloud/operators/workflows.py | 5 +-
.../providers/google/cloud/sensors/bigquery_dts.py | 3 +-
airflow/providers/google/cloud/sensors/dataplex.py | 3 +-
.../google/cloud/sensors/dataproc_metastore.py | 4 +-
airflow/providers/google/cloud/sensors/gcs.py | 3 +-
airflow/providers/google/cloud/sensors/tasks.py | 2 +-
.../providers/google/cloud/sensors/workflows.py | 3 +-
.../google/cloud/transfers/bigquery_to_gcs.py | 3 +-
.../google/cloud/transfers/bigquery_to_sql.py | 2 +-
.../google/cloud/transfers/calendar_to_gcs.py | 6 +-
.../google/cloud/transfers/cassandra_to_gcs.py | 6 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 8 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 3 +-
.../google/cloud/transfers/presto_to_gcs.py | 9 +-
.../providers/google/cloud/transfers/sql_to_gcs.py | 2 +-
.../google/cloud/transfers/trino_to_gcs.py | 9 +-
.../google/cloud/triggers/bigquery_dts.py | 2 +-
.../providers/google/cloud/triggers/cloud_run.py | 143 ++
.../triggers/cloud_storage_transfer_service.py | 2 +-
.../google/cloud/triggers/kubernetes_engine.py | 6 +-
airflow/providers/google/cloud/triggers/pubsub.py | 4 +-
.../google/cloud/utils/bigquery_get_data.py | 9 +-
.../google/cloud/utils/field_validator.py | 26 +-
.../google/cloud/utils/mlengine_operator_utils.py | 6 +-
.../providers/google/common/hooks/base_google.py | 6 +-
airflow/providers/google/common/links/storage.py | 2 +-
.../google/common/utils/id_token_credentials.py | 5 +-
.../marketing_platform/hooks/campaign_manager.py | 6 +-
airflow/providers/google/provider.yaml | 19 +-
airflow/providers/google/suite/hooks/calendar.py | 6 +-
airflow/providers/grpc/CHANGELOG.rst | 17 +
airflow/providers/grpc/__init__.py | 6 +-
airflow/providers/grpc/hooks/grpc.py | 2 +-
airflow/providers/grpc/provider.yaml | 1 +
.../hashicorp/.latest-doc-only-change.txt | 2 +-
airflow/providers/hashicorp/hooks/vault.py | 8 +-
airflow/providers/http/CHANGELOG.rst | 8 +
airflow/providers/http/__init__.py | 6 +-
airflow/providers/http/operators/http.py | 6 +-
airflow/providers/http/provider.yaml | 1 +
airflow/providers/http/sensors/http.py | 13 +-
airflow/providers/http/triggers/http.py | 6 +-
airflow/providers/imap/CHANGELOG.rst | 8 +
airflow/providers/imap/__init__.py | 2 +-
airflow/providers/imap/hooks/imap.py | 6 +-
airflow/providers/imap/provider.yaml | 1 +
airflow/providers/influxdb/CHANGELOG.rst | 16 +
airflow/providers/influxdb/__init__.py | 6 +-
airflow/providers/influxdb/hooks/influxdb.py | 5 +-
airflow/providers/influxdb/provider.yaml | 1 +
airflow/providers/jdbc/hooks/jdbc.py | 6 +-
.../providers/jenkins/.latest-doc-only-change.txt | 2 +-
airflow/providers/microsoft/azure/CHANGELOG.rst | 30 +
airflow/providers/microsoft/azure/__init__.py | 2 +-
.../microsoft/azure/example_dag/__init__.py | 16 -
airflow/providers/microsoft/azure/hooks/adx.py | 31 +-
airflow/providers/microsoft/azure/hooks/asb.py | 111 +-
.../providers/microsoft/azure/hooks/base_azure.py | 16 +-
airflow/providers/microsoft/azure/hooks/batch.py | 42 +-
.../microsoft/azure/hooks/container_instance.py | 77 +-
.../microsoft/azure/hooks/container_registry.py | 10 +-
airflow/providers/microsoft/azure/hooks/cosmos.py | 52 +-
.../microsoft/azure/hooks/data_factory.py | 26 +-
.../providers/microsoft/azure/hooks/data_lake.py | 46 +-
airflow/providers/microsoft/azure/hooks/synapse.py | 8 +-
airflow/providers/microsoft/azure/hooks/wasb.py | 13 +-
.../microsoft/azure/log/wasb_task_handler.py | 5 +-
airflow/providers/microsoft/azure/operators/adx.py | 4 +-
airflow/providers/microsoft/azure/operators/asb.py | 10 +-
.../providers/microsoft/azure/operators/batch.py | 7 +-
.../azure/operators/container_instances.py | 3 +-
.../microsoft/azure/operators/data_factory.py | 7 +-
.../providers/microsoft/azure/operators/synapse.py | 13 +-
airflow/providers/microsoft/azure/provider.yaml | 11 +-
.../microsoft/azure/sensors/data_factory.py | 7 +-
.../azure/transfers/oracle_to_azure_data_lake.py | 2 +-
.../microsoft/azure/triggers/data_factory.py | 2 +-
airflow/providers/microsoft/azure/utils.py | 50 +
airflow/providers/microsoft/psrp/operators/psrp.py | 3 +-
airflow/providers/microsoft/winrm/CHANGELOG.rst | 16 +
airflow/providers/microsoft/winrm/__init__.py | 6 +-
.../providers/microsoft/winrm/operators/winrm.py | 2 +-
airflow/providers/microsoft/winrm/provider.yaml | 1 +
.../providers/mongo/.latest-doc-only-change.txt | 2 +-
airflow/providers/mongo/hooks/mongo.py | 9 +-
airflow/providers/mysql/CHANGELOG.rst | 9 +
airflow/providers/mysql/__init__.py | 2 +-
airflow/providers/mysql/hooks/mysql.py | 3 +-
airflow/providers/mysql/provider.yaml | 1 +
.../providers/mysql/transfers/vertica_to_mysql.py | 59 +-
airflow/providers/neo4j/hooks/neo4j.py | 6 +-
airflow/providers/odbc/.latest-doc-only-change.txt | 2 +-
.../providers/openfaas/.latest-doc-only-change.txt | 2 +-
airflow/providers/openlineage/CHANGELOG.rst | 17 +
airflow/providers/openlineage/__init__.py | 2 +-
airflow/providers/openlineage/extractors/base.py | 7 +-
airflow/providers/openlineage/plugins/adapter.py | 2 +-
airflow/providers/openlineage/provider.yaml | 1 +
airflow/providers/openlineage/sqlparser.py | 2 +-
airflow/providers/openlineage/utils/sql.py | 17 +-
.../providers/opsgenie/.latest-doc-only-change.txt | 2 +-
airflow/providers/oracle/CHANGELOG.rst | 11 +
airflow/providers/oracle/__init__.py | 2 +-
airflow/providers/oracle/hooks/oracle.py | 6 +-
airflow/providers/oracle/provider.yaml | 1 +
.../providers/oracle/transfers/oracle_to_oracle.py | 2 +-
.../pagerduty/.latest-doc-only-change.txt | 2 +-
.../providers/pagerduty/hooks/pagerduty_events.py | 6 +-
.../papermill/.latest-doc-only-change.txt | 2 +-
airflow/providers/plexus/CHANGELOG.rst | 17 +
airflow/providers/plexus/__init__.py | 6 +-
airflow/providers/plexus/provider.yaml | 1 +
airflow/providers/postgres/hooks/postgres.py | 29 +-
airflow/providers/presto/CHANGELOG.rst | 8 +
airflow/providers/presto/__init__.py | 2 +-
airflow/providers/presto/hooks/presto.py | 6 +-
airflow/providers/presto/provider.yaml | 1 +
.../providers/redis/.latest-doc-only-change.txt | 2 +-
airflow/providers/redis/log/redis_task_handler.py | 10 +-
airflow/providers/redis/provider.yaml | 6 +-
airflow/providers/salesforce/CHANGELOG.rst | 17 +
airflow/providers/salesforce/__init__.py | 6 +-
airflow/providers/salesforce/hooks/salesforce.py | 2 +-
airflow/providers/salesforce/operators/bulk.py | 3 +-
airflow/providers/salesforce/provider.yaml | 1 +
.../providers/samba/.latest-doc-only-change.txt | 2 +-
airflow/providers/samba/hooks/samba.py | 5 +-
.../providers/segment/.latest-doc-only-change.txt | 2 +-
airflow/providers/sendgrid/CHANGELOG.rst | 16 +
airflow/providers/sendgrid/__init__.py | 6 +-
airflow/providers/sendgrid/provider.yaml | 1 +
airflow/providers/sftp/CHANGELOG.rst | 18 +
airflow/providers/sftp/__init__.py | 2 +-
airflow/providers/sftp/hooks/sftp.py | 7 +-
airflow/providers/sftp/operators/sftp.py | 2 +-
airflow/providers/sftp/provider.yaml | 1 +
airflow/providers/sftp/sensors/sftp.py | 2 +-
airflow/providers/singularity/CHANGELOG.rst | 16 +
airflow/providers/singularity/__init__.py | 6 +-
.../providers/singularity/operators/singularity.py | 2 +-
airflow/providers/singularity/provider.yaml | 1 +
airflow/providers/slack/CHANGELOG.rst | 45 +
airflow/providers/slack/__init__.py | 2 +-
airflow/providers/slack/hooks/slack.py | 93 +-
airflow/providers/slack/hooks/slack_webhook.py | 232 +--
airflow/providers/slack/operators/slack.py | 10 +-
airflow/providers/slack/operators/slack_webhook.py | 70 +-
airflow/providers/slack/provider.yaml | 1 +
airflow/providers/slack/transfers/sql_to_slack.py | 17 +-
airflow/providers/smtp/CHANGELOG.rst | 8 +
airflow/providers/smtp/__init__.py | 2 +-
airflow/providers/smtp/hooks/smtp.py | 6 +-
airflow/providers/smtp/operators/smtp.py | 6 +-
airflow/providers/smtp/provider.yaml | 1 +
airflow/providers/snowflake/CHANGELOG.rst | 11 +
airflow/providers/snowflake/__init__.py | 2 +-
airflow/providers/snowflake/hooks/snowflake.py | 4 +-
.../providers/snowflake/hooks/snowflake_sql_api.py | 11 +-
airflow/providers/snowflake/operators/snowflake.py | 3 +
airflow/providers/snowflake/provider.yaml | 1 +
.../snowflake/triggers/snowflake_trigger.py | 6 +-
airflow/providers/ssh/CHANGELOG.rst | 17 +
airflow/providers/ssh/__init__.py | 6 +-
airflow/providers/ssh/hooks/ssh.py | 45 +-
airflow/providers/ssh/provider.yaml | 1 +
.../providers/tableau/.latest-doc-only-change.txt | 2 +-
airflow/providers/tableau/hooks/tableau.py | 6 +-
.../providers/tabular/.latest-doc-only-change.txt | 2 +-
.../providers/telegram/.latest-doc-only-change.txt | 2 +-
airflow/providers/trino/CHANGELOG.rst | 13 +
airflow/providers/trino/__init__.py | 2 +-
airflow/providers/trino/hooks/trino.py | 35 +-
airflow/providers/trino/provider.yaml | 1 +
airflow/providers/yandex/CHANGELOG.rst | 22 +
airflow/providers/yandex/__init__.py | 15 +-
.../yandex/operators/yandexcloud_dataproc.py | 5 +
airflow/providers/yandex/provider.yaml | 1 +
.../providers/zendesk/.latest-doc-only-change.txt | 2 +-
airflow/providers/zendesk/hooks/zendesk.py | 10 +-
airflow/providers_manager.py | 2 +-
airflow/secrets/metastore.py | 3 +-
airflow/sensors/__init__.py | 56 +-
airflow/sensors/base.py | 13 +-
airflow/sensors/bash.py | 87 +-
airflow/sensors/date_time.py | 12 +-
airflow/sensors/filesystem.py | 8 +-
airflow/sensors/python.py | 7 +-
airflow/sensors/time_delta.py | 16 +-
airflow/sensors/time_sensor.py | 8 +-
airflow/sensors/weekday.py | 6 +-
airflow/sentry.py | 2 +-
airflow/serialization/serde.py | 6 +-
airflow/serialization/serialized_objects.py | 33 +-
airflow/settings.py | 1 -
airflow/task/task_runner/__init__.py | 7 +-
airflow/task/task_runner/base_task_runner.py | 6 +-
airflow/task/task_runner/cgroup_task_runner.py | 7 +-
airflow/task/task_runner/standard_task_runner.py | 5 +-
airflow/template/templater.py | 2 +-
airflow/ti_deps/dep_context.py | 3 +-
airflow/ti_deps/deps/prev_dagrun_dep.py | 3 +-
airflow/ti_deps/deps/trigger_rule_dep.py | 5 +-
airflow/timetables/events.py | 2 +-
airflow/triggers/file.py | 2 +-
airflow/typing_compat.py | 8 +-
airflow/utils/airflow_flask_app.py | 7 +-
airflow/utils/cli.py | 3 +-
airflow/utils/code_utils.py | 28 +-
airflow/utils/dag_edges.py | 8 +-
airflow/utils/dates.py | 8 +-
airflow/utils/db.py | 12 +-
airflow/utils/db_cleanup.py | 26 +-
airflow/utils/dot_renderer.py | 14 +-
airflow/utils/helpers.py | 2 +-
airflow/utils/log/__init__.py | 30 +-
airflow/utils/log/colored_log.py | 6 +-
airflow/utils/log/file_task_handler.py | 2 +-
airflow/utils/log/json_formatter.py | 2 +-
airflow/utils/log/log_reader.py | 10 +-
airflow/utils/log/logging_mixin.py | 7 +-
airflow/utils/log/secrets_masker.py | 5 +-
airflow/utils/log/timezone_aware.py | 6 +-
airflow/utils/log/trigger_handler.py | 4 +-
airflow/utils/mixins.py | 5 +-
airflow/utils/module_loading.py | 6 +-
airflow/utils/session.py | 6 +-
airflow/utils/sqlalchemy.py | 8 +-
airflow/utils/state.py | 12 +-
airflow/utils/task_group.py | 3 +-
airflow/www/api/experimental/endpoints.py | 7 +-
airflow/www/extensions/init_appbuilder.py | 14 +-
airflow/www/extensions/init_auth_manager.py | 20 +-
airflow/www/extensions/init_manifest_files.py | 2 +-
airflow/www/extensions/init_views.py | 6 +-
airflow/www/extensions/init_wsgi_middlewares.py | 2 +-
airflow/www/fab_security/manager.py | 14 +-
airflow/www/jest-setup.js | 1 -
airflow/www/security.py | 27 +-
airflow/www/utils.py | 12 +-
airflow/www/views.py | 37 +-
chart/Chart.lock | 6 +-
chart/Chart.yaml | 2 +-
chart/charts/postgresql-12.1.9.tgz | Bin 57966 -> 0 bytes
chart/charts/postgresql-12.10.0.tgz | Bin 0 -> 61605 bytes
chart/newsfragments/33747.significant.rst | 3 +
chart/newsfragments/33748.significant.rst | 3 +
.../templates/pgbouncer/pgbouncer-deployment.yaml | 3 +
chart/values.schema.json | 10 +-
chart/values.yaml | 5 +-
dev/README_RELEASE_PROVIDER_PACKAGES.md | 6 +-
dev/assign_cherry_picked_prs_with_milestone.py | 4 +-
.../airflow_breeze/commands/ci_image_commands.py | 2 +
.../commands/ci_image_commands_config.py | 1 +
.../airflow_breeze/commands/developer_commands.py | 2 +-
.../commands/production_image_commands.py | 83 +-
.../commands/production_image_commands_config.py | 4 +-
.../commands/release_management_commands.py | 113 +-
.../commands/release_management_commands_config.py | 10 +
.../src/airflow_breeze/commands/setup_commands.py | 2 +-
dev/breeze/src/airflow_breeze/global_constants.py | 5 +-
.../src/airflow_breeze/params/build_ci_params.py | 5 +-
.../src/airflow_breeze/params/build_prod_params.py | 7 +-
.../airflow_breeze/params/common_build_params.py | 14 +-
.../airflow_breeze/utils/add_back_references.py | 64 +-
.../src/airflow_breeze/utils/backtracking.py | 2 +-
dev/breeze/src/airflow_breeze/utils/cdxgen.py | 8 +-
.../src/airflow_breeze/utils/common_options.py | 19 +-
dev/breeze/src/airflow_breeze/utils/docs_errors.py | 6 +-
dev/breeze/src/airflow_breeze/utils/github.py | 2 +-
.../src/airflow_breeze/utils/helm_chart_utils.py | 6 +-
.../src/airflow_breeze/utils/kubernetes_utils.py | 6 +-
dev/breeze/src/airflow_breeze/utils/parallel.py | 9 +-
.../airflow_breeze/utils/publish_docs_builder.py | 49 +-
.../airflow_breeze/utils/publish_docs_helpers.py | 2 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 14 +-
.../src/airflow_breeze/utils/selective_checks.py | 31 +-
.../src/airflow_breeze/utils/spelling_checks.py | 4 +-
.../airflow_breeze/utils/suspended_providers.py | 4 +-
dev/breeze/tests/test_provider_dependencies.py | 4 +-
dev/breeze/tests/test_run_utils.py | 16 +-
dev/breeze/tests/test_selective_checks.py | 21 +-
dev/check_files.py | 2 +-
dev/perf/scheduler_dag_execution_timing.py | 31 +-
dev/perf/sql_queries.py | 2 +-
dev/prepare_release_issue.py | 11 +-
dev/provider_packages/prepare_provider_packages.py | 5 +-
dev/system_tests/update_issue_status.py | 16 +-
dev/validate_version_added_fields_in_config.py | 5 +-
docs/apache-airflow-providers-alibaba/commits.rst | 22 +-
docs/apache-airflow-providers-alibaba/index.rst | 2 +-
docs/apache-airflow-providers-amazon/commits.rst | 36 +-
.../connections/aws.rst | 4 +-
docs/apache-airflow-providers-amazon/index.rst | 17 +-
.../commits.rst | 17 +-
.../apache-airflow-providers-apache-beam/index.rst | 6 +-
.../changelog.rst | 2 -
.../commits.rst | 16 +
.../index.rst | 6 +-
.../commits.rst | 24 +-
.../apache-airflow-providers-apache-hive/index.rst | 15 +-
.../commits.rst | 17 +-
.../apache-airflow-providers-apache-livy/index.rst | 6 +-
.../commits.rst | 15 +-
.../index.rst | 6 +-
.../commits.rst | 15 +-
.../index.rst | 6 +-
.../changelog.rst | 2 -
docs/apache-airflow-providers-arangodb/commits.rst | 19 +
docs/apache-airflow-providers-arangodb/index.rst | 2 +-
docs/apache-airflow-providers-celery/commits.rst | 16 +-
docs/apache-airflow-providers-celery/index.rst | 8 +-
.../commits.rst | 30 +-
.../index.rst | 2 +-
.../commits.rst | 16 +-
docs/apache-airflow-providers-common-sql/index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 2 +-
.../commits.rst | 20 +-
docs/apache-airflow-providers-databricks/index.rst | 14 +-
.../apache-airflow-providers-datadog/changelog.rst | 2 -
docs/apache-airflow-providers-datadog/commits.rst | 18 +
docs/apache-airflow-providers-datadog/index.rst | 2 +-
.../apache-airflow-providers-dbt-cloud/commits.rst | 15 +-
docs/apache-airflow-providers-dbt-cloud/index.rst | 6 +-
docs/apache-airflow-providers-docker/commits.rst | 22 +-
docs/apache-airflow-providers-docker/index.rst | 2 +-
.../commits.rst | 18 +-
.../index.rst | 6 +-
docs/apache-airflow-providers-exasol/commits.rst | 14 +-
docs/apache-airflow-providers-exasol/index.rst | 6 +-
docs/apache-airflow-providers-ftp/commits.rst | 15 +-
docs/apache-airflow-providers-ftp/index.rst | 6 +-
docs/apache-airflow-providers-github/changelog.rst | 2 -
docs/apache-airflow-providers-github/commits.rst | 18 +
docs/apache-airflow-providers-github/index.rst | 2 +-
docs/apache-airflow-providers-google/commits.rst | 35 +
docs/apache-airflow-providers-google/index.rst | 8 +-
.../operators/cloud/cloud_run.rst | 128 ++
docs/apache-airflow-providers-grpc/changelog.rst | 2 -
docs/apache-airflow-providers-grpc/commits.rst | 18 +
docs/apache-airflow-providers-grpc/index.rst | 2 +-
docs/apache-airflow-providers-http/commits.rst | 14 +-
docs/apache-airflow-providers-http/index.rst | 2 +-
docs/apache-airflow-providers-imap/commits.rst | 14 +-
docs/apache-airflow-providers-imap/index.rst | 2 +-
.../changelog.rst | 2 -
docs/apache-airflow-providers-influxdb/commits.rst | 17 +
docs/apache-airflow-providers-influxdb/index.rst | 2 +-
.../commits.rst | 25 +-
.../index.rst | 9 +-
.../operators/adls.rst | 2 +-
.../operators/{adls.rst => batch.rst} | 25 +-
.../sensors/wasb_sensors.rst | 4 +-
.../transfer/local_to_adls.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../index.rst | 2 +-
docs/apache-airflow-providers-mysql/commits.rst | 15 +-
docs/apache-airflow-providers-mysql/index.rst | 6 +-
.../commits.rst | 19 +-
.../apache-airflow-providers-openlineage/index.rst | 6 +-
docs/apache-airflow-providers-oracle/commits.rst | 17 +-
docs/apache-airflow-providers-oracle/index.rst | 6 +-
docs/apache-airflow-providers-plexus/changelog.rst | 2 -
docs/apache-airflow-providers-plexus/commits.rst | 18 +
docs/apache-airflow-providers-plexus/index.rst | 2 +-
docs/apache-airflow-providers-presto/commits.rst | 14 +-
docs/apache-airflow-providers-presto/index.rst | 6 +-
.../operators/transfer/gcs_to_presto.rst | 6 -
.../operators/qubole.rst | 5 -
.../operators/qubole_check.rst | 5 -
.../changelog.rst | 2 -
.../commits.rst | 18 +
docs/apache-airflow-providers-salesforce/index.rst | 2 +-
.../changelog.rst | 2 -
docs/apache-airflow-providers-sendgrid/commits.rst | 17 +
docs/apache-airflow-providers-sendgrid/index.rst | 2 +-
docs/apache-airflow-providers-sftp/commits.rst | 16 +-
docs/apache-airflow-providers-sftp/index.rst | 7 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../apache-airflow-providers-singularity/index.rst | 2 +-
docs/apache-airflow-providers-slack/commits.rst | 17 +-
docs/apache-airflow-providers-slack/index.rst | 6 +-
.../operators/slack_operator_howto_guide.rst | 2 +-
docs/apache-airflow-providers-smtp/commits.rst | 14 +-
docs/apache-airflow-providers-smtp/index.rst | 2 +-
.../apache-airflow-providers-snowflake/commits.rst | 16 +-
docs/apache-airflow-providers-snowflake/index.rst | 7 +-
docs/apache-airflow-providers-ssh/changelog.rst | 2 -
docs/apache-airflow-providers-ssh/commits.rst | 18 +
docs/apache-airflow-providers-ssh/index.rst | 2 +-
docs/apache-airflow-providers-trino/commits.rst | 15 +-
docs/apache-airflow-providers-trino/index.rst | 19 +-
.../operators/transfer/gcs_to_trino.rst | 6 -
docs/apache-airflow-providers-yandex/commits.rst | 31 +-
docs/apache-airflow-providers-yandex/index.rst | 10 +-
docs/apache-airflow/core-concepts/operators.rst | 33 +
docs/apache-airflow/deprecated-rest-api-ref.rst | 2 +-
docs/apache-airflow/howto/operator/file.rst | 2 +-
docs/apache-airflow/img/task_lifecycle_diagram.png | Bin 58659 -> 32164 bytes
docs/apache-airflow/security/access-control.rst | 44 +
docs/apache-airflow/security/webserver.rst | 12 +-
docs/conf.py | 3 +-
docs/exts/docs_build/code_utils.py | 2 +-
docs/exts/docs_build/errors.py | 2 +-
docs/exts/docs_build/spelling_checks.py | 2 +-
docs/spelling_wordlist.txt | 3 +
generated/provider_dependencies.json | 29 +-
helm_tests/airflow_aux/test_configmap.py | 14 +-
helm_tests/other/test_pgbouncer.py | 18 +
images/breeze/output-commands-hash.txt | 54 +-
images/breeze/output_ci-image_build.svg | 140 +-
images/breeze/output_ci-image_pull.svg | 2 +-
images/breeze/output_k8s_build-k8s-image.svg | 2 +-
images/breeze/output_k8s_configure-cluster.svg | 6 +-
images/breeze/output_k8s_create-cluster.svg | 6 +-
images/breeze/output_k8s_delete-cluster.svg | 2 +-
images/breeze/output_k8s_deploy-airflow.svg | 6 +-
images/breeze/output_k8s_k9s.svg | 2 +-
images/breeze/output_k8s_logs.svg | 2 +-
images/breeze/output_k8s_run-complete-tests.svg | 6 +-
images/breeze/output_k8s_shell.svg | 2 +-
images/breeze/output_k8s_status.svg | 2 +-
images/breeze/output_k8s_tests.svg | 6 +-
images/breeze/output_k8s_upload-k8s-image.svg | 6 +-
images/breeze/output_prod-image_build.svg | 126 +-
images/breeze/output_prod-image_pull.svg | 2 +-
...put_release-management_generate-constraints.svg | 2 +-
...elease-management_install-provider-packages.svg | 2 +-
.../output_release-management_publish-docs.svg | 54 +-
.../breeze/output_sbom_update-sbom-information.svg | 2 +-
images/breeze/output_testing_tests.svg | 2 +-
kubernetes_tests/test_base.py | 4 +-
kubernetes_tests/test_kubernetes_pod_operator.py | 18 +-
.../pre_commit_check_provider_yaml_files.py | 2 +
.../pre_commit_compile_www_assets_dev.py | 5 +-
.../pre_commit_replace_bad_characters.py | 8 +-
.../ci/pre_commit/pre_commit_sort_in_the_wild.py | 2 +-
.../ci/pre_commit/pre_commit_unittest_testcase.py | 12 +-
.../pre_commit_update_common_sql_api_stubs.py | 12 +-
.../ci/pre_commit/pre_commit_version_heads_map.py | 3 +
.../docker/install_from_docker_context_files.sh | 45 +-
scripts/in_container/_in_container_utils.sh | 16 +
.../in_container/run_provider_yaml_files_check.py | 424 ++++--
scripts/tools/generate-integrations-json.py | 2 +-
tests/always/test_example_dags.py | 2 +-
.../endpoints/test_connection_endpoint.py | 2 +-
.../endpoints/test_dag_run_endpoint.py | 16 +-
.../endpoints/test_event_log_endpoint.py | 2 +-
.../test_mapped_task_instance_endpoint.py | 27 +-
.../api_connexion/endpoints/test_user_endpoint.py | 2 +-
.../common/experimental/test_pool.py | 2 +-
.../auth/managers/fab/cli_commands}/__init__.py | 0
.../fab/cli_commands}/test_role_command.py | 10 +-
.../fab/cli_commands}/test_sync_perm_command.py | 6 +-
.../fab/cli_commands}/test_user_command.py | 8 +-
tests/auth/managers/fab/test_fab_auth_manager.py | 8 +-
tests/cli/commands/test_celery_command.py | 25 +-
tests/cli/commands/test_cheat_sheet_command.py | 2 +-
tests/cli/commands/test_config_command.py | 24 +-
tests/cli/commands/test_dag_command.py | 15 +-
tests/cli/commands/test_internal_api_command.py | 124 +-
tests/cli/commands/test_jobs_command.py | 4 +-
tests/cli/commands/test_kubernetes_command.py | 38 +-
tests/cli/commands/test_task_command.py | 92 +-
tests/cli/commands/test_variable_command.py | 41 +-
tests/cli/commands/test_webserver_command.py | 53 +-
tests/cli/test_cli_parser.py | 2 +-
tests/core/test_configuration.py | 4 +-
tests/dag_processing/test_job_runner.py | 106 +-
tests/dags/test_example_bash_operator.py | 2 +-
tests/dags/test_miscellaneous.py | 2 +-
tests/dags_corrupted/test_impersonation_custom.py | 2 +-
tests/decorators/test_setup_teardown.py | 2 +-
.../providers/trino/hooks/test_trino.py | 11 +
tests/jobs/test_scheduler_job.py | 36 +-
tests/models/test_cleartasks.py | 4 +-
tests/models/test_dag.py | 2 +-
tests/models/test_dagrun.py | 6 +-
tests/models/test_taskinstance.py | 27 +
tests/operators/test_bash.py | 2 +-
tests/plugins/test_plugin_ignore.py | 103 +-
tests/plugins/test_plugins_manager.py | 11 +-
tests/providers/amazon/aws/hooks/test_s3.py | 214 ++-
.../amazon/aws/log/test_cloudwatch_task_handler.py | 24 +-
tests/providers/amazon/aws/operators/test_ecs.py | 111 +-
tests/providers/amazon/aws/utils/eks_test_utils.py | 8 +-
.../providers/amazon/aws/utils/test_identifiers.py | 74 +
.../amazon/aws/waiters/test_custom_waiters.py | 56 +
.../apache/kylin/operators/test_kylin_cube.py | 4 +-
.../cncf/kubernetes/models/test_secret.py | 4 +-
.../cncf/kubernetes/operators/test_pod.py | 90 +-
.../cncf/kubernetes/utils/test_pod_manager.py | 73 +
.../databricks/operators/test_databricks_sql.py | 3 +-
.../log/elasticmock/utilities/__init__.py | 2 +-
.../providers/google/cloud/hooks/test_cloud_run.py | 273 ++++
.../google/cloud/hooks/test_datacatalog.py | 6 +-
.../providers/google/cloud/hooks/test_os_login.py | 8 +-
.../google/cloud/operators/test_cloud_run.py | 308 ++++
.../google/cloud/operators/test_datacatalog.py | 6 +-
.../cloud/operators/test_dataproc_metastore.py | 5 +-
.../google/cloud/triggers/test_cloud_run.py | 155 ++
tests/providers/google/cloud/utils/airflow_util.py | 5 +-
tests/providers/http/sensors/test_http.py | 54 +-
tests/providers/microsoft/azure/hooks/test_adx.py | 278 ++--
tests/providers/microsoft/azure/hooks/test_asb.py | 66 +-
.../microsoft/azure/hooks/test_azure_batch.py | 55 +-
.../azure/hooks/test_azure_container_instance.py | 25 +-
.../azure/hooks/test_azure_container_registry.py | 15 +-
.../azure/hooks/test_azure_container_volume.py | 31 +-
.../microsoft/azure/hooks/test_azure_cosmos.py | 21 +-
.../azure/hooks/test_azure_data_factory.py | 232 ++-
.../microsoft/azure/hooks/test_azure_data_lake.py | 67 +-
.../microsoft/azure/hooks/test_azure_fileshare.py | 34 +-
.../microsoft/azure/hooks/test_azure_synapse.py | 83 +-
.../microsoft/azure/hooks/test_base_azure.py | 72 +-
tests/providers/microsoft/azure/hooks/test_wasb.py | 595 ++++----
.../microsoft/azure/operators/test_azure_batch.py | 32 +-
.../operators/test_azure_container_instances.py | 106 +-
.../microsoft/azure/operators/test_azure_cosmos.py | 16 +-
.../azure/operators/test_azure_data_factory.py | 15 +-
.../azure/operators/test_azure_synapse.py | 21 +-
.../providers/microsoft/azure/sensors/test_wasb.py | 5 +-
tests/providers/microsoft/azure/test_utils.py | 21 +-
.../transfers/test_oracle_to_azure_data_lake.py | 33 +-
tests/providers/microsoft/conftest.py | 68 +
tests/providers/mongo/hooks/test_mongo.py | 5 +-
tests/providers/openlineage/utils/test_sql.py | 24 +-
tests/providers/oracle/operators/test_oracle.py | 2 +-
tests/providers/slack/hooks/test_slack.py | 61 +-
tests/providers/slack/hooks/test_slack_webhook.py | 169 +--
tests/providers/slack/operators/test_slack.py | 53 +-
.../slack/operators/test_slack_webhook.py | 71 +-
.../providers/slack/transfers/test_sql_to_slack.py | 20 +-
tests/providers/snowflake/hooks/test_snowflake.py | 6 +-
.../snowflake/hooks/test_snowflake_sql_api.py | 6 +-
tests/providers/ssh/hooks/test_ssh.py | 2 +-
tests/providers/trino/operators/test_trino.py | 64 +
.../yandex/operators/test_yandexcloud_dataproc.py | 1 +
tests/sensors/test_base.py | 30 +-
tests/sensors/test_external_task_sensor.py | 2 +-
.../providers/amazon/aws/example_ecs_fargate.py | 2 -
.../apache/kafka/example_dag_event_listener.py | 3 +-
tests/system/providers/github/example_github.py | 4 +-
.../providers/google/cloud/cloud_run}/__init__.py | 0
.../google/cloud/cloud_run/example_cloud_run.py | 261 ++++
.../azure/example_azure_batch_operator.py | 61 +
.../microsoft/azure/example_azure_service_bus.py | 2 +-
.../microsoft/azure}/example_wasb_sensors.py | 6 +
.../task/task_runner/test_standard_task_runner.py | 65 +-
tests/test_utils/azure_system_helpers.py | 2 +-
tests/test_utils/mock_executor.py | 3 +-
tests/test_utils/terraform.py | 2 +-
tests/utils/test_db_cleanup.py | 3 +-
tests/utils/test_edgemodifier.py | 2 +-
tests/utils/test_sqlalchemy.py | 13 +-
tests/www/test_security.py | 134 ++
tests/www/views/test_views_cluster_activity.py | 2 -
tests/www/views/test_views_home.py | 2 +-
983 files changed, 11908 insertions(+), 7403 deletions(-)
copy airflow/{api/auth => auth/managers/fab/cli_commands}/__init__.py (100%)
create mode 100644 airflow/auth/managers/fab/cli_commands/definition.py
rename airflow/{cli/commands => auth/managers/fab/cli_commands}/role_command.py (93%)
rename airflow/{cli/commands => auth/managers/fab/cli_commands}/sync_perm_command.py (94%)
rename airflow/{cli/commands => auth/managers/fab/cli_commands}/user_command.py (94%)
rename airflow/{utils/cli_app_builder.py => auth/managers/fab/cli_commands/utils.py} (88%)
create mode 100644 airflow/providers/amazon/aws/utils/identifiers.py
create mode 100644 airflow/providers/apache/kafka/.latest-doc-only-change.txt
create mode 100644 airflow/providers/apprise/.latest-doc-only-change.txt
create mode 100644 airflow/providers/google/ads/.gitignore
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/interceptors/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/common/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/common/types/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/enums/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/enums/types/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/errors/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/errors/types/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/resources/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/resources/types/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/services/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/services/customer_service/transports/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/services/google_ads_service/transports/__init__.py
delete mode 100644 airflow/providers/google/ads/_vendor/googleads/v12/services/types/__init__.py
copy airflow/providers/google/cloud/hooks/{cloud_batch.py => cloud_run.py} (56%)
copy airflow/providers/google/cloud/operators/{cloud_batch.py => cloud_run.py} (57%)
create mode 100644 airflow/providers/google/cloud/triggers/cloud_run.py
delete mode 100644 airflow/providers/microsoft/azure/example_dag/__init__.py
delete mode 100644 chart/charts/postgresql-12.1.9.tgz
create mode 100644 chart/charts/postgresql-12.10.0.tgz
create mode 100644 chart/newsfragments/33747.significant.rst
create mode 100644 chart/newsfragments/33748.significant.rst
create mode 100644 docs/apache-airflow-providers-google/operators/cloud/cloud_run.rst
copy docs/apache-airflow-providers-microsoft-azure/operators/{adls.rst => batch.rst} (62%)
rename {airflow/providers/google/ads/_vendor => tests/auth/managers/fab/cli_commands}/__init__.py (100%)
rename tests/{cli/commands => auth/managers/fab/cli_commands}/test_role_command.py (96%)
rename tests/{cli/commands => auth/managers/fab/cli_commands}/test_sync_perm_command.py (89%)
rename tests/{cli/commands => auth/managers/fab/cli_commands}/test_user_command.py (98%)
create mode 100644 tests/providers/amazon/aws/utils/test_identifiers.py
create mode 100644 tests/providers/google/cloud/hooks/test_cloud_run.py
create mode 100644 tests/providers/google/cloud/operators/test_cloud_run.py
create mode 100644 tests/providers/google/cloud/triggers/test_cloud_run.py
create mode 100644 tests/providers/microsoft/conftest.py
rename {airflow/providers/google/ads/_vendor/googleads => tests/system/providers/google/cloud/cloud_run}/__init__.py (100%)
create mode 100644 tests/system/providers/google/cloud/cloud_run/example_cloud_run.py
create mode 100644 tests/system/providers/microsoft/azure/example_azure_batch_operator.py
rename {airflow/providers/microsoft/azure/example_dag => tests/system/providers/microsoft/azure}/example_wasb_sensors.py (91%)
[airflow] 01/01: system tests: implement operator, variable transport
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch openlineage-system-tests
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 25a95e132f83da3200b6d9391f960e1fa0972cd9
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Thu Jun 1 15:12:53 2023 +0200
system tests: implement operator, variable transport
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
.../providers/google/cloud/operators/bigquery.py | 18 +++
airflow/providers/openlineage/plugins/listener.py | 11 +-
airflow/providers/openlineage/provider.yaml | 4 +-
.../providers/openlineage/transport/__init__.py | 16 ++
.../providers/openlineage/transport/variable.py | 51 ++++++
.../apache-airflow-providers-openlineage/index.rst | 4 +-
generated/provider_dependencies.json | 4 +-
tests/system/conftest.py | 14 ++
.../cloud/bigquery/example_bigquery_queries.py | 13 +-
tests/test_utils/openlineage.py | 173 +++++++++++++++++++++
10 files changed, 298 insertions(+), 10 deletions(-)
diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
index ae7e48eed7..04ffa03e9e 100644
--- a/airflow/providers/google/cloud/operators/bigquery.py
+++ b/airflow/providers/google/cloud/operators/bigquery.py
@@ -1074,6 +1074,24 @@ class BigQueryGetDataOperator(GoogleCloudBaseOperator):
self.log.info("Total extracted rows: %s", len(event["records"]))
return event["records"]
+ def get_openlineage_facets_on_start(self):
+ from openlineage.client.run import Dataset
+
+ from airflow.providers.openlineage.extractors import OperatorLineage
+
+ if self.project_id is None:
+ self.project_id = BigQueryHook(
+ gcp_conn_id=self.gcp_conn_id,
+ impersonation_chain=self.impersonation_chain,
+ use_legacy_sql=self.use_legacy_sql,
+ ).project_id
+
+ return OperatorLineage(
+ inputs=[
+ Dataset(namespace="bigquery", name=f"{self.project_id}.{self.dataset_id}.{self.table_id}")
+ ]
+ )
+
class BigQueryExecuteQueryOperator(GoogleCloudBaseOperator):
"""Executes BigQuery SQL queries in a specific BigQuery database.
diff --git a/airflow/providers/openlineage/plugins/listener.py b/airflow/providers/openlineage/plugins/listener.py
index 4a6b75f677..38156f2d02 100644
--- a/airflow/providers/openlineage/plugins/listener.py
+++ b/airflow/providers/openlineage/plugins/listener.py
@@ -46,6 +46,7 @@ class OpenLineageListener:
self.log = logging.getLogger(__name__)
self.extractor_manager = ExtractorManager()
self.adapter = OpenLineageAdapter()
+ self.current_ti: TaskInstance | None = None
@hookimpl
def on_task_instance_running(
@@ -59,6 +60,7 @@ class OpenLineageListener:
return
self.log.debug("OpenLineage listener got notification about task instance start")
+ self.current_ti = task_instance
dagrun = task_instance.dag_run
task = task_instance.task
dag = task.dag
@@ -101,12 +103,13 @@ class OpenLineageListener:
**get_airflow_run_facet(dagrun, dag, task_instance, task, task_uuid),
},
)
-
on_running()
+
@hookimpl
def on_task_instance_success(self, previous_state, task_instance: TaskInstance, session):
self.log.debug("OpenLineage listener got notification about task instance success")
+ self.current_ti = task_instance
dagrun = task_instance.dag_run
task = task_instance.task
@@ -135,6 +138,7 @@ class OpenLineageListener:
@hookimpl
def on_task_instance_failed(self, previous_state, task_instance: TaskInstance, session):
self.log.debug("OpenLineage listener got notification about task instance failure")
+ self.current_ti = task_instance
dagrun = task_instance.dag_run
task = task_instance.task
@@ -174,8 +178,9 @@ class OpenLineageListener:
def before_stopping(self, component):
self.log.debug("before_stopping: %s", component.__class__.__name__)
# TODO: configure this with Airflow config
- with timeout(30):
- self.executor.shutdown(wait=True)
+ if self._executor:
+ with timeout(30):
+ self.executor.shutdown(wait=True)
@hookimpl
def on_dag_run_running(self, dag_run: DagRun, msg: str):
diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml
index 72c85a0412..b2d93e14f1 100644
--- a/airflow/providers/openlineage/provider.yaml
+++ b/airflow/providers/openlineage/provider.yaml
@@ -31,8 +31,8 @@ dependencies:
- apache-airflow>=2.7.0
- apache-airflow-providers-common-sql>=1.6.0
- attrs>=22.2
- - openlineage-integration-common>=0.28.0
- - openlineage-python>=0.28.0
+ - openlineage-integration-common>=0.29.2
+ - openlineage-python>=0.29.2
integrations:
- integration-name: OpenLineage
diff --git a/airflow/providers/openlineage/transport/__init__.py b/airflow/providers/openlineage/transport/__init__.py
new file mode 100644
index 0000000000..13a83393a9
--- /dev/null
+++ b/airflow/providers/openlineage/transport/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/airflow/providers/openlineage/transport/variable.py b/airflow/providers/openlineage/transport/variable.py
new file mode 100644
index 0000000000..fc9dd4a630
--- /dev/null
+++ b/airflow/providers/openlineage/transport/variable.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.models import Variable
+from airflow.plugins_manager import AirflowPlugin, plugins
+from airflow.utils.log.logging_mixin import LoggingMixin
+from openlineage.client.run import DatasetEvent, JobEvent, RunEvent
+from openlineage.client.serde import Serde
+from openlineage.client.transport import Transport
+
+
+class VariableTransport(Transport, LoggingMixin):
+ """This transport sends OpenLineage events to Variables.
+ Key schema is <DAG_ID>.<TASK_ID>.event.<EVENT_TYPE>.
+ It's made to be used in system tests, stored data read by OpenLineageTestOperator.
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ ...
+
+ def emit(self, event: RunEvent | DatasetEvent | JobEvent):
+ from airflow.providers.openlineage.plugins.openlineage import OpenLineageProviderPlugin
+
+ plugin: AirflowPlugin | None = next( # type: ignore[assignment]
+ filter(lambda x: isinstance(x, OpenLineageProviderPlugin), plugins) # type: ignore[arg-type]
+ )
+ if not plugin:
+ raise RuntimeError("OpenLineage listener should be set up here")
+
+ listener = plugin.listeners[0] # type: ignore
+ ti = listener.current_ti # type: ignore
+
+ key = f"{ti.dag_id}.{ti.task_id}.event.{event.eventType.value.lower()}" # type: ignore[union-attr]
+ str_event = Serde.to_json(event)
+ Variable.set(key=key, value=str_event)
diff --git a/docs/apache-airflow-providers-openlineage/index.rst b/docs/apache-airflow-providers-openlineage/index.rst
index 5657aee164..f69e0d42b8 100644
--- a/docs/apache-airflow-providers-openlineage/index.rst
+++ b/docs/apache-airflow-providers-openlineage/index.rst
@@ -116,8 +116,8 @@ PIP package Version required
``apache-airflow`` ``>=2.7.0``
``apache-airflow-providers-common-sql`` ``>=1.6.0``
``attrs`` ``>=22.2``
-``openlineage-integration-common`` ``>=0.28.0``
-``openlineage-python`` ``>=0.28.0``
+``openlineage-integration-common`` ``>=0.29.2``
+``openlineage-python`` ``>=0.29.2``
======================================= ==================
Cross provider package dependencies
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index b82c592b4f..c085350690 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -662,8 +662,8 @@
"apache-airflow-providers-common-sql>=1.6.0",
"apache-airflow>=2.7.0",
"attrs>=22.2",
- "openlineage-integration-common>=0.28.0",
- "openlineage-python>=0.28.0"
+ "openlineage-integration-common>=0.29.2",
+ "openlineage-python>=0.29.2"
],
"cross-providers-deps": [
"common.sql"
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 58eca1287c..21bc381c28 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -24,9 +24,23 @@ from unittest import mock
import pytest
+from airflow import plugins_manager
+from airflow.providers.openlineage.plugins.openlineage import OpenLineageProviderPlugin
+
REQUIRED_ENV_VARS = ("SYSTEM_TESTS_ENV_ID",)
+@pytest.fixture(scope="package", autouse=True)
+def setup_openlineage():
+ with mock.patch.dict(
+ "os.environ",
+ AIRFLOW__OPENLINEAGE__TRANSPORT='{"type": "airflow.providers.openlineage.transport.variable'
+ '.VariableTransport"}',
+ ):
+ plugins_manager.register_plugin(OpenLineageProviderPlugin())
+ yield
+
+
@pytest.fixture(scope="package", autouse=True)
def use_debug_executor():
with mock.patch.dict("os.environ", AIRFLOW__CORE__EXECUTOR="DebugExecutor"):
diff --git a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py
index 3ce1bc2801..57d0ace625 100644
--- a/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/cloud/bigquery/example_bigquery_queries.py
@@ -38,6 +38,7 @@ from airflow.providers.google.cloud.operators.bigquery import (
BigQueryValueCheckOperator,
)
from airflow.utils.trigger_rule import TriggerRule
+from tests.test_utils.openlineage import OpenlineageTestOperator
ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
@@ -235,11 +236,21 @@ for index, location in enumerate(locations, 1):
trigger_rule=TriggerRule.ALL_DONE,
)
+ openlineage_test = OpenlineageTestOperator(
+ task_id="openlineage_test",
+ event_templates={
+ f"{DAG_ID}.get_data.event.start": {
+ "eventType": "START",
+ "inputs": [{"namespace": "bigquery", "name": f"{PROJECT_ID}.{DATASET}.{TABLE_1}"}],
+ }
+ },
+ )
+
# TEST SETUP
create_dataset >> [create_table_1, create_table_2]
# TEST BODY
[create_table_1, create_table_2] >> insert_query_job >> [select_query_job, execute_insert_query]
- execute_insert_query >> get_data >> get_data_result >> delete_dataset
+ execute_insert_query >> get_data >> get_data_result >> delete_dataset >> openlineage_test
execute_insert_query >> execute_query_save >> bigquery_execute_multi_query >> delete_dataset
execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset
execute_insert_query >> [column_check, table_check] >> delete_dataset
diff --git a/tests/test_utils/openlineage.py b/tests/test_utils/openlineage.py
new file mode 100644
index 0000000000..5dc7e7c45f
--- /dev/null
+++ b/tests/test_utils/openlineage.py
@@ -0,0 +1,173 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from urllib.parse import urlparse
+
+from jinja2 import Environment
+
+from airflow.models.baseoperator import BaseOperator
+from airflow.models.variable import Variable
+from airflow.utils.context import Context
+
+log = logging.getLogger(__name__)
+
+
+def any(result):
+ return result
+
+
+def is_datetime(result):
+ try:
+ x = parse(result) # noqa
+ return "true"
+ except: # noqa
+ pass
+ return "false"
+
+
+def is_uuid(result):
+ try:
+ uuid.UUID(result) # noqa
+ return "true"
+ except: # noqa
+ pass
+ return "false"
+
+
+def env_var(var: str, default: str | None = None) -> str:
+ """The env_var() function. Return the environment variable named 'var'.
+ If there is no such environment variable set, return the default.
+ If the default is None, raise an exception for an undefined variable.
+ """
+ if var in os.environ:
+ return os.environ[var]
+ elif default is not None:
+ return default
+ else:
+ msg = f"Env var required but not provided: '{var}'"
+ raise Exception(msg)
+
+
+def not_match(result, pattern) -> str:
+ if pattern in result:
+ raise Exception(f"Found {pattern} in {result}")
+ return "true"
+
+
+def url_scheme_authority(url) -> str:
+ parsed = urlparse(url)
+ return f"{parsed.scheme}://{parsed.netloc}"
+
+
+def url_path(url) -> str:
+ return urlparse(url).path
+
+
+def setup_jinja() -> Environment:
+ env = Environment()
+ env.globals["any"] = any
+ env.globals["is_datetime"] = is_datetime
+ env.globals["is_uuid"] = is_uuid
+ env.globals["env_var"] = env_var
+ env.globals["not_match"] = not_match
+ env.filters["url_scheme_authority"] = url_scheme_authority
+ env.filters["url_path"] = url_path
+ return env
+
+
+env = setup_jinja()
+
+
+def match(expected, result) -> bool:
+ """
+ Check if result is "equal" to expected value. Omits keys not specified in expected value
+ and resolves any jinja templates found.
+ """
+ if isinstance(expected, dict):
+ # Take a look only at keys present at expected dictionary
+ for k, v in expected.items():
+ if k not in result:
+ log.error("Key %s not in received event %s\nExpected %s", k, result, expected)
+ return False
+ if not match(v, result[k]):
+ log.error(
+ "For key %s, expected value %s not equals received %s\nExpected: %s, request: %s",
+ k,
+ v,
+ result[k],
+ expected,
+ result,
+ )
+ return False
+ elif isinstance(expected, list):
+ if len(expected) != len(result):
+ log.error("Length does not match: expected %d, result: %d", len(expected), len(result))
+ return False
+ for i, x in enumerate(expected):
+ if not match(x, result[i]):
+ log.error(
+ "List not matched at %d\nexpected:\n%s\nresult: \n%s",
+ i,
+ json.dumps(x),
+ json.dumps(result[i]),
+ )
+ return False
+ elif isinstance(expected, str):
+ if "{{" in expected:
+ # Evaluate jinja: in some cases, we want to check only if key exists, or if
+ # value has the right type
+ rendered = env.from_string(expected).render(result=result)
+ if rendered == "true" or rendered == result:
+ return True
+ log.error("Rendered value %s does not equal 'true' or %s", rendered, result)
+ return False
+ elif expected != result:
+ log.error("Expected value %s does not equal result %s", expected, result)
+ return False
+ elif expected != result:
+ log.error("Object of type %s: %s does not match %s", type(expected), expected, result)
+ return False
+ return True
+
+
+class OpenlineageTestOperator(BaseOperator):
+ """Operator for testing purposes.
+ It compares expected event templates set on initialization with ones emitted by OpenLineage integration
+ and stored in Variables by VariableTransport.
+ :param event_templates: dictionary where key is the key used by VariableTransport in format of
+ <DAG_ID>.<TASK_ID>.event.<EVENT_TYPE>, and value is event template (fragment)
+ that need to be in received events.
+ :raises: ValueError if the received events do not match with expected ones.
+ """
+
+ def __init__(self, event_templates: dict[str, dict], **kwargs):
+ super().__init__(**kwargs)
+ self.event_templates = event_templates
+
+ def execute(self, context: Context):
+ for key, template in self.event_templates.items():
+ send_event = Variable.get(key=key)
+ self.log.error("Events: %s", send_event)
+ if send_event:
+ self.log.error("Events: %s, %s, %s", send_event, len(send_event), type(send_event))
+ if not match(template, json.loads(send_event)):
+ raise ValueError("Event received does not match one specified in test")