You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/07/24 10:35:59 UTC
[airflow] branch load-plugins-from-providers updated (b78c4ceca8 -> e6bd5213ec)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch load-plugins-from-providers
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit b78c4ceca8 plugins: load plugins from providers
add 8b7ae76026 Fixup docs and optimize system test for DataprocSubmitJobOperator (Hadoop job) (#32722)
add e8287734cb Fixup docstring for deprecated DataprocSubmitHiveJobOperator (#32723)
add dda3dcdcfc Add deferrable mode to ExternalTaskSensor (#29260)
add ac524826f3 Refactor setup/teardown ctx mgr to operate freely with other task definitions (#32687)
add 3c14753b03 Fix BigQueryGetDataOperator where project_id is not being respected in deferrable mode (#32488)
add 99b8a90346 Filtering and ordering results of DataprocListBatchesOperator (#32500)
add 8e67546660 Fix DataformCreateWorkflowInvocationOperator system test (#32599)
add 75ed3bc3f8 [bugfix] fix AWS triggers where deserialization would crash if region was not specified (#32729)
add d70fecfaf6 Add initial docs for setup / teardown (#32169)
add 15d42b4320 Reduce default for max TIs per query, enforce <= parallelism (#32572)
add c7c0deecb4 Refactor Sqlalchemy queries to 2.0 style (Part 6) (#32645)
add b09e1f97d5 Fix dagProcessor not including webserver-config volume (#32644)
add d5bf74c7b0 Removing hardcoded flag value and using constant (#32713)
add 978adb309a Install sqlalchemy-spanner package into Google provider (#31925)
add 531eb41bff Metrics - Enabled encrypted OTel Endpoint (#32524)
add 49921763eb Migrate system test for PostgresToGCSOperator to new design AIP-47 (#32641)
add 440c9eb2b4 Fixup system test for DataprocSubmitJobOperator (PySpark job) (#32740)
add fbeddc3017 Fixup docstring for deprecated DataprocSubmitPigJobOperator (#32739)
add 0fbb05a459 Fixup system test for DataprocSubmitJobOperator (SparkSQL job) (#32745)
add 98a999034c Add OpenLineage support for MySQL. (#31609)
add 5b082c38a6 openlineage, snowflake: add OpenLineage support for Snowflake (#31696)
add 00aa6ea72c Fix Pandas2 compatibility for Hive (#32752)
add 8e054bb57d Fix ARM image building after Cython 3.0.0 release (#32748)
add 067365f561 Arjunanan6/add company (#32742)
add 815655101b Add Deferrable mode to StepFunctionStartExecutionOperator (#32563)
add 73b90c48b1 Allow configuration to be contributed by providers (#32604)
add 62fe683f33 provider_info schema correction (#32754)
add d7899ecfaf Bring back automated airflow home dir creation (#32755)
add 3d89e75eb9 Fix failing Celery Executor tests after config migration (#32763)
add 53c6305bd0 Improve getting the query count in Airflow API endpoints (#32630)
add 3e467ba510 Fix prefix group false graph (#32764)
add 4c878798ef Sort extra_links for predictable order in UI. (#32762)
add fcc6f284c7 Update the watcher resource version in SparkK8SOp when it's too old (#32768)
add 43a5b47505 Improving the instructions for updating example dag paths (#32771)
add 57f203251b FIX AWS deferrable operators by using AioCredentials when using `assume_role` (#32733)
add accdb0bf64 Update PIP to 23.2.1 - released today (#32774)
add 56c41d460c Introduce decorator to load providers configuration (#32765)
add 82e6226738 Fix Datafusion system tests (#32749)
add 6362ba5ab4 Move writing configuration for webserver to main (webserver limited) (#32766)
add 42b4b43c4c Add Redis task handler (#31855)
add 01a61b0e7c Documentation Update to enhance readability (#32730)
add 505fe854e2 Remove unnecessary cast (#32782)
add 685328e357 Quarantine test_backfill_integration in dask executor (#32780)
add 73bc49adb1 Fix depends_on_past work for dynamic tasks (#32397)
new e6bd5213ec plugins: load plugins from providers
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b78c4ceca8)
\
N -- N -- N refs/heads/load-plugins-from-providers (e6bd5213ec)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/ci.yml | 47 ++-
Dockerfile | 59 ++-
Dockerfile.ci | 50 ++-
IMAGES.rst | 2 +-
INTHEWILD.md | 1 +
TESTING.rst | 3 +-
airflow/__main__.py | 13 +
airflow/api_connexion/endpoints/dag_endpoint.py | 7 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 12 +-
.../endpoints/dag_warning_endpoint.py | 5 +-
.../api_connexion/endpoints/dataset_endpoint.py | 3 +-
.../endpoints/task_instance_endpoint.py | 12 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 5 +-
airflow/cli/cli_config.py | 40 +-
airflow/cli/commands/celery_command.py | 5 +
airflow/cli/commands/config_command.py | 9 +
airflow/cli/commands/connection_command.py | 8 +
airflow/cli/commands/dag_command.py | 23 +-
airflow/cli/commands/dag_processor_command.py | 2 +
airflow/cli/commands/db_command.py | 11 +
airflow/cli/commands/info_command.py | 2 +
airflow/cli/commands/internal_api_command.py | 2 +
airflow/cli/commands/jobs_command.py | 2 +
airflow/cli/commands/kerberos_command.py | 2 +
airflow/cli/commands/kubernetes_command.py | 3 +
airflow/cli/commands/plugins_command.py | 2 +
airflow/cli/commands/pool_command.py | 7 +
airflow/cli/commands/provider_command.py | 29 +-
airflow/cli/commands/role_command.py | 7 +
airflow/cli/commands/rotate_fernet_key_command.py | 2 +
airflow/cli/commands/scheduler_command.py | 2 +
airflow/cli/commands/standalone_command.py | 2 +
airflow/cli/commands/sync_perm_command.py | 2 +
airflow/cli/commands/task_command.py | 7 +
airflow/cli/commands/triggerer_command.py | 2 +
airflow/cli/commands/user_command.py | 7 +
airflow/cli/commands/variable_command.py | 7 +
airflow/cli/commands/webserver_command.py | 2 +
airflow/config_templates/config.yml | 266 +-------------
airflow/configuration.py | 402 ++++++++++++++++-----
.../example_setup_teardown_taskflow.py | 85 +++--
airflow/executors/executor_loader.py | 3 +
airflow/metrics/otel_logger.py | 6 +-
airflow/models/abstractoperator.py | 2 +-
airflow/models/serialized_dag.py | 50 +--
airflow/models/trigger.py | 75 ++--
airflow/provider_info.schema.json | 12 +-
.../providers/CREATING_COMMUNITY_PROVIDERS.rst | 99 -----
airflow/providers/amazon/aws/hooks/base_aws.py | 35 +-
.../amazon/aws/operators/step_function.py | 36 +-
airflow/providers/amazon/aws/triggers/batch.py | 2 +-
airflow/providers/amazon/aws/triggers/ecs.py | 4 +-
airflow/providers/amazon/aws/triggers/eks.py | 6 +-
airflow/providers/amazon/aws/triggers/rds.py | 6 +-
.../aws/triggers/{athena.py => step_function.py} | 41 ++-
.../{emr-containers.json => stepfunctions.json} | 20 +-
airflow/providers/amazon/provider.yaml | 3 +
airflow/providers/apache/hive/hooks/hive.py | 2 +-
.../celery/executors/celery_executor_utils.py | 22 +-
.../celery/executors/celery_kubernetes_executor.py | 19 +-
.../providers/celery/executors/default_celery.py | 22 +-
airflow/providers/celery/provider.yaml | 253 +++++++++++++
.../cncf/kubernetes/operators/spark_kubernetes.py | 30 +-
.../cloud/example_dags/example_postgres_to_gcs.py | 51 ---
airflow/providers/google/cloud/hooks/datafusion.py | 23 +-
airflow/providers/google/cloud/hooks/dataproc.py | 12 +
airflow/providers/google/cloud/hooks/spanner.py | 49 ++-
.../providers/google/cloud/operators/bigquery.py | 34 +-
.../providers/google/cloud/operators/dataproc.py | 32 +-
.../google/cloud/transfers/postgres_to_gcs.py | 4 +
airflow/providers/google/provider.yaml | 2 +
.../microsoft/azure/log/wasb_task_handler.py | 2 +
airflow/providers/mysql/hooks/mysql.py | 25 ++
.../{api/auth => providers/redis/log}/__init__.py | 0
airflow/providers/redis/log/redis_task_handler.py | 105 ++++++
airflow/providers/redis/provider.yaml | 3 +
airflow/providers/snowflake/hooks/snowflake.py | 71 +++-
airflow/providers_manager.py | 39 +-
airflow/sensors/external_task.py | 43 +++
airflow/ti_deps/deps/prev_dagrun_dep.py | 129 +++++--
airflow/triggers/external_task.py | 80 +++-
airflow/utils/cli.py | 3 +-
airflow/utils/db.py | 15 +
airflow/utils/db_cleanup.py | 6 +-
airflow/utils/log/file_task_handler.py | 2 +-
airflow/utils/providers_configuration_loader.py | 58 +++
airflow/utils/setup_teardown.py | 246 ++++++++++---
airflow/www/static/js/dag/details/graph/Node.tsx | 1 +
airflow/www/static/js/dag/details/graph/utils.ts | 30 +-
airflow/www/static/js/utils/graph.ts | 72 ++--
airflow/www/views.py | 13 +-
.../dag-processor/dag-processor-deployment.yaml | 5 +
dev/README_RELEASE_AIRFLOW.md | 2 +-
.../airflow_breeze/commands/testing_commands.py | 2 +
dev/breeze/src/airflow_breeze/global_constants.py | 2 +-
dev/breeze/src/airflow_breeze/utils/docs_errors.py | 4 +-
.../airflow_breeze/utils/publish_docs_builder.py | 3 +-
.../airflow_breeze/utils/publish_docs_helpers.py | 2 +
.../src/airflow_breeze/utils/spelling_checks.py | 4 +-
dev/example_dags/README.md | 15 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../operators/step_functions.rst | 1 +
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../configurations-ref.rst} | 5 +-
docs/apache-airflow-providers-celery/index.rst | 1 +
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../configurations-ref.rst | 71 ----
docs/apache-airflow-providers-google/index.rst | 1 -
.../installing-providers-from-sources.rst | 2 +-
.../{mysql_to_gcs.rst => postgres_to_gcs.rst} | 23 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-redis/index.rst | 7 +
.../installing-providers-from-sources.rst | 2 +-
.../logging/index.rst} | 16 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
...errable-operator-ref.rst => configurations.rst} | 13 +-
.../howto/create-custom-providers.rst | 268 ++++++++++++++
docs/apache-airflow-providers/index.rst | 241 ++----------
.../redirects.txt | 5 +-
.../logging-monitoring/metrics.rst | 17 +
.../administration-and-deployment/scheduler.rst | 7 +-
.../authoring-and-scheduling/plugins.rst | 1 -
docs/apache-airflow/configurations-ref.rst | 81 +----
docs/apache-airflow/core-concepts/dags.rst | 18 +-
.../core-concepts/executor/celery.rst | 2 +
.../core-concepts/executor/celery_kubernetes.rst | 2 +
.../howto/create-custom-decorator.rst | 3 +-
docs/apache-airflow/howto/index.rst | 1 +
.../howto/operator/external_task_sensor.rst | 9 +
docs/apache-airflow/howto/setup-and-teardown.rst | 184 ++++++++++
docs/apache-airflow/tutorial/taskflow.rst | 46 ++-
docs/conf.py | 35 +-
docs/docker-stack/build-arg-ref.rst | 2 +-
.../installing-providers-from-sources.rst | 0
.../includes/providers-configurations-ref.rst} | 23 +-
.../includes/sections-and-options.rst} | 30 +-
docs/exts/operators_and_hooks_ref.py | 22 +-
docs/exts/{ => templates}/auth_backend.rst.jinja2 | 0
.../exts/templates/configuration.rst.jinja2 | 5 +-
docs/exts/{ => templates}/connections.rst.jinja2 | 0
.../deferrable_operators_list.rst.jinja2} | 2 +-
docs/exts/{ => templates}/executors.rst.jinja2 | 0
docs/exts/{ => templates}/extra_links.rst.jinja2 | 0
docs/exts/{ => templates}/logging.rst.jinja2 | 0
docs/exts/{ => templates}/notifications.rst.jinja2 | 0
.../operators_and_hooks_ref-transfers.rst.jinja2 | 0
.../operators_and_hooks_ref.rst.jinja2 | 0
.../exts/{ => templates}/secret_backend.rst.jinja2 | 0
docs/spelling_wordlist.txt | 3 +
generated/provider_dependencies.json | 5 +-
helm_tests/airflow_core/test_dag_processor.py | 39 ++
images/breeze/output-commands-hash.txt | 4 +-
images/breeze/output_testing.svg | 24 +-
images/breeze/output_testing_tests.svg | 114 +++---
newsfragments/32572.significant.rst | 10 +
scripts/docker/common.sh | 2 +-
scripts/docker/install_mssql.sh | 41 +++
tests/cli/commands/test_celery_command.py | 2 +-
tests/cli/commands/test_config_command.py | 9 +
tests/cli/test_cli_parser.py | 81 ++++-
tests/conftest.py | 7 +
tests/core/test_configuration.py | 85 ++++-
tests/decorators/test_setup_teardown.py | 44 +++
tests/executors/test_dask_executor.py | 4 +
.../integration/executors/test_celery_executor.py | 16 +-
tests/providers/amazon/aws/hooks/test_base_aws.py | 4 +
.../amazon/aws/operators/test_step_function.py | 17 +
...est_emr_serverless.py => test_step_function.py} | 28 +-
.../celery/executors/test_celery_executor.py | 8 +-
.../executors/test_celery_kubernetes_executor.py | 2 +-
.../providers/google/cloud/hooks/test_dataproc.py | 4 +
tests/providers/google/cloud/hooks/test_spanner.py | 35 +-
.../google/cloud/operators/test_bigquery.py | 13 +-
.../google/cloud/operators/test_dataproc.py | 6 +
.../cloud/transfers/test_postgres_to_gcs_system.py | 2 +-
tests/providers/hashicorp/hooks/test_vault.py | 89 ++++-
tests/providers/mysql/operators/test_mysql.py | 77 ++++
.../auth => tests/providers/redis/log}/__init__.py | 0
.../providers/redis/log/test_redis_task_handler.py | 94 +++++
tests/providers/snowflake/hooks/test_snowflake.py | 30 ++
.../snowflake/operators/test_snowflake_sql.py | 68 ++++
tests/sensors/test_external_task_sensor.py | 82 ++++-
tests/serialization/test_dag_serialization.py | 2 +-
.../system/providers/core}/__init__.py | 0
.../example_external_task_child_deferrable.py} | 25 +-
.../example_external_task_parent_deferrable.py} | 71 ++--
.../google/cloud/dataform/example_dataform.py | 19 +-
.../google/cloud/datafusion/example_datafusion.py | 60 +--
.../cloud/datafusion/example_datafusion_async.py | 68 ++--
.../cloud/dataproc/example_dataproc_hadoop.py | 32 +-
.../google/cloud/dataproc/example_dataproc_hive.py | 11 +-
.../google/cloud/dataproc/example_dataproc_pig.py | 14 +-
.../cloud/dataproc/example_dataproc_pyspark.py | 65 +++-
.../cloud/dataproc/example_dataproc_spark_sql.py | 12 +-
.../cloud/transfers/example_postgres_to_gcs.py | 227 ++++++++++++
tests/ti_deps/deps/test_prev_dagrun_dep.py | 123 ++++---
tests/triggers/test_external_task.py | 6 +
tests/www/views/test_views.py | 5 +-
273 files changed, 4410 insertions(+), 1781 deletions(-)
rename docs/apache-airflow-providers/howto/create-update-providers.rst => airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst (71%)
copy airflow/providers/amazon/aws/triggers/{athena.py => step_function.py} (58%)
copy airflow/providers/amazon/aws/waiters/{emr-containers.json => stepfunctions.json} (52%)
delete mode 100644 airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py
copy airflow/{api/auth => providers/redis/log}/__init__.py (100%)
create mode 100644 airflow/providers/redis/log/redis_task_handler.py
create mode 100644 airflow/utils/providers_configuration_loader.py
copy docs/{apache-airflow-providers-airbyte/changelog.rst => apache-airflow-providers-celery/configurations-ref.rst} (92%)
delete mode 100644 docs/apache-airflow-providers-google/configurations-ref.rst
copy docs/apache-airflow-providers-google/operators/transfer/{mysql_to_gcs.rst => postgres_to_gcs.rst} (71%)
copy docs/{apache-airflow-providers-elasticsearch/hooks/elasticsearch_sql_hook.rst => apache-airflow-providers-redis/logging/index.rst} (60%)
copy docs/apache-airflow-providers/core-extensions/{deferrable-operator-ref.rst => configurations.rst} (71%)
create mode 100644 docs/apache-airflow-providers/howto/create-custom-providers.rst
copy docs/{helm-chart => apache-airflow-providers}/redirects.txt (89%)
create mode 100644 docs/apache-airflow/howto/setup-and-teardown.rst
rename docs/{ => exts/includes}/installing-providers-from-sources.rst (100%)
copy docs/{apache-airflow-providers-airbyte/connections.rst => exts/includes/providers-configurations-ref.rst} (67%)
copy docs/{apache-airflow/configurations-ref.rst => exts/includes/sections-and-options.rst} (64%)
rename docs/exts/{ => templates}/auth_backend.rst.jinja2 (100%)
copy airflow/www/templates/appbuilder/index.html => docs/exts/templates/configuration.rst.jinja2 (83%)
rename docs/exts/{ => templates}/connections.rst.jinja2 (100%)
rename docs/exts/{deferrable_operatos_list.rst.jinja2 => templates/deferrable_operators_list.rst.jinja2} (98%)
rename docs/exts/{ => templates}/executors.rst.jinja2 (100%)
rename docs/exts/{ => templates}/extra_links.rst.jinja2 (100%)
rename docs/exts/{ => templates}/logging.rst.jinja2 (100%)
rename docs/exts/{ => templates}/notifications.rst.jinja2 (100%)
rename docs/exts/{ => templates}/operators_and_hooks_ref-transfers.rst.jinja2 (100%)
rename docs/exts/{ => templates}/operators_and_hooks_ref.rst.jinja2 (100%)
rename docs/exts/{ => templates}/secret_backend.rst.jinja2 (100%)
create mode 100644 newsfragments/32572.significant.rst
copy tests/providers/amazon/aws/triggers/{test_emr_serverless.py => test_step_function.py} (67%)
copy {airflow/api/auth => tests/providers/redis/log}/__init__.py (100%)
create mode 100644 tests/providers/redis/log/test_redis_task_handler.py
copy {airflow/api_connexion => tests/system/providers/core}/__init__.py (100%)
copy tests/{dags_corrupted/test_nonstring_owner.py => system/providers/core/example_external_task_child_deferrable.py} (65%)
copy tests/system/providers/{google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py => core/example_external_task_parent_deferrable.py} (50%)
create mode 100644 tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py
[airflow] 01/01: plugins: load plugins from providers
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch load-plugins-from-providers
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit e6bd5213ec80bb5fc39b21986a6a67a9c233ade1
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Tue Jun 27 14:24:24 2023 +0200
plugins: load plugins from providers
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/__init__.py | 9 ++++-----
airflow/plugins_manager.py | 33 +++++++++++++++++++++++++++++++-
airflow/providers_manager.py | 35 ++++++++++++++++++++++++++++++++++
tests/always/test_providers_manager.py | 30 ++++++++++++++++++++++++++++-
tests/plugins/test_plugins_manager.py | 33 ++++++++++++++++++++++++++++++++
5 files changed, 133 insertions(+), 7 deletions(-)
diff --git a/airflow/__init__.py b/airflow/__init__.py
index d330b9332a..2e8e088ebc 100644
--- a/airflow/__init__.py
+++ b/airflow/__init__.py
@@ -106,11 +106,6 @@ def __getattr__(name: str):
return val
-if not settings.LAZY_LOAD_PLUGINS:
- from airflow import plugins_manager
-
- plugins_manager.ensure_plugins_loaded()
-
if not settings.LAZY_LOAD_PROVIDERS:
from airflow import providers_manager
@@ -118,6 +113,10 @@ if not settings.LAZY_LOAD_PROVIDERS:
manager.initialize_providers_list()
manager.initialize_providers_hooks()
manager.initialize_providers_extra_links()
+if not settings.LAZY_LOAD_PLUGINS:
+ from airflow import plugins_manager
+
+ plugins_manager.ensure_plugins_loaded()
# This is never executed, but tricks static analyzers (PyDev, PyCharm,)
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 946f064269..1692f0cafd 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -38,7 +38,7 @@ from types import ModuleType
from airflow import settings
from airflow.utils.entry_points import entry_points_with_dist
from airflow.utils.file import find_path_from_directory
-from airflow.utils.module_loading import qualname
+from airflow.utils.module_loading import import_string, qualname
if TYPE_CHECKING:
from airflow.hooks.base import BaseHook
@@ -50,6 +50,7 @@ log = logging.getLogger(__name__)
import_errors: dict[str, str] = {}
plugins: list[AirflowPlugin] | None = None
+loaded_plugins: set[str] = set()
# Plugin components to integrate as modules
registered_hooks: list[BaseHook] | None = None
@@ -205,10 +206,16 @@ def is_valid_plugin(plugin_obj):
def register_plugin(plugin_instance):
"""
Start plugin load and register it after success initialization.
+ If plugin is already registered, do nothing.
:param plugin_instance: subclass of AirflowPlugin
"""
global plugins
+
+ if plugin_instance.__name__ in loaded_plugins:
+ return
+
+ loaded_plugins.add(plugin_instance.__name__)
plugin_instance.on_load()
plugins.append(plugin_instance)
@@ -267,6 +274,27 @@ def load_plugins_from_plugin_directory():
import_errors[file_path] = str(e)
+def load_providers_plugins():
+ from airflow.providers_manager import ProvidersManager
+
+ global import_errors
+ log.debug("Loading plugins from providers")
+ providers_manager = ProvidersManager()
+ providers_manager.initialize_providers_plugins()
+ for plugin in providers_manager.plugins:
+ log.debug("Importing plugin %s from class %s", plugin.name, plugin.plugin_class)
+
+ try:
+ plugin_instance = import_string(plugin.plugin_class)
+ if not is_valid_plugin(plugin_instance):
+ log.warning("Plugin %s is not a valid plugin", plugin.name)
+ continue
+ register_plugin(plugin_instance)
+ except ImportError:
+ log.exception("Failed to load plugin %s from class name %s", plugin.name, plugin.plugin_class)
+ continue
+
+
def make_module(name: str, objects: list[Any]):
"""Creates new module."""
if not objects:
@@ -306,6 +334,9 @@ def ensure_plugins_loaded():
load_plugins_from_plugin_directory()
load_entrypoint_plugins()
+ if not settings.LAZY_LOAD_PROVIDERS:
+ load_providers_plugins()
+
# We don't do anything with these for now, but we want to keep track of
# them so we can integrate them in to the UI's Connection screens
for plugin in plugins:
diff --git a/airflow/providers_manager.py b/airflow/providers_manager.py
index bda6df3441..6fa7f5f9f1 100644
--- a/airflow/providers_manager.py
+++ b/airflow/providers_manager.py
@@ -217,6 +217,14 @@ class TriggerInfo(NamedTuple):
integration_name: str
+class PluginInfo(NamedTuple):
+ """Plugin class, name and provider it comes from."""
+
+ name: str
+ plugin_class: str
+ provider_name: str
+
+
class HookInfo(NamedTuple):
"""Hook information."""
@@ -421,6 +429,8 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
self._customized_form_fields_schema_validator = (
_create_customized_form_field_behaviours_schema_validator()
)
+ # Set of plugins contained in providers
+ self._plugins_set: set[PluginInfo] = set()
@provider_info_cache("list")
def initialize_providers_list(self):
@@ -516,6 +526,11 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
self.initialize_providers_list()
self._discover_auth_backends()
+ @provider_info_cache("plugins")
+ def initialize_providers_plugins(self):
+ self.initialize_providers_list()
+ self._discover_plugins()
+
def _discover_all_providers_from_packages(self) -> None:
"""
Discover all providers by scanning packages installed.
@@ -1024,6 +1039,20 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
if provider.data.get("config"):
self._provider_configs[provider_package] = provider.data.get("config")
+ def _discover_plugins(self) -> None:
+ """Retrieve all plugins defined in the providers."""
+ for provider_package, provider in self._provider_dict.items():
+ if provider.data.get("plugins"):
+ for plugin_dict in provider.data["plugins"]:
+ if _correctness_check(provider_package, plugin_dict["plugin-class"], provider):
+ self._plugins_set.add(
+ PluginInfo(
+ name=plugin_dict["name"],
+ plugin_class=plugin_dict["plugin-class"],
+ provider_name=provider_package,
+ )
+ )
+
@provider_info_cache("triggers")
def initialize_providers_triggers(self):
"""Initialization of providers triggers."""
@@ -1062,6 +1091,12 @@ class ProvidersManager(LoggingMixin, metaclass=Singleton):
# When we return hooks here it will only be used to retrieve hook information
return self._hooks_lazy_dict
+ @property
+ def plugins(self) -> list[PluginInfo]:
+ """Returns information about plugins available in providers."""
+ self.initialize_providers_plugins()
+ return sorted(self._plugins_set, key=lambda x: x.plugin_class)
+
@property
def taskflow_decorators(self) -> dict[str, TaskDecorator]:
self.initialize_providers_taskflow_decorator()
diff --git a/tests/always/test_providers_manager.py b/tests/always/test_providers_manager.py
index b99dbcb84f..7e05d1cfb1 100644
--- a/tests/always/test_providers_manager.py
+++ b/tests/always/test_providers_manager.py
@@ -28,7 +28,13 @@ from flask_babel import lazy_gettext
from wtforms import BooleanField, Field, StringField
from airflow.exceptions import AirflowOptionalProviderFeatureException
-from airflow.providers_manager import HookClassProvider, LazyDictWithCache, ProviderInfo, ProvidersManager
+from airflow.providers_manager import (
+ HookClassProvider,
+ LazyDictWithCache,
+ PluginInfo,
+ ProviderInfo,
+ ProvidersManager,
+)
class TestProviderManager:
@@ -157,6 +163,28 @@ class TestProviderManager:
" and 'airflow.providers.dummy.hooks.dummy.DummyHook2'."
) in self._caplog.records[0].message
+ def test_providers_manager_register_plugins(self):
+ providers_manager = ProvidersManager()
+ providers_manager._provider_dict["apache-airflow-providers-apache-hive"] = ProviderInfo(
+ version="0.0.1",
+ data={
+ "plugins": [
+ {
+ "name": "plugin1",
+ "plugin-class": "airflow.providers.apache.hive.plugins.hive.HivePlugin",
+ }
+ ]
+ },
+ package_or_source="package",
+ )
+ providers_manager._discover_plugins()
+ assert len(providers_manager._plugins_set) == 1
+ assert providers_manager._plugins_set.pop() == PluginInfo(
+ name="plugin1",
+ plugin_class="airflow.providers.apache.hive.plugins.hive.HivePlugin",
+ provider_name="apache-airflow-providers-apache-hive",
+ )
+
def test_hooks(self):
with pytest.warns(expected_warning=None) as warning_records:
with self._caplog.at_level(logging.WARNING):
diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py
index 019e2a69a8..bf74863407 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -160,6 +160,13 @@ def test_flaskappbuilder_nomenu_views():
class TestPluginsManager:
+ @pytest.fixture(autouse=True, scope="function")
+ def clean_plugins(self):
+ from airflow import plugins_manager
+
+ plugins_manager.loaded_plugins = set()
+ plugins_manager.plugins = []
+
def test_no_log_when_no_plugins(self, caplog):
with mock_plugin_manager(plugins=[]):
@@ -378,6 +385,32 @@ class TestPluginsManager:
assert get_listener_manager().has_listeners
assert get_listener_manager().pm.get_plugins().pop().__name__ == "tests.listeners.empty_listener"
+ def test_should_import_plugin_from_providers(self):
+ from airflow import plugins_manager
+
+ with mock.patch("airflow.plugins_manager.plugins", []):
+ assert len(plugins_manager.plugins) == 0
+ plugins_manager.load_providers_plugins()
+ assert len(plugins_manager.plugins) >= 2
+
+ def test_does_not_double_import_entrypoint_provider_plugins(self):
+ from airflow import plugins_manager
+
+ mock_entrypoint = mock.Mock()
+ mock_entrypoint.name = "test-entrypoint-plugin"
+ mock_entrypoint.module = "module_name_plugin"
+
+ mock_dist = mock.Mock()
+ mock_dist.metadata = {"Name": "test-entrypoint-plugin"}
+ mock_dist.version = "1.0.0"
+ mock_dist.entry_points = [mock_entrypoint]
+
+ with mock.patch("airflow.plugins_manager.plugins", []):
+ assert len(plugins_manager.plugins) == 0
+ plugins_manager.load_entrypoint_plugins()
+ plugins_manager.load_providers_plugins()
+ assert len(plugins_manager.plugins) == 2
+
class TestPluginsDirectorySource:
def test_should_return_correct_path_name(self):