You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2024/08/22 09:14:29 UTC
(airflow) branch aip-62/openlineage-impl updated (8d434eccc7 -> 0d5427543f)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch aip-62/openlineage-impl
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 8d434eccc7 feat: openlineage listener captures hook-level lineage
add 5790cf7128 Fix: Pass hook parameters to SnowflakeSqlApiHook and prep them for API call (#41150)
add ccf1f7f03f Fix Kinesis Analytics test (#41489)
add 0e7c757a1b fix: get task dependencies without serializing task tree to string (#41494)
add 666183f8d9 Mark TestSparkKubernetes test as db test (#41500)
add d48982692c Fix Non-DB test calculation for main builds (#41499)
add c8fc0ade09 Remove deprecated code is AWS provider (#41407)
add a70ee7209c Add incremental export and cross account export functionality in `DynamoDBToS3Operator` (#41304)
add 325f545d08 Fix variable interpolation in Airflow core release guide (#41510)
add 393978d65c fix: docker decorator pyi signature (#41509)
add f2b7bb8803 Partial fix for example_dynamodb_to_s3.py (#41517)
add 4251a9119c Fixing user_confirmation for quit action for prepare-provider-documentation (#41507)
add 796699b0d8 Update version_added field for config that are released in 2.10 (#41530)
add 166de82b6b Removed deprecated param from local_filesystem (#41533)
add a033c5f15a Fix failing pydantic v1 tests (#41534)
add dd3c3a7a43 Incorrect try number subtraction producing invalid span id for OTEL airflow (issue #41501) (#41502)
add 6ba2658aef Util helper deprecated functions removal (#41520)
add e35ae2db7c Airflow 3: airflow date utils date_range and days_ago deprecations removal (#41496)
add 8807f73dfd Delete experimental API (#41434)
add 55d49cc12f Refactor DataprocCreateBatchOperator and Dataproc system tests (#41527)
add 4361596981 Airflow 3: store_serialized_dags deprecated parameter usage removal (#40931)
add 8c2f56375a Skip task in example_dynamodb_to_s3.py (#41546)
add 7622192441 Fix `ElasticsearchSQLHook` fails with `AttributeError: __enter__` (#41537)
add e92cf81805 Fix `AwsTaskLogFetcher` missing logs (#41515)
add 60fe8c9329 restore python 3.12 support for papermill (#41548)
add 24f4cc3278 Chart: Default airflow version to 2.10.0 (#41529)
add a083dcde40 Fix changelog template for new providers w/o relevant commits (#41566)
add c0c92d37f3 Support pre-versions in version handling (#41565)
add f2e92eb6a2 airflow deprecated settings session_lifetime_days, force_log_out_after and policy removal (#41550)
add 865220965f smtp email user and password deprecated config removal (#41539)
add 55e60992ad base excutor deprecated unused validate_command function removal (#41551)
add 5c323a9c56 Remove debian bullseye support (#41568)
add b0fac0e09d Cleanup installed packages when running provider compatibility tests (#41488)
add 8f8565fac0 Airflow 3: processor_poll_interval deprecated parameter usage removal (#41096)
add 75fb7acbac Prepare docs for Aug 2nd wave of providers (#41559)
add c88192c466 Upgrade build and chart dependencies (#41570)
add db5b3b7df3 Refactor dataplex system tests (#41573)
add 7153965a6e decorators deprecated apply_defaults removal (#41579)
add 317a28ed43 Add changelog for airflow python client 2.10.0 (#41583)
add 9a84907462 Skip AWS auth test when python3-saml not installed (#41582)
add 8def71479a Kubernetes 1.31 support (#41581)
add a2b8696bdb Openlineage s3 to redshift operator integration (#41575)
add 9d5af80be8 kubernetes remove 1.27 support (#41590)
add 587015ddc8 Update the usage of credentials in the sagemaker test (#41595)
add 563bc494ab Fix UI rendering when XCom is INT, FLOAT, BOOL or NULL (#41516)
add 5cb582b51f fix: Missing 'slots_occupied' in `CeleryKubernetesExecutor` and `LocalKubernetesExecutor` (#41602)
add 716c4300a6 Remove deprecated log handler argument filename_template (#41552)
add 2545918b22 Fix InletEventsAccessors type stub (#41572)
add 2d3722de9b Update start-airflow executor defaulting for Celery integ (#41603)
add 6889b6b49a Fix running on_success_callback if `AirflowSkipException` raised (#40936)
add 1b602d5026 Limit watchtower as depenendcy as 3.3.0 breaks moin. (#41612)
add c87950173a custom dependency detector removal (#41609)
add d6df0786cf Make kerberos an optional and devel dependency for impala and fab (#41616)
add c78a004210 Add fixes by breeze/precommit-lint static checks (#41604) (#41618)
add d7d944e381 Fix: Keep compatibility with old FAB versions (#41549)
add 50e2ecb2b8 Wait sensor async (#41557)
add ef5da26166 Add missing flag (#41464)
add e306e7f7bc Enable running Pull Requests against v2-10-stable branch (#41624)
add 73cc7538ad remove the removed --use-migration-files argument of "airflow db reset" command in run_generate_migration.sh (#41621)
add 83a6cb4ff3 Airflow 2.10.0 has been released (#41528)
add 86e12a9a30 fix: replace dagTree with downstream_task_ids (#41587)
add 23e971600a Remove RAW HTML support from Trigger Form UI (#40029)
add a13ccbbdec Make latest botocore tests green (#41626)
add 79f6383c38 filename template arg in providers file task handlers backward compitability support (#41633)
add d404a1438f SSHHook: check if existing connection is still alive (#41061)
add 2727d5db92 Update the version of google-ads (#41638)
add 9af26368df fix: remove part of openlineage extraction from S3ToRedshiftOperator (#41631)
add 1b6699ed54 task command deprecated option ignore-depends-on-past removal (#41635)
add b3d73af67c fix(rest-api): Add order_by query param to TI listing APIs (#41283) (#41307)
add 410b57795b fix: select_query should have precedence over default query in RedshiftToS3Operator (#41634)
add 4eb71d60ca add kerberos env to base container env, add webserver-config volume) (#41645)
add 7e2c48cae5 Update `example_emr_eks` system test to run `eksctl` in one task (#41654)
add 9608ebff5e Fix wrong logs depupe in UI (#41346)
add 16f0073c19 Add retrieve output docker swarm operator (#41531)
new 0d5427543f feat: openlineage listener captures hook-level lineage
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (8d434eccc7)
\
N -- N -- N refs/heads/aip-62/openlineage-impl (0d5427543f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ISSUE_TEMPLATE/airflow_bug_report.yml | 2 +-
.github/boring-cyborg.yml | 3 -
.github/workflows/basic-tests.yml | 4 +-
.github/workflows/check-providers.yml | 5 +
.github/workflows/ci-image-build.yml | 2 +-
.github/workflows/ci.yml | 3 +-
.github/workflows/prod-image-build.yml | 2 +-
.github/workflows/prod-image-extra-checks.yml | 21 -
.github/workflows/push-image-cache.yml | 2 +-
Dockerfile | 23 +-
Dockerfile.ci | 55 +-
INSTALL | 6 +-
README.md | 28 +-
RELEASE_NOTES.rst | 293 +++++++-
airflow/api/client/__init__.py | 12 +-
airflow/api/client/api_client.py | 94 ---
airflow/api/client/json_client.py | 164 -----
airflow/api/client/local_client.py | 15 +-
airflow/api/common/experimental/__init__.py | 56 --
airflow/api/common/experimental/delete_dag.py | 30 -
airflow/api/common/experimental/get_code.py | 45 --
.../api/common/experimental/get_dag_run_state.py | 46 --
airflow/api/common/experimental/get_dag_runs.py | 57 --
airflow/api/common/experimental/get_lineage.py | 53 --
airflow/api/common/experimental/get_task.py | 39 -
.../api/common/experimental/get_task_instance.py | 48 --
airflow/api/common/experimental/mark_tasks.py | 36 -
airflow/api/common/experimental/pool.py | 104 ---
airflow/api/common/experimental/trigger_dag.py | 30 -
.../endpoints/task_instance_endpoint.py | 80 ++-
airflow/api_connexion/openapi/v1.yaml | 29 +-
.../api_connexion/schemas/task_instance_schema.py | 1 +
airflow/cli/cli_config.py | 7 -
airflow/cli/commands/task_command.py | 1 -
airflow/config_templates/airflow_local_settings.py | 11 -
airflow/config_templates/config.yml | 71 +-
airflow/config_templates/unit_tests.cfg | 2 -
airflow/configuration.py | 1 -
airflow/decorators/__init__.pyi | 3 +-
airflow/executors/base_executor.py | 20 +-
airflow/executors/local_executor.py | 2 +-
airflow/executors/sequential_executor.py | 2 +-
airflow/jobs/scheduler_job_runner.py | 14 +-
airflow/models/dag.py | 30 +-
airflow/models/dagbag.py | 23 -
airflow/models/taskinstance.py | 3 +-
airflow/providers/airbyte/CHANGELOG.rst | 16 +
airflow/providers/airbyte/__init__.py | 2 +-
airflow/providers/airbyte/provider.yaml | 3 +-
airflow/providers/alibaba/CHANGELOG.rst | 16 +
airflow/providers/alibaba/__init__.py | 2 +-
.../alibaba/cloud/log/oss_task_handler.py | 4 +-
airflow/providers/alibaba/provider.yaml | 3 +-
airflow/providers/amazon/CHANGELOG.rst | 29 +-
airflow/providers/amazon/__init__.py | 2 +-
.../amazon/aws/log/cloudwatch_task_handler.py | 5 +-
.../providers/amazon/aws/log/s3_task_handler.py | 6 +-
airflow/providers/amazon/aws/sensors/athena.py | 8 +-
airflow/providers/amazon/aws/sensors/batch.py | 40 +-
airflow/providers/amazon/aws/sensors/bedrock.py | 5 +-
.../amazon/aws/sensors/cloud_formation.py | 13 +-
airflow/providers/amazon/aws/sensors/comprehend.py | 8 +-
airflow/providers/amazon/aws/sensors/dms.py | 16 +-
airflow/providers/amazon/aws/sensors/ec2.py | 8 +-
airflow/providers/amazon/aws/sensors/ecs.py | 10 +-
airflow/providers/amazon/aws/sensors/eks.py | 12 +-
airflow/providers/amazon/aws/sensors/emr.py | 45 +-
airflow/providers/amazon/aws/sensors/glacier.py | 8 +-
airflow/providers/amazon/aws/sensors/glue.py | 9 -
.../amazon/aws/sensors/glue_catalog_partition.py | 8 +-
.../providers/amazon/aws/sensors/glue_crawler.py | 8 +-
.../amazon/aws/sensors/kinesis_analytics.py | 5 +-
.../amazon/aws/sensors/lambda_function.py | 10 +-
.../amazon/aws/sensors/opensearch_serverless.py | 5 +-
airflow/providers/amazon/aws/sensors/quicksight.py | 7 +-
.../amazon/aws/sensors/redshift_cluster.py | 8 +-
airflow/providers/amazon/aws/sensors/s3.py | 17 +-
airflow/providers/amazon/aws/sensors/sagemaker.py | 10 +-
airflow/providers/amazon/aws/sensors/sqs.py | 14 +-
.../providers/amazon/aws/sensors/step_function.py | 8 +-
.../amazon/aws/transfers/dynamodb_to_s3.py | 51 +-
.../amazon/aws/transfers/redshift_to_s3.py | 20 +-
.../amazon/aws/transfers/s3_to_redshift.py | 76 +-
airflow/providers/amazon/aws/utils/openlineage.py | 136 ++++
.../providers/amazon/aws/utils/task_log_fetcher.py | 12 +
airflow/providers/amazon/provider.yaml | 5 +-
airflow/providers/apache/beam/CHANGELOG.rst | 16 +
airflow/providers/apache/beam/__init__.py | 2 +-
airflow/providers/apache/beam/provider.yaml | 3 +-
airflow/providers/apache/cassandra/CHANGELOG.rst | 16 +
airflow/providers/apache/cassandra/__init__.py | 2 +-
airflow/providers/apache/cassandra/provider.yaml | 3 +-
airflow/providers/apache/drill/CHANGELOG.rst | 16 +
airflow/providers/apache/drill/__init__.py | 2 +-
airflow/providers/apache/drill/provider.yaml | 3 +-
airflow/providers/apache/druid/CHANGELOG.rst | 16 +
airflow/providers/apache/druid/__init__.py | 2 +-
airflow/providers/apache/druid/provider.yaml | 3 +-
airflow/providers/apache/flink/CHANGELOG.rst | 16 +
airflow/providers/apache/flink/__init__.py | 2 +-
airflow/providers/apache/flink/provider.yaml | 3 +-
airflow/providers/apache/hdfs/CHANGELOG.rst | 19 +
airflow/providers/apache/hdfs/__init__.py | 2 +-
.../providers/apache/hdfs/log/hdfs_task_handler.py | 6 +-
airflow/providers/apache/hdfs/provider.yaml | 3 +-
airflow/providers/apache/hive/CHANGELOG.rst | 19 +
airflow/providers/apache/hive/__init__.py | 2 +-
airflow/providers/apache/hive/provider.yaml | 3 +-
airflow/providers/apache/iceberg/CHANGELOG.rst | 20 +
airflow/providers/apache/iceberg/__init__.py | 2 +-
airflow/providers/apache/iceberg/provider.yaml | 3 +-
airflow/providers/apache/impala/CHANGELOG.rst | 16 +
airflow/providers/apache/impala/__init__.py | 2 +-
airflow/providers/apache/impala/provider.yaml | 5 +-
airflow/providers/apache/kafka/CHANGELOG.rst | 19 +
airflow/providers/apache/kafka/__init__.py | 2 +-
airflow/providers/apache/kafka/provider.yaml | 3 +-
airflow/providers/apache/kylin/CHANGELOG.rst | 18 +
airflow/providers/apache/kylin/__init__.py | 2 +-
airflow/providers/apache/kylin/provider.yaml | 3 +-
airflow/providers/apache/livy/CHANGELOG.rst | 19 +
airflow/providers/apache/livy/__init__.py | 2 +-
airflow/providers/apache/livy/provider.yaml | 3 +-
airflow/providers/apache/pig/CHANGELOG.rst | 19 +
airflow/providers/apache/pig/__init__.py | 2 +-
airflow/providers/apache/pig/provider.yaml | 3 +-
airflow/providers/apache/pinot/CHANGELOG.rst | 16 +
airflow/providers/apache/pinot/__init__.py | 2 +-
airflow/providers/apache/pinot/provider.yaml | 3 +-
airflow/providers/apache/spark/CHANGELOG.rst | 13 +
airflow/providers/apache/spark/__init__.py | 2 +-
airflow/providers/apache/spark/provider.yaml | 3 +-
airflow/providers/apprise/CHANGELOG.rst | 16 +
airflow/providers/apprise/__init__.py | 2 +-
airflow/providers/apprise/provider.yaml | 3 +-
airflow/providers/arangodb/CHANGELOG.rst | 16 +
airflow/providers/arangodb/__init__.py | 2 +-
airflow/providers/arangodb/provider.yaml | 3 +-
airflow/providers/asana/CHANGELOG.rst | 16 +
airflow/providers/asana/__init__.py | 2 +-
airflow/providers/asana/provider.yaml | 3 +-
airflow/providers/atlassian/jira/CHANGELOG.rst | 19 +
airflow/providers/atlassian/jira/__init__.py | 2 +-
airflow/providers/atlassian/jira/provider.yaml | 3 +-
airflow/providers/celery/CHANGELOG.rst | 17 +
airflow/providers/celery/__init__.py | 2 +-
.../celery/executors/celery_kubernetes_executor.py | 5 +
airflow/providers/celery/provider.yaml | 3 +-
airflow/providers/cloudant/CHANGELOG.rst | 16 +
airflow/providers/cloudant/__init__.py | 2 +-
airflow/providers/cloudant/provider.yaml | 3 +-
airflow/providers/cncf/kubernetes/CHANGELOG.rst | 19 +
airflow/providers/cncf/kubernetes/__init__.py | 2 +-
.../executors/local_kubernetes_executor.py | 5 +
.../providers/cncf/kubernetes/hooks/kubernetes.py | 1 +
airflow/providers/cncf/kubernetes/provider.yaml | 3 +-
airflow/providers/cohere/CHANGELOG.rst | 19 +
airflow/providers/cohere/__init__.py | 2 +-
airflow/providers/cohere/provider.yaml | 3 +-
airflow/providers/common/compat/CHANGELOG.rst | 16 +
airflow/providers/common/compat/__init__.py | 2 +-
airflow/providers/common/compat/provider.yaml | 3 +-
airflow/providers/common/sql/CHANGELOG.rst | 17 +
airflow/providers/common/sql/__init__.py | 2 +-
airflow/providers/common/sql/provider.yaml | 3 +-
airflow/providers/databricks/CHANGELOG.rst | 16 +
airflow/providers/databricks/__init__.py | 2 +-
airflow/providers/databricks/provider.yaml | 3 +-
airflow/providers/datadog/CHANGELOG.rst | 16 +
airflow/providers/datadog/__init__.py | 2 +-
airflow/providers/datadog/provider.yaml | 3 +-
airflow/providers/dbt/cloud/CHANGELOG.rst | 19 +
airflow/providers/dbt/cloud/__init__.py | 2 +-
airflow/providers/dbt/cloud/provider.yaml | 3 +-
airflow/providers/dingding/CHANGELOG.rst | 19 +
airflow/providers/dingding/__init__.py | 2 +-
airflow/providers/dingding/provider.yaml | 3 +-
airflow/providers/discord/CHANGELOG.rst | 16 +
airflow/providers/discord/__init__.py | 2 +-
airflow/providers/discord/provider.yaml | 3 +-
airflow/providers/docker/CHANGELOG.rst | 17 +
airflow/providers/docker/__init__.py | 2 +-
airflow/providers/docker/operators/docker.py | 27 +-
airflow/providers/docker/operators/docker_swarm.py | 38 +
airflow/providers/docker/provider.yaml | 3 +-
airflow/providers/elasticsearch/CHANGELOG.rst | 20 +
airflow/providers/elasticsearch/__init__.py | 2 +-
.../providers/elasticsearch/hooks/elasticsearch.py | 8 +
.../providers/elasticsearch/log/es_task_handler.py | 4 +-
airflow/providers/elasticsearch/provider.yaml | 3 +-
airflow/providers/exasol/CHANGELOG.rst | 16 +
airflow/providers/exasol/__init__.py | 2 +-
airflow/providers/exasol/provider.yaml | 3 +-
airflow/providers/fab/CHANGELOG.rst | 17 +
airflow/providers/fab/__init__.py | 2 +-
.../fab/auth_manager/cli_commands/user_command.py | 11 +-
airflow/providers/fab/provider.yaml | 11 +-
airflow/providers/facebook/CHANGELOG.rst | 19 +
airflow/providers/facebook/__init__.py | 2 +-
airflow/providers/facebook/provider.yaml | 3 +-
airflow/providers/ftp/CHANGELOG.rst | 16 +
airflow/providers/ftp/__init__.py | 2 +-
airflow/providers/ftp/provider.yaml | 3 +-
airflow/providers/github/CHANGELOG.rst | 19 +
airflow/providers/github/__init__.py | 2 +-
airflow/providers/github/provider.yaml | 3 +-
airflow/providers/google/CHANGELOG.rst | 28 +
airflow/providers/google/__init__.py | 2 +-
.../providers/google/cloud/log/gcs_task_handler.py | 3 +-
.../providers/google/cloud/operators/dataproc.py | 160 ++---
.../providers/google/cloud/triggers/dataproc.py | 5 +-
airflow/providers/google/provider.yaml | 5 +-
airflow/providers/grpc/CHANGELOG.rst | 16 +
airflow/providers/grpc/__init__.py | 2 +-
airflow/providers/grpc/provider.yaml | 3 +-
airflow/providers/hashicorp/CHANGELOG.rst | 16 +
airflow/providers/hashicorp/__init__.py | 2 +-
airflow/providers/hashicorp/provider.yaml | 3 +-
airflow/providers/http/CHANGELOG.rst | 19 +
airflow/providers/http/__init__.py | 2 +-
airflow/providers/http/provider.yaml | 3 +-
airflow/providers/imap/CHANGELOG.rst | 16 +
airflow/providers/imap/__init__.py | 2 +-
airflow/providers/imap/provider.yaml | 3 +-
airflow/providers/influxdb/CHANGELOG.rst | 16 +
airflow/providers/influxdb/__init__.py | 2 +-
airflow/providers/influxdb/provider.yaml | 3 +-
airflow/providers/jdbc/CHANGELOG.rst | 16 +
airflow/providers/jdbc/__init__.py | 2 +-
airflow/providers/jdbc/provider.yaml | 3 +-
airflow/providers/jenkins/CHANGELOG.rst | 19 +
airflow/providers/jenkins/__init__.py | 2 +-
airflow/providers/jenkins/provider.yaml | 3 +-
airflow/providers/microsoft/azure/CHANGELOG.rst | 27 +
airflow/providers/microsoft/azure/__init__.py | 2 +-
.../microsoft/azure/log/wasb_task_handler.py | 4 +-
airflow/providers/microsoft/azure/provider.yaml | 3 +-
airflow/providers/microsoft/mssql/CHANGELOG.rst | 21 +
airflow/providers/microsoft/mssql/__init__.py | 2 +-
airflow/providers/microsoft/mssql/provider.yaml | 3 +-
airflow/providers/microsoft/psrp/CHANGELOG.rst | 19 +
airflow/providers/microsoft/psrp/__init__.py | 2 +-
airflow/providers/microsoft/psrp/provider.yaml | 3 +-
airflow/providers/microsoft/winrm/CHANGELOG.rst | 16 +
airflow/providers/microsoft/winrm/__init__.py | 2 +-
airflow/providers/microsoft/winrm/provider.yaml | 3 +-
airflow/providers/mongo/CHANGELOG.rst | 16 +
airflow/providers/mongo/__init__.py | 2 +-
airflow/providers/mongo/provider.yaml | 3 +-
airflow/providers/mysql/CHANGELOG.rst | 16 +
airflow/providers/mysql/__init__.py | 2 +-
airflow/providers/mysql/provider.yaml | 3 +-
airflow/providers/neo4j/CHANGELOG.rst | 16 +
airflow/providers/neo4j/__init__.py | 2 +-
airflow/providers/neo4j/provider.yaml | 3 +-
airflow/providers/odbc/CHANGELOG.rst | 16 +
airflow/providers/odbc/__init__.py | 2 +-
airflow/providers/odbc/provider.yaml | 3 +-
airflow/providers/openai/CHANGELOG.rst | 16 +
airflow/providers/openai/__init__.py | 2 +-
airflow/providers/openai/provider.yaml | 3 +-
airflow/providers/openfaas/CHANGELOG.rst | 16 +
airflow/providers/openfaas/__init__.py | 2 +-
airflow/providers/openfaas/provider.yaml | 3 +-
airflow/providers/openlineage/CHANGELOG.rst | 29 +
airflow/providers/openlineage/__init__.py | 2 +-
.../providers/openlineage/extractors/manager.py | 12 +-
airflow/providers/openlineage/provider.yaml | 3 +-
airflow/providers/openlineage/utils/utils.py | 78 +-
airflow/providers/opensearch/CHANGELOG.rst | 16 +
airflow/providers/opensearch/__init__.py | 2 +-
airflow/providers/opensearch/provider.yaml | 3 +-
airflow/providers/opsgenie/CHANGELOG.rst | 16 +
airflow/providers/opsgenie/__init__.py | 2 +-
airflow/providers/opsgenie/provider.yaml | 3 +-
airflow/providers/oracle/CHANGELOG.rst | 16 +
airflow/providers/oracle/__init__.py | 2 +-
airflow/providers/oracle/provider.yaml | 3 +-
airflow/providers/pagerduty/CHANGELOG.rst | 16 +
airflow/providers/pagerduty/__init__.py | 2 +-
airflow/providers/pagerduty/provider.yaml | 3 +-
airflow/providers/papermill/CHANGELOG.rst | 17 +
airflow/providers/papermill/__init__.py | 2 +-
airflow/providers/papermill/provider.yaml | 11 +-
airflow/providers/pgvector/CHANGELOG.rst | 16 +
airflow/providers/pgvector/__init__.py | 2 +-
airflow/providers/pgvector/provider.yaml | 3 +-
airflow/providers/pinecone/CHANGELOG.rst | 16 +
airflow/providers/pinecone/__init__.py | 2 +-
airflow/providers/pinecone/provider.yaml | 3 +-
airflow/providers/postgres/CHANGELOG.rst | 16 +
airflow/providers/postgres/__init__.py | 2 +-
airflow/providers/postgres/provider.yaml | 3 +-
airflow/providers/presto/CHANGELOG.rst | 16 +
airflow/providers/presto/__init__.py | 2 +-
airflow/providers/presto/provider.yaml | 3 +-
airflow/providers/qdrant/CHANGELOG.rst | 16 +
airflow/providers/qdrant/__init__.py | 2 +-
airflow/providers/qdrant/provider.yaml | 3 +-
airflow/providers/redis/CHANGELOG.rst | 16 +
airflow/providers/redis/__init__.py | 2 +-
airflow/providers/redis/provider.yaml | 3 +-
airflow/providers/salesforce/CHANGELOG.rst | 16 +
airflow/providers/salesforce/__init__.py | 2 +-
airflow/providers/salesforce/provider.yaml | 3 +-
airflow/providers/samba/CHANGELOG.rst | 19 +
airflow/providers/samba/__init__.py | 2 +-
airflow/providers/samba/provider.yaml | 3 +-
airflow/providers/segment/CHANGELOG.rst | 16 +
airflow/providers/segment/__init__.py | 2 +-
airflow/providers/segment/provider.yaml | 3 +-
airflow/providers/sendgrid/CHANGELOG.rst | 16 +
airflow/providers/sendgrid/__init__.py | 2 +-
airflow/providers/sendgrid/provider.yaml | 3 +-
airflow/providers/sftp/CHANGELOG.rst | 16 +
airflow/providers/sftp/__init__.py | 2 +-
airflow/providers/sftp/provider.yaml | 3 +-
airflow/providers/singularity/CHANGELOG.rst | 16 +
airflow/providers/singularity/__init__.py | 2 +-
airflow/providers/singularity/provider.yaml | 3 +-
airflow/providers/slack/CHANGELOG.rst | 16 +
airflow/providers/slack/__init__.py | 2 +-
airflow/providers/slack/provider.yaml | 3 +-
airflow/providers/smtp/CHANGELOG.rst | 19 +
airflow/providers/smtp/__init__.py | 2 +-
airflow/providers/smtp/provider.yaml | 3 +-
airflow/providers/snowflake/CHANGELOG.rst | 21 +
airflow/providers/snowflake/__init__.py | 2 +-
.../providers/snowflake/hooks/snowflake_sql_api.py | 11 +-
airflow/providers/snowflake/operators/snowflake.py | 1 +
airflow/providers/snowflake/provider.yaml | 3 +-
airflow/providers/sqlite/CHANGELOG.rst | 16 +
airflow/providers/sqlite/__init__.py | 2 +-
airflow/providers/sqlite/provider.yaml | 3 +-
airflow/providers/ssh/CHANGELOG.rst | 16 +
airflow/providers/ssh/__init__.py | 2 +-
airflow/providers/ssh/hooks/ssh.py | 159 +++--
airflow/providers/ssh/operators/ssh.py | 1 -
airflow/providers/ssh/provider.yaml | 3 +-
airflow/providers/tableau/CHANGELOG.rst | 19 +
airflow/providers/tableau/__init__.py | 2 +-
airflow/providers/tableau/provider.yaml | 3 +-
airflow/providers/tabular/CHANGELOG.rst | 12 +
airflow/providers/tabular/__init__.py | 2 +-
airflow/providers/tabular/provider.yaml | 3 +-
airflow/providers/telegram/CHANGELOG.rst | 12 +
airflow/providers/telegram/__init__.py | 2 +-
airflow/providers/telegram/provider.yaml | 3 +-
airflow/providers/teradata/CHANGELOG.rst | 16 +
airflow/providers/teradata/__init__.py | 2 +-
airflow/providers/teradata/provider.yaml | 3 +-
airflow/providers/trino/CHANGELOG.rst | 16 +
airflow/providers/trino/__init__.py | 2 +-
airflow/providers/trino/provider.yaml | 3 +-
airflow/providers/vertica/CHANGELOG.rst | 16 +
airflow/providers/vertica/__init__.py | 2 +-
airflow/providers/vertica/provider.yaml | 3 +-
airflow/providers/weaviate/CHANGELOG.rst | 16 +
airflow/providers/weaviate/__init__.py | 2 +-
airflow/providers/weaviate/provider.yaml | 3 +-
airflow/providers/yandex/CHANGELOG.rst | 23 +
airflow/providers/yandex/__init__.py | 2 +-
airflow/providers/yandex/provider.yaml | 3 +-
airflow/providers/ydb/CHANGELOG.rst | 17 +
airflow/providers/ydb/__init__.py | 2 +-
airflow/providers/ydb/provider.yaml | 3 +-
airflow/providers/zendesk/CHANGELOG.rst | 16 +
airflow/providers/zendesk/__init__.py | 2 +-
airflow/providers/zendesk/provider.yaml | 3 +-
airflow/reproducible_build.yaml | 4 +-
airflow/secrets/local_filesystem.py | 24 -
airflow/sensors/base.py | 5 -
airflow/sensors/time_delta.py | 39 +-
airflow/serialization/serialized_objects.py | 27 +-
airflow/settings.py | 30 +-
airflow/traces/utils.py | 7 +-
airflow/utils/cli.py | 10 +-
airflow/utils/context.pyi | 2 +-
airflow/utils/dates.py | 121 +---
airflow/utils/decorators.py | 34 +-
airflow/utils/email.py | 15 +-
airflow/utils/helpers.py | 24 +-
airflow/utils/log/file_task_handler.py | 21 +-
airflow/www/api/__init__.py | 17 -
airflow/www/api/experimental/__init__.py | 17 -
airflow/www/api/experimental/endpoints.py | 434 -----------
airflow/www/app.py | 6 +-
airflow/www/extensions/init_security.py | 2 +-
airflow/www/extensions/init_views.py | 20 -
.../www/static/js/components/RenderedJsonField.tsx | 21 +-
airflow/www/static/js/components/utils.test.ts | 84 +++
.../utils.ts} | 29 +-
.../js/dag/details/taskInstance/Xcom/XcomEntry.tsx | 9 +-
airflow/www/static/js/types/api-generated.ts | 43 +-
airflow/www/templates/airflow/trigger.html | 4 +-
airflow/www/utils.py | 3 +-
airflow/www/views.py | 43 --
chart/Chart.yaml | 22 +-
chart/files/pod-template-file.kubernetes-helm-yaml | 11 +
chart/newsfragments/41529.significant.rst | 3 +
chart/values.schema.json | 6 +-
chart/values.yaml | 6 +-
clients/python/CHANGELOG.md | 20 +
clients/python/version.txt | 2 +-
.../12_airflow_dependencies_and_extras.rst | 6 +-
contributing-docs/testing/unit_tests.rst | 16 +-
dev/README_RELEASE_AIRFLOW.md | 2 +-
dev/breeze/doc/ci/05_workflows.md | 1 -
dev/breeze/doc/ci/06_diagrams.md | 1 -
dev/breeze/doc/images/output-commands.svg | 42 +-
dev/breeze/doc/images/output_ci-image_build.svg | 154 ++--
dev/breeze/doc/images/output_ci-image_build.txt | 2 +-
.../doc/images/output_k8s_configure-cluster.svg | 6 +-
.../doc/images/output_k8s_configure-cluster.txt | 2 +-
.../doc/images/output_k8s_create-cluster.svg | 6 +-
.../doc/images/output_k8s_create-cluster.txt | 2 +-
.../doc/images/output_k8s_delete-cluster.svg | 4 +-
.../doc/images/output_k8s_delete-cluster.txt | 2 +-
.../doc/images/output_k8s_deploy-airflow.svg | 6 +-
.../doc/images/output_k8s_deploy-airflow.txt | 2 +-
dev/breeze/doc/images/output_k8s_k9s.svg | 4 +-
dev/breeze/doc/images/output_k8s_k9s.txt | 2 +-
dev/breeze/doc/images/output_k8s_logs.svg | 4 +-
dev/breeze/doc/images/output_k8s_logs.txt | 2 +-
.../doc/images/output_k8s_run-complete-tests.svg | 6 +-
.../doc/images/output_k8s_run-complete-tests.txt | 2 +-
dev/breeze/doc/images/output_k8s_shell.svg | 4 +-
dev/breeze/doc/images/output_k8s_shell.txt | 2 +-
dev/breeze/doc/images/output_k8s_status.svg | 4 +-
dev/breeze/doc/images/output_k8s_status.txt | 2 +-
dev/breeze/doc/images/output_k8s_tests.svg | 6 +-
dev/breeze/doc/images/output_k8s_tests.txt | 2 +-
.../doc/images/output_k8s_upload-k8s-image.svg | 6 +-
.../doc/images/output_k8s_upload-k8s-image.txt | 2 +-
dev/breeze/doc/images/output_prod-image_build.svg | 180 +++--
dev/breeze/doc/images/output_prod-image_build.txt | 2 +-
...elease-management_install-provider-packages.svg | 104 +--
...elease-management_install-provider-packages.txt | 2 +-
...release-management_verify-provider-packages.svg | 102 +--
...release-management_verify-provider-packages.txt | 2 +-
dev/breeze/doc/images/output_shell.svg | 150 ++--
dev/breeze/doc/images/output_shell.txt | 2 +-
dev/breeze/doc/images/output_start-airflow.svg | 84 ++-
dev/breeze/doc/images/output_start-airflow.txt | 2 +-
dev/breeze/doc/images/output_testing_db-tests.svg | 104 +--
dev/breeze/doc/images/output_testing_db-tests.txt | 2 +-
.../doc/images/output_testing_non-db-tests.svg | 104 +--
.../doc/images/output_testing_non-db-tests.txt | 2 +-
dev/breeze/doc/images/output_testing_tests.svg | 104 +--
dev/breeze/doc/images/output_testing_tests.txt | 2 +-
.../src/airflow_breeze/commands/common_options.py | 6 +
.../airflow_breeze/commands/developer_commands.py | 30 +-
.../commands/developer_commands_config.py | 2 +
.../commands/release_management_commands.py | 7 +
.../commands/release_management_commands_config.py | 2 +
.../airflow_breeze/commands/testing_commands.py | 6 +
.../commands/testing_commands_config.py | 3 +
dev/breeze/src/airflow_breeze/global_constants.py | 10 +-
.../src/airflow_breeze/params/shell_params.py | 2 +
.../prepare_providers/provider_documentation.py | 4 +-
.../templates/CHANGELOG_TEMPLATE.rst.jinja2 | 6 +-
dev/breeze/src/airflow_breeze/utils/run_tests.py | 17 +-
dev/breeze/src/airflow_breeze/utils/versions.py | 2 +-
.../tests/test_pytest_args_for_test_types.py | 19 +-
dev/breeze/tests/test_selective_checks.py | 36 +-
docs/apache-airflow-providers-airbyte/commits.rst | 14 +-
docs/apache-airflow-providers-airbyte/index.rst | 10 +-
docs/apache-airflow-providers-alibaba/commits.rst | 14 +-
docs/apache-airflow-providers-alibaba/index.rst | 6 +-
docs/apache-airflow-providers-amazon/commits.rst | 23 +-
docs/apache-airflow-providers-amazon/index.rst | 10 +-
.../transfer/dynamodb_to_s3.rst | 16 +-
.../commits.rst | 14 +-
.../apache-airflow-providers-apache-beam/index.rst | 10 +-
.../commits.rst | 14 +-
.../index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 10 +-
.../commits.rst | 14 +-
.../index.rst | 10 +-
.../commits.rst | 14 +-
.../index.rst | 10 +-
.../commits.rst | 17 +-
.../apache-airflow-providers-apache-hdfs/index.rst | 6 +-
.../commits.rst | 17 +-
.../apache-airflow-providers-apache-hive/index.rst | 10 +-
.../commits.rst | 18 +-
.../index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 10 +-
.../commits.rst | 17 +-
.../index.rst | 6 +-
.../commits.rst | 17 +-
.../index.rst | 6 +-
.../commits.rst | 17 +-
.../apache-airflow-providers-apache-livy/index.rst | 10 +-
.../commits.rst | 17 +-
docs/apache-airflow-providers-apache-pig/index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 10 +-
.../commits.rst | 15 +-
.../index.rst | 10 +-
docs/apache-airflow-providers-apprise/commits.rst | 14 +-
docs/apache-airflow-providers-apprise/index.rst | 6 +-
docs/apache-airflow-providers-arangodb/commits.rst | 14 +-
docs/apache-airflow-providers-arangodb/index.rst | 6 +-
docs/apache-airflow-providers-asana/commits.rst | 14 +-
docs/apache-airflow-providers-asana/index.rst | 6 +-
.../commits.rst | 17 +-
.../index.rst | 6 +-
docs/apache-airflow-providers-celery/commits.rst | 15 +-
docs/apache-airflow-providers-celery/index.rst | 10 +-
docs/apache-airflow-providers-cloudant/commits.rst | 14 +-
docs/apache-airflow-providers-cloudant/index.rst | 6 +-
.../commits.rst | 17 +-
.../index.rst | 6 +-
docs/apache-airflow-providers-cohere/commits.rst | 17 +-
docs/apache-airflow-providers-cohere/index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 6 +-
.../commits.rst | 15 +-
docs/apache-airflow-providers-common-sql/index.rst | 10 +-
.../commits.rst | 14 +-
docs/apache-airflow-providers-databricks/index.rst | 10 +-
docs/apache-airflow-providers-datadog/commits.rst | 14 +-
docs/apache-airflow-providers-datadog/index.rst | 6 +-
.../apache-airflow-providers-dbt-cloud/commits.rst | 17 +-
docs/apache-airflow-providers-dbt-cloud/index.rst | 10 +-
docs/apache-airflow-providers-dingding/commits.rst | 17 +-
docs/apache-airflow-providers-dingding/index.rst | 10 +-
docs/apache-airflow-providers-discord/commits.rst | 14 +-
docs/apache-airflow-providers-discord/index.rst | 10 +-
docs/apache-airflow-providers-docker/commits.rst | 15 +-
docs/apache-airflow-providers-docker/index.rst | 6 +-
.../commits.rst | 15 +-
.../index.rst | 10 +-
docs/apache-airflow-providers-exasol/commits.rst | 14 +-
docs/apache-airflow-providers-exasol/index.rst | 10 +-
.../auth-manager/api-authentication.rst | 17 -
docs/apache-airflow-providers-fab/commits.rst | 15 +-
docs/apache-airflow-providers-fab/index.rst | 2 +-
docs/apache-airflow-providers-facebook/commits.rst | 17 +-
docs/apache-airflow-providers-facebook/index.rst | 6 +-
docs/apache-airflow-providers-ftp/commits.rst | 14 +-
docs/apache-airflow-providers-ftp/index.rst | 10 +-
docs/apache-airflow-providers-github/commits.rst | 17 +-
docs/apache-airflow-providers-github/index.rst | 6 +-
docs/apache-airflow-providers-google/commits.rst | 18 +-
docs/apache-airflow-providers-google/index.rst | 10 +-
docs/apache-airflow-providers-grpc/commits.rst | 14 +-
docs/apache-airflow-providers-grpc/index.rst | 6 +-
.../apache-airflow-providers-hashicorp/commits.rst | 14 +-
docs/apache-airflow-providers-hashicorp/index.rst | 10 +-
docs/apache-airflow-providers-http/commits.rst | 17 +-
docs/apache-airflow-providers-http/index.rst | 6 +-
docs/apache-airflow-providers-imap/commits.rst | 14 +-
docs/apache-airflow-providers-imap/index.rst | 6 +-
docs/apache-airflow-providers-influxdb/commits.rst | 14 +-
docs/apache-airflow-providers-influxdb/index.rst | 6 +-
docs/apache-airflow-providers-jdbc/commits.rst | 14 +-
docs/apache-airflow-providers-jdbc/index.rst | 10 +-
docs/apache-airflow-providers-jenkins/commits.rst | 17 +-
docs/apache-airflow-providers-jenkins/index.rst | 6 +-
.../commits.rst | 17 +-
.../index.rst | 10 +-
.../commits.rst | 16 +-
.../index.rst | 11 +-
.../commits.rst | 17 +-
.../index.rst | 6 +-
.../commits.rst | 14 +-
.../index.rst | 6 +-
docs/apache-airflow-providers-mongo/commits.rst | 14 +-
docs/apache-airflow-providers-mongo/index.rst | 6 +-
docs/apache-airflow-providers-mysql/commits.rst | 14 +-
docs/apache-airflow-providers-mysql/index.rst | 10 +-
docs/apache-airflow-providers-neo4j/commits.rst | 14 +-
docs/apache-airflow-providers-neo4j/index.rst | 6 +-
docs/apache-airflow-providers-odbc/commits.rst | 14 +-
docs/apache-airflow-providers-odbc/index.rst | 10 +-
docs/apache-airflow-providers-openai/commits.rst | 14 +-
docs/apache-airflow-providers-openai/index.rst | 6 +-
docs/apache-airflow-providers-openfaas/commits.rst | 14 +-
docs/apache-airflow-providers-openfaas/index.rst | 6 +-
.../commits.rst | 20 +-
.../apache-airflow-providers-openlineage/index.rst | 10 +-
.../commits.rst | 14 +-
docs/apache-airflow-providers-opensearch/index.rst | 6 +-
docs/apache-airflow-providers-opsgenie/commits.rst | 14 +-
docs/apache-airflow-providers-opsgenie/index.rst | 6 +-
docs/apache-airflow-providers-oracle/commits.rst | 14 +-
docs/apache-airflow-providers-oracle/index.rst | 10 +-
.../apache-airflow-providers-pagerduty/commits.rst | 14 +-
docs/apache-airflow-providers-pagerduty/index.rst | 6 +-
.../apache-airflow-providers-papermill/commits.rst | 15 +-
docs/apache-airflow-providers-papermill/index.rst | 8 +-
docs/apache-airflow-providers-pgvector/commits.rst | 14 +-
docs/apache-airflow-providers-pgvector/index.rst | 10 +-
docs/apache-airflow-providers-pinecone/commits.rst | 14 +-
docs/apache-airflow-providers-pinecone/index.rst | 6 +-
docs/apache-airflow-providers-postgres/commits.rst | 14 +-
docs/apache-airflow-providers-postgres/index.rst | 10 +-
docs/apache-airflow-providers-presto/commits.rst | 14 +-
docs/apache-airflow-providers-presto/index.rst | 10 +-
docs/apache-airflow-providers-qdrant/commits.rst | 20 +-
docs/apache-airflow-providers-qdrant/index.rst | 6 +-
docs/apache-airflow-providers-redis/commits.rst | 14 +-
docs/apache-airflow-providers-redis/index.rst | 6 +-
.../commits.rst | 14 +-
docs/apache-airflow-providers-salesforce/index.rst | 6 +-
docs/apache-airflow-providers-samba/commits.rst | 17 +-
docs/apache-airflow-providers-samba/index.rst | 10 +-
docs/apache-airflow-providers-segment/commits.rst | 14 +-
docs/apache-airflow-providers-segment/index.rst | 6 +-
docs/apache-airflow-providers-sendgrid/commits.rst | 14 +-
docs/apache-airflow-providers-sendgrid/index.rst | 6 +-
docs/apache-airflow-providers-sftp/commits.rst | 14 +-
docs/apache-airflow-providers-sftp/index.rst | 10 +-
.../commits.rst | 14 +-
.../apache-airflow-providers-singularity/index.rst | 6 +-
docs/apache-airflow-providers-slack/commits.rst | 14 +-
docs/apache-airflow-providers-slack/index.rst | 10 +-
docs/apache-airflow-providers-smtp/commits.rst | 17 +-
docs/apache-airflow-providers-smtp/index.rst | 6 +-
.../apache-airflow-providers-snowflake/commits.rst | 15 +-
docs/apache-airflow-providers-snowflake/index.rst | 10 +-
docs/apache-airflow-providers-sqlite/commits.rst | 14 +-
docs/apache-airflow-providers-sqlite/index.rst | 10 +-
docs/apache-airflow-providers-ssh/commits.rst | 14 +-
docs/apache-airflow-providers-ssh/index.rst | 6 +-
docs/apache-airflow-providers-tableau/commits.rst | 17 +-
docs/apache-airflow-providers-tableau/index.rst | 6 +-
docs/apache-airflow-providers-tabular/commits.rst | 14 +-
docs/apache-airflow-providers-tabular/index.rst | 10 +-
docs/apache-airflow-providers-telegram/commits.rst | 14 +-
docs/apache-airflow-providers-telegram/index.rst | 6 +-
docs/apache-airflow-providers-teradata/commits.rst | 14 +-
docs/apache-airflow-providers-teradata/index.rst | 10 +-
docs/apache-airflow-providers-trino/commits.rst | 14 +-
docs/apache-airflow-providers-trino/index.rst | 10 +-
docs/apache-airflow-providers-vertica/commits.rst | 14 +-
docs/apache-airflow-providers-vertica/index.rst | 10 +-
docs/apache-airflow-providers-weaviate/commits.rst | 14 +-
docs/apache-airflow-providers-weaviate/index.rst | 6 +-
docs/apache-airflow-providers-yandex/commits.rst | 18 +-
docs/apache-airflow-providers-yandex/index.rst | 14 +-
docs/apache-airflow-providers-ydb/commits.rst | 15 +-
docs/apache-airflow-providers-ydb/index.rst | 10 +-
docs/apache-airflow-providers-zendesk/commits.rst | 14 +-
docs/apache-airflow-providers-zendesk/index.rst | 6 +-
docs/apache-airflow/core-concepts/params.rst | 12 +-
docs/apache-airflow/deprecated-rest-api-ref.rst | 132 ----
docs/apache-airflow/extra-packages-ref.rst | 2 -
docs/apache-airflow/howto/docker-compose/index.rst | 2 +-
docs/apache-airflow/howto/email-config.rst | 4 -
.../howto/upgrading-from-1-10/index.rst | 50 --
docs/apache-airflow/index.rst | 1 -
docs/apache-airflow/installation/dependencies.rst | 20 -
docs/apache-airflow/installation/prerequisites.rst | 3 +-
.../installation/supported-versions.rst | 2 +-
docs/docker-stack/build.rst | 21 -
docs/docker-stack/changelog.rst | 4 +
.../docker-examples/customizing/debian-bullseye.sh | 37 -
generated/PYPI_README.md | 18 +-
generated/provider_dependencies.json | 18 +-
hatch_build.py | 4 +-
helm_tests/airflow_aux/test_pod_template_file.py | 39 +
newsfragments/37948.feature.rst | 1 -
newsfragments/38891.significant.rst | 10 -
newsfragments/39336.significant.rst | 7 -
newsfragments/39823.bugfix.rst | 1 -
newsfragments/40029.significant.rst | 1 +
newsfragments/40145.significant.rst | 5 -
newsfragments/40379.improvement.rst | 1 -
newsfragments/40701.feature.rst | 1 -
newsfragments/40703.feature.rst | 1 -
newsfragments/40874.significant.rst | 1 -
newsfragments/40931.significant.rst | 1 +
newsfragments/40936.bugfix.rst | 1 +
newsfragments/41039.feature.rst | 1 -
newsfragments/41096.significant.rst | 1 +
newsfragments/41116.feature.rst | 1 -
newsfragments/41434.significant.rst | 6 +
newsfragments/41496.significant.rst | 5 +
newsfragments/41520.significant.rst | 5 +
newsfragments/41533.significant.rst | 26 +
newsfragments/41539.significant.rst | 1 +
newsfragments/41550.significant.rst | 3 +
newsfragments/41552.significant.rst | 1 +
newsfragments/41579.significant.rst | 1 +
newsfragments/41609.significant.rst | 1 +
newsfragments/41635.significant.rst | 1 +
pyproject.toml | 10 +-
scripts/ci/pre_commit/check_system_tests.py | 3 +-
.../ci/pre_commit/check_tests_in_right_folders.py | 1 -
scripts/ci/pre_commit/supported_versions.py | 2 +-
scripts/cov/restapi_coverage.py | 4 +-
scripts/cov/www_coverage.py | 1 -
scripts/docker/entrypoint_ci.sh | 32 +-
scripts/docker/install_os_dependencies.sh | 19 +-
.../in_container/install_airflow_and_providers.py | 13 +-
scripts/in_container/install_devel_deps.py | 77 ++
scripts/in_container/run_generate_migration.sh | 4 +-
tests/always/test_example_dags.py | 20 +-
tests/api_connexion/conftest.py | 2 +-
.../endpoints/test_task_instance_endpoint.py | 113 +++
tests/api_connexion/test_auth.py | 18 +-
tests/api_connexion/test_cors.py | 26 +-
tests/api_experimental/__init__.py | 17 -
tests/api_experimental/auth/__init__.py | 16 -
tests/api_experimental/auth/backend/__init__.py | 16 -
.../auth/backend/test_basic_auth.py | 109 ---
tests/api_experimental/auth/test_client.py | 58 --
tests/api_experimental/client/__init__.py | 17 -
tests/api_experimental/client/test_json_client.py | 86 ---
tests/api_experimental/client/test_local_client.py | 242 -------
tests/api_experimental/common/__init__.py | 17 -
.../common/experimental/__init__.py | 17 -
.../common/experimental/test_pool.py | 137 ----
tests/api_experimental/common/test_delete_dag.py | 174 -----
tests/api_experimental/common/test_mark_tasks.py | 790 ---------------------
tests/api_experimental/common/test_trigger_dag.py | 107 ---
tests/api_experimental/conftest.py | 43 --
tests/cli/commands/test_task_command.py | 1 -
tests/core/test_configuration.py | 1 -
tests/core/test_settings.py | 18 +-
tests/executors/test_base_executor.py | 8 -
tests/integration/api_experimental/__init__.py | 16 -
.../integration/api_experimental/auth/__init__.py | 16 -
.../api_experimental/auth/backend/__init__.py | 16 -
.../auth/backend/test_kerberos_auth.py | 109 ---
tests/models/test_dag.py | 57 ++
tests/models/test_dagrun.py | 25 +
tests/models/test_taskinstance.py | 14 +-
.../alibaba/cloud/log/test_oss_task_handler.py | 4 +
.../test_aws_security_manager_override.py | 2 +
.../aws/auth_manager/test_aws_auth_manager.py | 1 +
.../amazon/aws/log/test_cloudwatch_task_handler.py | 8 +
.../amazon/aws/log/test_s3_task_handler.py | 4 +
tests/providers/amazon/aws/sensors/test_athena.py | 15 +-
tests/providers/amazon/aws/sensors/test_batch.py | 46 +-
tests/providers/amazon/aws/sensors/test_bedrock.py | 54 +-
.../amazon/aws/sensors/test_cloud_formation.py | 27 +-
.../amazon/aws/sensors/test_comprehend.py | 28 +-
tests/providers/amazon/aws/sensors/test_dms.py | 24 +-
tests/providers/amazon/aws/sensors/test_ecs.py | 20 +-
.../aws/sensors/test_emr_serverless_application.py | 15 +-
.../amazon/aws/sensors/test_emr_serverless_job.py | 13 +-
tests/providers/amazon/aws/sensors/test_glacier.py | 14 +-
tests/providers/amazon/aws/sensors/test_glue.py | 10 +-
.../aws/sensors/test_glue_catalog_partition.py | 10 +-
.../amazon/aws/sensors/test_glue_crawler.py | 10 +-
.../amazon/aws/sensors/test_glue_data_quality.py | 28 +-
.../amazon/aws/sensors/test_kinesis_analytics.py | 28 +-
.../amazon/aws/sensors/test_lambda_function.py | 10 +-
.../aws/sensors/test_opensearch_serverless.py | 17 +-
.../amazon/aws/sensors/test_quicksight.py | 17 +-
tests/providers/amazon/aws/sensors/test_s3.py | 26 +-
.../amazon/aws/sensors/test_sagemaker_base.py | 10 +-
tests/providers/amazon/aws/sensors/test_sqs.py | 28 +-
.../amazon/aws/sensors/test_step_function.py | 17 +-
.../amazon/aws/transfers/test_dynamodb_to_s3.py | 2 +
.../amazon/aws/transfers/test_redshift_to_s3.py | 117 +++
.../amazon/aws/transfers/test_s3_to_redshift.py | 298 ++++++++
.../providers/amazon/aws/utils/test_openlineage.py | 168 +++++
tests/providers/celery/cli/test_celery_command.py | 8 +
.../celery/executors/test_celery_executor.py | 5 +-
.../cncf/kubernetes/hooks/test_kubernetes.py | 8 +-
.../kubernetes/operators/test_spark_kubernetes.py | 1 +
.../databricks/plugins/test_databricks_workflow.py | 2 -
.../docker/operators/test_docker_swarm.py | 24 +-
.../elasticsearch/hooks/test_elasticsearch.py | 15 +
.../elasticsearch/log/test_es_task_handler.py | 11 +
tests/providers/fab/auth_manager/conftest.py | 2 +-
.../google/cloud/log/test_gcs_task_handler.py | 9 +
.../google/cloud/operators/test_dataproc.py | 8 +-
.../google/cloud/triggers/test_dataproc.py | 30 +-
.../common/auth_backend/test_google_openid.py | 67 +-
.../microsoft/azure/log/test_wasb_task_handler.py | 10 +
.../openlineage/extractors/test_manager.py | 1 +
tests/providers/openlineage/utils/test_utils.py | 280 ++++----
tests/providers/opensearch/conftest.py | 11 +-
.../providers/opensearch/hooks/test_opensearch.py | 3 +-
.../opensearch/operators/test_opensearch.py | 3 +
.../snowflake/hooks/test_snowflake_sql_api.py | 140 ++++
tests/providers/ssh/hooks/test_ssh.py | 23 +
tests/sensors/test_time_delta.py | 23 +-
tests/serialization/test_dag_serialization.py | 52 --
.../aws/example_bedrock_retrieve_and_generate.py | 6 +-
.../providers/amazon/aws/example_dynamodb_to_s3.py | 75 +-
.../system/providers/amazon/aws/example_emr_eks.py | 29 +-
.../amazon/aws/example_kinesis_analytics.py | 2 +
.../providers/amazon/aws/example_sagemaker.py | 6 +-
.../elasticsearch/example_elasticsearch_query.py | 9 +-
.../google/cloud/dataplex/example_dataplex.py | 2 +-
.../google/cloud/dataplex/example_dataplex_dp.py | 5 +-
.../google/cloud/dataplex/example_dataplex_dq.py | 7 +-
.../cloud/dataproc/example_dataproc_batch.py | 26 +-
.../dataproc/example_dataproc_batch_deferrable.py | 3 +
.../dataproc/example_dataproc_batch_persistent.py | 4 +
...proc_cluster_create_existing_stopped_cluster.py | 5 +
.../example_dataproc_cluster_deferrable.py | 4 +
.../dataproc/example_dataproc_cluster_diagnose.py | 3 +
.../dataproc/example_dataproc_cluster_generator.py | 3 +
.../example_dataproc_cluster_start_stop.py | 3 +
.../dataproc/example_dataproc_cluster_update.py | 4 +
.../google/cloud/dataproc/example_dataproc_gke.py | 3 +
.../cloud/dataproc/example_dataproc_hadoop.py | 3 +
.../google/cloud/dataproc/example_dataproc_hive.py | 3 +
.../google/cloud/dataproc/example_dataproc_pig.py | 3 +
.../cloud/dataproc/example_dataproc_presto.py | 3 +
.../cloud/dataproc/example_dataproc_pyspark.py | 3 +
.../cloud/dataproc/example_dataproc_spark.py | 3 +
.../cloud/dataproc/example_dataproc_spark_async.py | 3 +
.../dataproc/example_dataproc_spark_deferrable.py | 3 +
.../cloud/dataproc/example_dataproc_spark_sql.py | 3 +
.../cloud/dataproc/example_dataproc_sparkr.py | 3 +
.../cloud/dataproc/example_dataproc_trino.py | 3 +
tests/test_utils/decorators.py | 3 +-
tests/utils/test_dates.py | 63 +-
tests/utils/test_email.py | 11 +-
tests/utils/test_log_handlers.py | 23 +-
tests/www/api/__init__.py | 17 -
tests/www/api/experimental/__init__.py | 17 -
tests/www/api/experimental/conftest.py | 49 --
.../www/api/experimental/test_dag_runs_endpoint.py | 127 ----
tests/www/api/experimental/test_endpoints.py | 486 -------------
tests/www/test_init_views.py | 43 --
tests/www/test_utils.py | 59 +-
tests/www/views/test_views_trigger_dag.py | 51 +-
829 files changed, 7499 insertions(+), 7880 deletions(-)
delete mode 100644 airflow/api/client/api_client.py
delete mode 100644 airflow/api/client/json_client.py
delete mode 100644 airflow/api/common/experimental/__init__.py
delete mode 100644 airflow/api/common/experimental/delete_dag.py
delete mode 100644 airflow/api/common/experimental/get_code.py
delete mode 100644 airflow/api/common/experimental/get_dag_run_state.py
delete mode 100644 airflow/api/common/experimental/get_dag_runs.py
delete mode 100644 airflow/api/common/experimental/get_lineage.py
delete mode 100644 airflow/api/common/experimental/get_task.py
delete mode 100644 airflow/api/common/experimental/get_task_instance.py
delete mode 100644 airflow/api/common/experimental/mark_tasks.py
delete mode 100644 airflow/api/common/experimental/pool.py
delete mode 100644 airflow/api/common/experimental/trigger_dag.py
create mode 100644 airflow/providers/amazon/aws/utils/openlineage.py
delete mode 100644 airflow/www/api/__init__.py
delete mode 100644 airflow/www/api/experimental/__init__.py
delete mode 100644 airflow/www/api/experimental/endpoints.py
create mode 100644 airflow/www/static/js/components/utils.test.ts
copy airflow/www/static/js/{utils/URLSearchParamWrapper.ts => components/utils.ts} (60%)
create mode 100644 chart/newsfragments/41529.significant.rst
delete mode 100644 docs/apache-airflow/deprecated-rest-api-ref.rst
delete mode 100755 docs/docker-stack/docker-examples/customizing/debian-bullseye.sh
delete mode 100644 newsfragments/37948.feature.rst
delete mode 100644 newsfragments/38891.significant.rst
delete mode 100644 newsfragments/39336.significant.rst
delete mode 100644 newsfragments/39823.bugfix.rst
create mode 100644 newsfragments/40029.significant.rst
delete mode 100644 newsfragments/40145.significant.rst
delete mode 100644 newsfragments/40379.improvement.rst
delete mode 100644 newsfragments/40701.feature.rst
delete mode 100644 newsfragments/40703.feature.rst
delete mode 100644 newsfragments/40874.significant.rst
create mode 100644 newsfragments/40931.significant.rst
create mode 100644 newsfragments/40936.bugfix.rst
delete mode 100644 newsfragments/41039.feature.rst
create mode 100644 newsfragments/41096.significant.rst
delete mode 100644 newsfragments/41116.feature.rst
create mode 100644 newsfragments/41434.significant.rst
create mode 100644 newsfragments/41496.significant.rst
create mode 100644 newsfragments/41520.significant.rst
create mode 100644 newsfragments/41533.significant.rst
create mode 100644 newsfragments/41539.significant.rst
create mode 100644 newsfragments/41550.significant.rst
create mode 100644 newsfragments/41552.significant.rst
create mode 100644 newsfragments/41579.significant.rst
create mode 100644 newsfragments/41609.significant.rst
create mode 100644 newsfragments/41635.significant.rst
create mode 100755 scripts/in_container/install_devel_deps.py
delete mode 100644 tests/api_experimental/__init__.py
delete mode 100644 tests/api_experimental/auth/__init__.py
delete mode 100644 tests/api_experimental/auth/backend/__init__.py
delete mode 100644 tests/api_experimental/auth/backend/test_basic_auth.py
delete mode 100644 tests/api_experimental/auth/test_client.py
delete mode 100644 tests/api_experimental/client/__init__.py
delete mode 100644 tests/api_experimental/client/test_json_client.py
delete mode 100644 tests/api_experimental/client/test_local_client.py
delete mode 100644 tests/api_experimental/common/__init__.py
delete mode 100644 tests/api_experimental/common/experimental/__init__.py
delete mode 100644 tests/api_experimental/common/experimental/test_pool.py
delete mode 100644 tests/api_experimental/common/test_delete_dag.py
delete mode 100644 tests/api_experimental/common/test_mark_tasks.py
delete mode 100644 tests/api_experimental/common/test_trigger_dag.py
delete mode 100644 tests/api_experimental/conftest.py
delete mode 100644 tests/integration/api_experimental/__init__.py
delete mode 100644 tests/integration/api_experimental/auth/__init__.py
delete mode 100644 tests/integration/api_experimental/auth/backend/__init__.py
delete mode 100644 tests/integration/api_experimental/auth/backend/test_kerberos_auth.py
create mode 100644 tests/providers/amazon/aws/utils/test_openlineage.py
delete mode 100644 tests/www/api/__init__.py
delete mode 100644 tests/www/api/experimental/__init__.py
delete mode 100644 tests/www/api/experimental/conftest.py
delete mode 100644 tests/www/api/experimental/test_dag_runs_endpoint.py
delete mode 100644 tests/www/api/experimental/test_endpoints.py
delete mode 100644 tests/www/test_init_views.py
(airflow) 01/01: feat: openlineage listener captures hook-level lineage
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch aip-62/openlineage-impl
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 0d5427543f3363b09dfdce0c3a8d51c218d18089
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Wed Aug 14 17:19:36 2024 +0200
feat: openlineage listener captures hook-level lineage
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
airflow/io/path.py | 1 -
.../providers/openlineage/extractors/manager.py | 42 +++++-
.../providers/openlineage/plugins/openlineage.py | 5 +
airflow/providers/openlineage/provider.yaml | 1 +
generated/provider_dependencies.json | 1 +
.../openlineage/extractors/test_manager.py | 148 ++++++++++++++++++---
6 files changed, 173 insertions(+), 25 deletions(-)
diff --git a/airflow/io/path.py b/airflow/io/path.py
index 7c8d1f9f19..8afcab19ea 100644
--- a/airflow/io/path.py
+++ b/airflow/io/path.py
@@ -54,7 +54,6 @@ class TrackingFileWrapper(LoggingMixin):
if callable(attr):
# If the attribute is a method, wrap it in another method to intercept the call
def wrapper(*args, **kwargs):
- self.log.error("Calling method: %s", name)
if name == "read":
get_hook_lineage_collector().add_input_dataset(context=self._path, uri=str(self._path))
elif name == "write":
diff --git a/airflow/providers/openlineage/extractors/manager.py b/airflow/providers/openlineage/extractors/manager.py
index 5b9ad6ac1b..c38b2c30f1 100644
--- a/airflow/providers/openlineage/extractors/manager.py
+++ b/airflow/providers/openlineage/extractors/manager.py
@@ -24,7 +24,9 @@ from airflow.providers.openlineage.extractors.base import DefaultExtractor
from airflow.providers.openlineage.extractors.bash import BashExtractor
from airflow.providers.openlineage.extractors.python import PythonExtractor
from airflow.providers.openlineage.utils.utils import (
+ IS_AIRFLOW_2_10_OR_HIGHER,
get_unknown_source_attribute_run_facet,
+ translate_airflow_dataset,
try_import_from_string,
)
from airflow.utils.log.logging_mixin import LoggingMixin
@@ -90,7 +92,6 @@ class ExtractorManager(LoggingMixin):
f"task_id={task.task_id} "
f"airflow_run_id={dagrun.run_id} "
)
-
if extractor:
# Extracting advanced metadata is only possible when extractor for particular operator
# is defined. Without it, we can't extract any input or output data.
@@ -105,14 +106,25 @@ class ExtractorManager(LoggingMixin):
task_metadata = self.validate_task_metadata(task_metadata)
if task_metadata:
if (not task_metadata.inputs) and (not task_metadata.outputs):
- self.extract_inlets_and_outlets(task_metadata, task.inlets, task.outlets)
-
+ if (
+ IS_AIRFLOW_2_10_OR_HIGHER
+ and (hook_lineage := self.get_hook_lineage()) is not None
+ ):
+ inputs, outputs = hook_lineage
+ task_metadata.inputs = inputs
+ task_metadata.outputs = outputs
+ else:
+ self.extract_inlets_and_outlets(task_metadata, task.inlets, task.outlets)
return task_metadata
except Exception as e:
self.log.warning(
"Failed to extract metadata using found extractor %s - %s %s", extractor, e, task_info
)
+ elif IS_AIRFLOW_2_10_OR_HIGHER and (hook_lineage := self.get_hook_lineage()) is not None:
+ inputs, outputs = hook_lineage
+ task_metadata = OperatorLineage(inputs=inputs, outputs=outputs)
+ return task_metadata
else:
self.log.debug("Unable to find an extractor %s", task_info)
@@ -168,6 +180,30 @@ class ExtractorManager(LoggingMixin):
if d:
task_metadata.outputs.append(d)
+ def get_hook_lineage(self) -> tuple[list[Dataset], list[Dataset]] | None:
+ try:
+ from airflow.lineage.hook import get_hook_lineage_collector
+ except ImportError:
+ return None
+
+ if not get_hook_lineage_collector().has_collected:
+ return None
+
+ return (
+ [
+ dataset
+ for dataset_info in get_hook_lineage_collector().collected_datasets.inputs
+ if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
+ is not None
+ ],
+ [
+ dataset
+ for dataset_info in get_hook_lineage_collector().collected_datasets.outputs
+ if (dataset := translate_airflow_dataset(dataset_info.dataset, dataset_info.context))
+ is not None
+ ],
+ )
+
@staticmethod
def convert_to_ol_dataset_from_object_storage_uri(uri: str) -> Dataset | None:
from urllib.parse import urlparse
diff --git a/airflow/providers/openlineage/plugins/openlineage.py b/airflow/providers/openlineage/plugins/openlineage.py
index 5b4f2c3dbf..ce70f3a52a 100644
--- a/airflow/providers/openlineage/plugins/openlineage.py
+++ b/airflow/providers/openlineage/plugins/openlineage.py
@@ -25,6 +25,7 @@ from airflow.providers.openlineage.plugins.macros import (
lineage_parent_id,
lineage_run_id,
)
+from airflow.providers.openlineage.utils.utils import IS_AIRFLOW_2_10_OR_HIGHER
class OpenLineageProviderPlugin(AirflowPlugin):
@@ -39,6 +40,10 @@ class OpenLineageProviderPlugin(AirflowPlugin):
if not conf.is_disabled():
macros = [lineage_job_namespace, lineage_job_name, lineage_run_id, lineage_parent_id]
listeners = [get_openlineage_listener()]
+ if IS_AIRFLOW_2_10_OR_HIGHER:
+ from airflow.lineage.hook import HookLineageReader
+
+ hook_lineage_readers = [HookLineageReader]
else:
macros = []
listeners = []
diff --git a/airflow/providers/openlineage/provider.yaml b/airflow/providers/openlineage/provider.yaml
index 82fff7b883..45494ce5dd 100644
--- a/airflow/providers/openlineage/provider.yaml
+++ b/airflow/providers/openlineage/provider.yaml
@@ -48,6 +48,7 @@ versions:
dependencies:
- apache-airflow>=2.8.0
- apache-airflow-providers-common-sql>=1.6.0
+ - apache-airflow-providers-common-compat>=1.1.0
- attrs>=22.2
- openlineage-integration-common>=1.16.0
- openlineage-python>=1.16.0
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index ce59ba6127..9ed0045b0f 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -939,6 +939,7 @@
},
"openlineage": {
"deps": [
+ "apache-airflow-providers-common-compat>=1.1.0",
"apache-airflow-providers-common-sql>=1.6.0",
"apache-airflow>=2.8.0",
"attrs>=22.2",
diff --git a/tests/providers/openlineage/extractors/test_manager.py b/tests/providers/openlineage/extractors/test_manager.py
index ccfd04d5e2..479347179b 100644
--- a/tests/providers/openlineage/extractors/test_manager.py
+++ b/tests/providers/openlineage/extractors/test_manager.py
@@ -17,26 +17,47 @@
# under the License.
from __future__ import annotations
+import tempfile
+from typing import TYPE_CHECKING, Any
+from unittest.mock import MagicMock
+
import pytest
-from openlineage.client.event_v2 import Dataset
+from openlineage.client.event_v2 import Dataset as OpenLineageDataset
from openlineage.client.facet_v2 import documentation_dataset, ownership_dataset, schema_dataset
+from airflow.datasets import Dataset
+from airflow.io.path import ObjectStoragePath
from airflow.lineage.entities import Column, File, Table, User
+from airflow.models.baseoperator import BaseOperator
+from airflow.models.taskinstance import TaskInstance
+from airflow.operators.python import PythonOperator
+from airflow.providers.openlineage.extractors import OperatorLineage
from airflow.providers.openlineage.extractors.manager import ExtractorManager
+from airflow.utils.state import State
+from tests.test_utils.compat import AIRFLOW_V_2_10_PLUS
+
+if TYPE_CHECKING:
+ from airflow.utils.context import Context
@pytest.mark.parametrize(
("uri", "dataset"),
(
- ("s3://bucket1/dir1/file1", Dataset(namespace="s3://bucket1", name="dir1/file1")),
- ("gs://bucket2/dir2/file2", Dataset(namespace="gs://bucket2", name="dir2/file2")),
- ("gcs://bucket3/dir3/file3", Dataset(namespace="gs://bucket3", name="dir3/file3")),
- ("hdfs://namenodehost:8020/file1", Dataset(namespace="hdfs://namenodehost:8020", name="file1")),
- ("hdfs://namenodehost/file2", Dataset(namespace="hdfs://namenodehost", name="file2")),
- ("file://localhost/etc/fstab", Dataset(namespace="file://localhost", name="etc/fstab")),
- ("file:///etc/fstab", Dataset(namespace="file://", name="etc/fstab")),
- ("https://test.com", Dataset(namespace="https", name="test.com")),
- ("https://test.com?param1=test1¶m2=test2", Dataset(namespace="https", name="test.com")),
+ ("s3://bucket1/dir1/file1", OpenLineageDataset(namespace="s3://bucket1", name="dir1/file1")),
+ ("gs://bucket2/dir2/file2", OpenLineageDataset(namespace="gs://bucket2", name="dir2/file2")),
+ ("gcs://bucket3/dir3/file3", OpenLineageDataset(namespace="gs://bucket3", name="dir3/file3")),
+ (
+ "hdfs://namenodehost:8020/file1",
+ OpenLineageDataset(namespace="hdfs://namenodehost:8020", name="file1"),
+ ),
+ ("hdfs://namenodehost/file2", OpenLineageDataset(namespace="hdfs://namenodehost", name="file2")),
+ ("file://localhost/etc/fstab", OpenLineageDataset(namespace="file://localhost", name="etc/fstab")),
+ ("file:///etc/fstab", OpenLineageDataset(namespace="file://", name="etc/fstab")),
+ ("https://test.com", OpenLineageDataset(namespace="https", name="test.com")),
+ (
+ "https://test.com?param1=test1¶m2=test2",
+ OpenLineageDataset(namespace="https", name="test.com"),
+ ),
("file:test.csv", None),
("not_an_url", None),
),
@@ -50,21 +71,36 @@ def test_convert_to_ol_dataset_from_object_storage_uri(uri, dataset):
("obj", "dataset"),
(
(
- Dataset(namespace="n1", name="f1"),
- Dataset(namespace="n1", name="f1"),
+ OpenLineageDataset(namespace="n1", name="f1"),
+ OpenLineageDataset(namespace="n1", name="f1"),
+ ),
+ (
+ File(url="s3://bucket1/dir1/file1"),
+ OpenLineageDataset(namespace="s3://bucket1", name="dir1/file1"),
+ ),
+ (
+ File(url="gs://bucket2/dir2/file2"),
+ OpenLineageDataset(namespace="gs://bucket2", name="dir2/file2"),
+ ),
+ (
+ File(url="gcs://bucket3/dir3/file3"),
+ OpenLineageDataset(namespace="gs://bucket3", name="dir3/file3"),
),
- (File(url="s3://bucket1/dir1/file1"), Dataset(namespace="s3://bucket1", name="dir1/file1")),
- (File(url="gs://bucket2/dir2/file2"), Dataset(namespace="gs://bucket2", name="dir2/file2")),
- (File(url="gcs://bucket3/dir3/file3"), Dataset(namespace="gs://bucket3", name="dir3/file3")),
(
File(url="hdfs://namenodehost:8020/file1"),
- Dataset(namespace="hdfs://namenodehost:8020", name="file1"),
+ OpenLineageDataset(namespace="hdfs://namenodehost:8020", name="file1"),
+ ),
+ (
+ File(url="hdfs://namenodehost/file2"),
+ OpenLineageDataset(namespace="hdfs://namenodehost", name="file2"),
+ ),
+ (
+ File(url="file://localhost/etc/fstab"),
+ OpenLineageDataset(namespace="file://localhost", name="etc/fstab"),
),
- (File(url="hdfs://namenodehost/file2"), Dataset(namespace="hdfs://namenodehost", name="file2")),
- (File(url="file://localhost/etc/fstab"), Dataset(namespace="file://localhost", name="etc/fstab")),
- (File(url="file:///etc/fstab"), Dataset(namespace="file://", name="etc/fstab")),
- (File(url="https://test.com"), Dataset(namespace="https", name="test.com")),
- (Table(cluster="c1", database="d1", name="t1"), Dataset(namespace="c1", name="d1.t1")),
+ (File(url="file:///etc/fstab"), OpenLineageDataset(namespace="file://", name="etc/fstab")),
+ (File(url="https://test.com"), OpenLineageDataset(namespace="https", name="test.com")),
+ (Table(cluster="c1", database="d1", name="t1"), OpenLineageDataset(namespace="c1", name="d1.t1")),
("gs://bucket2/dir2/file2", None),
("not_an_url", None),
),
@@ -167,3 +203,73 @@ def test_convert_to_ol_dataset_table():
assert result.namespace == "c1"
assert result.name == "d1.t1"
assert result.facets == expected_facets
+
+
+@pytest.mark.skipif(not AIRFLOW_V_2_10_PLUS, reason="Hook lineage works in Airflow >= 2.10.0")
+def test_extractor_manager_uses_hook_level_lineage(hook_lineage_collector):
+ dagrun = MagicMock()
+ task = MagicMock()
+ del task.get_openlineage_facets_on_start
+ del task.get_openlineage_facets_on_complete
+ ti = MagicMock()
+
+ hook_lineage_collector.add_input_dataset(None, uri="s3://bucket/input_key")
+ hook_lineage_collector.add_output_dataset(None, uri="s3://bucket/output_key")
+ extractor_manager = ExtractorManager()
+ metadata = extractor_manager.extract_metadata(dagrun=dagrun, task=task, complete=True, task_instance=ti)
+
+ assert metadata.inputs == [OpenLineageDataset(namespace="s3://bucket", name="input_key")]
+ assert metadata.outputs == [OpenLineageDataset(namespace="s3://bucket", name="output_key")]
+
+
+@pytest.mark.skipif(not AIRFLOW_V_2_10_PLUS, reason="Hook lineage works in Airflow >= 2.10.0")
+def test_extractor_manager_does_not_use_hook_level_lineage_when_operator(hook_lineage_collector):
+ class FakeSupportedOperator(BaseOperator):
+ def execute(self, context: Context) -> Any:
+ pass
+
+ def get_openlineage_facets_on_start(self):
+ return OperatorLineage(
+ inputs=[OpenLineageDataset(namespace="s3://bucket", name="proper_input_key")]
+ )
+
+ dagrun = MagicMock()
+ task = FakeSupportedOperator(task_id="test_task_extractor")
+ ti = MagicMock()
+ hook_lineage_collector.add_input_dataset(None, uri="s3://bucket/input_key")
+
+ extractor_manager = ExtractorManager()
+ metadata = extractor_manager.extract_metadata(dagrun=dagrun, task=task, complete=True, task_instance=ti)
+
+ # s3://bucket/input_key not here - use data from operator
+ assert metadata.inputs == [OpenLineageDataset(namespace="s3://bucket", name="proper_input_key")]
+ assert metadata.outputs == []
+
+
+@pytest.mark.db_test
+@pytest.mark.skipif(not AIRFLOW_V_2_10_PLUS, reason="Hook lineage works in Airflow >= 2.10.0")
+def test_extractor_manager_gets_data_from_pythonoperator(session, dag_maker, hook_lineage_collector):
+ path = None
+ with tempfile.NamedTemporaryFile() as f:
+ path = f.name
+ with dag_maker():
+
+ def use_read():
+ storage_path = ObjectStoragePath(path)
+ with storage_path.open("w") as out:
+ out.write("test")
+
+ task = PythonOperator(task_id="test_task_extractor_pythonoperator", python_callable=use_read)
+
+ dr = dag_maker.create_dagrun()
+ ti = TaskInstance(task=task, run_id=dr.run_id)
+ ti.state = State.QUEUED
+ session.merge(ti)
+ session.commit()
+
+ ti.run()
+
+ datasets = hook_lineage_collector.collected_datasets
+
+ assert len(datasets.outputs) == 1
+ assert datasets.outputs[0].dataset == Dataset(uri=path)