You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by mo...@apache.org on 2023/08/02 09:42:09 UTC
[airflow] branch openlineage-sagemaker-operators updated (0d8ea29f54 -> 21da40d741)
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a change to branch openlineage-sagemaker-operators
in repository https://gitbox.apache.org/repos/asf/airflow.git
discard 0d8ea29f54 openlineage, sagemaker: add OpenLineage support for SageMaker's Processing, Transform and Training operators
add 34edb6508d Add protection against accidental Providers Manager initialization (#32694)
add b9c84eb663 add deferrable mode to rds start & stop DB (#32437)
add a2eaca8977 Allow setup without teardown (#32679)
add 9d7c224681 D205 Support - Models (#32575)
add 94122d1a6f Fix chart named template comments (#32681)
add bbd290266c Fix bug in prune_dict where empty dict and list would be removed even in strict mode (#32573)
add 7092cfdbbf Fix bad delete logic for dagruns (#32684)
add 7123dc162b Remove click upper bound (#32634)
add faa8f543d5 Store config description in Airflow configuration object (#32669)
add dff360e561 build(pre-commit): add list of supported deferrable operators to doc (#32514)
add eea53a22c9 Add Eventbridge PutEvents operator and hook (#32498)
add 8b5da2134d Update Dag trigger API and command docs (#32696)
add 27b5f696a4 Add Deferrable mode for EMR Serverless Start Job Operator (#32534)
add 848c69a194 Refresh GKE OAuth2 tokens (#32673)
add 8b7ae76026 Fixup docs and optimize system test for DataprocSubmitJobOperator (Hadoop job) (#32722)
add e8287734cb Fixup docstring for deprecated DataprocSubmitHiveJobOperator (#32723)
add dda3dcdcfc Add deferrable mode to ExternalTaskSensor (#29260)
add ac524826f3 Refactor setup/teardown ctx mgr to operate freely with other task definitions (#32687)
add 3c14753b03 Fix BigQueryGetDataOperator where project_id is not being respected in deferrable mode (#32488)
add 99b8a90346 Filtering and ordering results of DataprocListBatchesOperator (#32500)
add 8e67546660 Fix DataformCreateWorkflowInvocationOperator system test (#32599)
add 75ed3bc3f8 [bugfix] fix AWS triggers where deserialization would crash if region was not specified (#32729)
add d70fecfaf6 Add initial docs for setup / teardown (#32169)
add 15d42b4320 Reduce default for max TIs per query, enforce <= parallelism (#32572)
add c7c0deecb4 Refactor Sqlalchemy queries to 2.0 style (Part 6) (#32645)
add b09e1f97d5 Fix dagProcessor not including webserver-config volume (#32644)
add d5bf74c7b0 Removing hardcoded flag value and using constant (#32713)
add 978adb309a Install sqlalchemy-spanner package into Google provider (#31925)
add 531eb41bff Metrics - Enabled encrypted OTel Endpoint (#32524)
add 49921763eb Migrate system test for PostgresToGCSOperator to new design AIP-47 (#32641)
add 440c9eb2b4 Fixup system test for DataprocSubmitJobOperator (PySpark job) (#32740)
add fbeddc3017 Fixup docstring for deprecated DataprocSubmitPigJobOperator (#32739)
add 0fbb05a459 Fixup system test for DataprocSubmitJobOperator (SparkSQL job) (#32745)
add 98a999034c Add OpenLineage support for MySQL. (#31609)
add 5b082c38a6 openlineage, snowflake: add OpenLineage support for Snowflake (#31696)
add 00aa6ea72c Fix Pandas2 compatibility for Hive (#32752)
add 8e054bb57d Fix ARM image building after Cython 3.0.0 release (#32748)
add 067365f561 Arjunanan6/add company (#32742)
add 815655101b Add Deferrable mode to StepFunctionStartExecutionOperator (#32563)
add 73b90c48b1 Allow configuration to be contributed by providers (#32604)
add 62fe683f33 provider_info schema correction (#32754)
add d7899ecfaf Bring back automated airflow home dir creation (#32755)
add 3d89e75eb9 Fix failing Celery Executor tests after config migration (#32763)
add 53c6305bd0 Improve getting the query count in Airflow API endpoints (#32630)
add 3e467ba510 Fix prefix group false graph (#32764)
add 4c878798ef Sort extra_links for predictable order in UI. (#32762)
add fcc6f284c7 Update the watcher resource version in SparkK8SOp when it's too old (#32768)
add 43a5b47505 Improving the instructions for updating example dag paths (#32771)
add 57f203251b FIX AWS deferrable operators by using AioCredentials when using `assume_role` (#32733)
add accdb0bf64 Update PIP to 23.2.1 - released today (#32774)
add 56c41d460c Introduce decorator to load providers configuration (#32765)
add 82e6226738 Fix Datafusion system tests (#32749)
add 6362ba5ab4 Move writing configuration for webserver to main (webserver limited) (#32766)
add 42b4b43c4c Add Redis task handler (#31855)
add 01a61b0e7c Documentation Update to enhance readability (#32730)
add 505fe854e2 Remove unnecessary cast (#32782)
add 685328e357 Quarantine test_backfill_integration in dask executor (#32780)
add 73bc49adb1 Fix depends_on_past work for dynamic tasks (#32397)
add 0339f94a50 Fix failing dagrun query due to sqlalchemy 2.0 refactor (#32801)
add 6313e52932 Add missing execute_complete method for `DatabricksRunNowOperator` (#32689)
add 6f75a8a917 improve performance of pr script in dev/stats (#32735)
add d05e42e5d2 Add `ExternalBranchPythonOperator` (#32787)
add 8012c9fce6 Add support for querying Redshift Serverless clusters (#32785)
add 282854b55f Add endpoint_url in test_connection (#32664)
add 1706f05858 EMR serverless Create/Start/Stop/Delete Application deferrable mode (#32513)
add 031e3945e4 Allow auth managers to override the security manager (#32525)
add 2ac237b3eb Add pre-Airflow-2-7 hardcoded defaults for config for older providers (#32775)
add b40df0f938 Also write webserver_config for worker and internal-api (#32812)
add 0c0b8a3a0e Move Hive configuration to Apache Hive provider (#32777)
add f12b161a88 Chart: CeleryExecutor namespace depends on Airflow version (#32753)
add fd4af0b5c3 Remove atlas configuration definition (#32776)
add 0c798252ef Extract Dask executor to new daskexecutor provider (#32772)
add f57ee511e0 Remove comment re cli fallbacks, they have been removed (#32818)
add c4b6f06f6e Handle login by auth managers (#32697)
add 58e21c66fd fix(providers/databricks): remove the execute method from to-be-deprecated DatabricksRunNowDeferrableOperator (#32806)
add ed689f2be9 Display execution_date in graph view task instance tooltip. (#32527)
add ca202512af listener: call on_task_instance_running after rendering templates (#32716)
add 6b880844ad openlineage, sftp: add OpenLineage support for sftp provider (#31360)
add a23bf4c06a plugins: load plugins from providers (#32692)
add ce2841bf6a Add default port to Openlineage authority method. (#32828)
add 9f3af9c862 Fix misspelled auth test package (#32830)
add 9570cb1482 Make Start and Stop SageMaker Pipelines operators deferrable (#32683)
add f2108892e8 Catch arbitrary exception from run_job to prevent zombie scheduler (#32707)
add 8809478f44 Refactor existence-checking SQL to helper (#32790)
add e93460383f Move all k8S classes to cncf.kubernetes provider (#32767)
add b7a0983b66 docs for `DataprocSubmitJobOperator` (Presto job) (#32798)
add a85d546276 Fix broken getting webserver config file from settings.WEBSERVER (#32835)
add 48fa7b5553 Fix system test for DataprocSubmitJobOperator (SparkR) (#32746)
add 4c2ef99772 Add Spot Instances support with Dataproc Operators (#31644)
add 43f7da702f Add system test and docs for DataprocSubmitJobOperator (Trino job) (#32796)
add 6b113816f5 openlineage, postgres: add OpenLineage support for Postgres (#31617)
add f17bc0f4bf Adding support for disabling token mounts using automountServiceAccountToken (#32808)
add f2e93310c4 Move user details view to fab auth manager (#32756)
add 583f407c54 Fixup docstring for deprecated DataprocSubmitSparkJobOperator and refactoring system tests (#32743)
add 9e3104b5ef Fix DataflowStartFlexTemplateOperator system test (#32584)
add 579ce065c5 Override base log folder by using task handler's base_log_folder (#32781)
add 3237cb3484 Adjust graph zoom based on selected task (#32792)
add 9194144dab Replace Ruff setting known-third-party with namespace-packages (#32873)
add 5aa62de16a Set openlineage provider as ready to be released (#32854)
add b73366799d openlineage, gcs: add openlineage methods for GcsToGcsOperator (#31350)
add 9d68492f87 misc: update MLEngine system tests (#32881)
add ed00026417 Fix hash for breeze dependencies (#32895)
add 42d3377b60 Apply pypi suffix to provider dependencies (including apache-airflow) (#32896)
add 46d7125214 Add missing deprecated imports after k8s move (#32891)
add 25124dfd8b Limit Appflow mypy to 1.28.12 as it introduces strange typing issue (#32901)
add 479cd43d6a New gantt tab (#31806)
add db8d737ad6 Fix typo in Dockerfile (#32898)
add 7ba7fb1173 Order the release commands properly (#32888)
add 980065aaf1 Make `get_url_user_profile()` in auth manager optional (#32886)
add 4b005147c1 Encourage PR description to leave an empty line (#32905)
add 319045492d Add get_user method to auth manager (#32838)
add 12228d16be Unwrap Proxy before checking __iter__ in is_container() (#32850)
add f8aae7946e Also add PyPI suffix to Arm images (#32902)
add 8a0f410010 Update openlineage provider to min version of airflow 2.7.0 (#32882)
add 5c8223c335 Bump common-sql version in provider (#32907)
add d800c1bc39 Support task mapping with setup teardown (#32820)
add 0924389a87 Fix MIN_AIRFLOW_VERSION_EXCEPTIONS for openlineage (#32909)
add f91c003092 fix: remove K8S 1.23 support (#32899)
add 0fbb05751a Remove old gantt chart and redirect to grid views gantt tab (#32908)
add f598b0644e Update CONTRIBUTORS_QUICK_START.rst (#32924)
add ddb685cf26 Further limit mypy-boto3-appflow as the fix is not in sight (#32927)
add 89404530aa Refactor WasbHook tests (#32922)
add fcbbf47864 [AIP-51] Executors vending CLI commands (#29055)
add 81b85ebcbd Fix `BigQueryInsertJobOperator` not exiting deferred state (#31591)
add f31af91dbd Raise original import error in CLI vending of executors (#32931)
add 1f21e44bea Fix cache builing on CI and building CI images locally (#32933)
add a5ac8f5f63 Change `area:kubernetes` label to `provider:kubernetes` (#32932)
add d51fa94aa2 Chart: Simplify looping in tests (#32929)
add 4a68e83d45 Assign labels for all providers (#32935)
add bcc7856b80 Add `termination_message_policy` parameter to `KubernetesPodOperator` (#32885)
add d06b7af69a Prepare docs for July 2023 3rd wave of Providers (#32875)
add ae351114b8 Fix case where selective checks skip Helm tests on helm-test-only change (#32941)
add b7fa5ad272 Normalising the helm service account tests (#32874)
add 05c870abd1 Allow increasing delay in breeze start-airflow (#32938)
add 16e0830a5d Make Http provider sync and async test consistent (#32715)
add ddcd474a5e Fix check if `virtualenv` is installed in `PythonVirtualenvOperator` (#32939)
add 1bc680fb66 Change latest redirection to stable one for microsoft azure -> google. (#32945)
add ec60b77e3b Fix README_RELEASE_PROVIDER_PACKAGES.md (#32944)
add 9ce15c8b1a Add docstring to the build-docs breeze developer command (#32947)
add a76a8a2692 Add back commit SHA to image generation (#32949)
add 143d5e5925 Add more detailed instructions on how to check pre-release candidates (#32948)
add 4cbc3bfadb Better links for manual testing process (#32952)
add e1dd9b5cd2 openlineage, ftp: add OpenLineage support for FTPFileTransferOperator (#31354)
add e2e2253bf5 Remove instruction on upgrade check release (#32953)
add 12b0b6b607 Fix mapped setup teardown classic operator (#32934)
add f32a4c7932 Cleans up Extras reference page (#32954)
add c422920c6e Add system test and docs for CloudDataTransferServiceGCSToGCSOperator (#32960)
add ce5eebd004 Fix system test for MetastoreHivePartitionSensor (#32861)
add 42465c5a94 Fix Dataproc system tests (#32807)
add f9cddf332d openlineage,gcs: use proper name for openlineage methods (#32956)
add ddcd30e7c7 Validate connection host field for Sqoop connection (#32968)
add 4f83e831d2 Validate conn_prefix in extra field for Spark JDBC hook (#32946)
add e46929b7b3 Quarantine the test test_send_tasks_to_celery_hang (#32971)
add 915f9e4060 Add GCS Requester Pays bucket support to GCSToS3Operator (#32760)
add 357d35c76d Output for quarantined tests is not hidden by default in CI (#32975)
add 196d336585 Add Amazon EventBridge PutRule hook and operator (#32869)
add 86193f5608 Increase the number of attempts in AWS system test `example_rds_export` (#32976)
add 2b0d88e450 Handle logout by auth manager (#32819)
add 2ab78ec441 Fix connection parameters of `SnowflakeValueCheckOperator` (#32605)
add 8f12e7e4a9 Make the decorators of `PythonOperator` sub-classes extend its decorator (#32845)
add 11629f65f1 Fix system-test pytest marker (#32978)
add 1e20ef215a Rename `db upgrade` to `db migrate` and add `connections create-default-connections` (#32810)
add 05494e5d8e Fix example_local_to_wasb system test (#32982)
add 17a3dd4135 Move redis subscribe to poke() method in Redis Sensor (#32984)
add 941c83e1a9 listener: move success hook to after SQLAlchemy commit (#32988)
add f86e3a1cf7 fix: save codemirror content to extra textarea (#32893)
add 679c5b6177 Limit Asana Python client until provider is adapted to 4.* version (#32995)
add a3afd7d4a8 Remove stale comment on setup cannot be mapped (#32998)
add cc13db83e8 Add Ryan Hatter to collaborators (#33008)
add 9b7ed4329a Remove is_setup flag from TaskInstance (#32997)
add 2a39914cbd Don't use database as fallback when no schema parsed. (#32959)
add 9fb26db0e7 Revert "Add links to DAGRun / DAG / Task in templates-ref.rst (#32245)" (#33010)
add e79769cf9b Clean-up of our new security page (#32951)
add 0347bfff90 Fix invalid rule name in example_eventbridge.py (#33015)
add 5e78a09495 Fail stop feature can work with setup / teardown (#32985)
add 60da13d191 Don't require that a teardown have a "work" task (#33018)
add 5f5293fc5a Refactor system tests for DataprocSubmitJobOperator (PySpark, SparkR jobs) (#33003)
add df74553ec4 Refactor account_url use in WasbHook (#32980)
new 21da40d741 openlineage, sagemaker: add OpenLineage support for SageMaker's Processing, Transform and Training operators
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0d8ea29f54)
\
N -- N -- N refs/heads/openlineage-sagemaker-operators (21da40d741)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.asf.yaml | 1 +
.github/CODEOWNERS | 1 -
.../airflow_providers_bug_report.yml | 1 +
.github/PULL_REQUEST_TEMPLATE.md | 3 +
.github/actions/build-ci-images/action.yml | 1 +
.github/actions/build-prod-images/action.yml | 2 +
.github/actions/migration_tests/action.yml | 8 +-
.github/boring-cyborg.yml | 527 +++++--
.github/mergeable.yml | 5 -
.github/workflows/build-images.yml | 3 +
.github/workflows/ci.yml | 63 +-
.github/workflows/release_dockerhub_image.yml | 5 +
.gitignore | 2 -
.pre-commit-config.yaml | 24 +-
.rat-excludes | 1 -
BREEZE.rst | 7 +
CONTRIBUTING.rst | 18 +-
CONTRIBUTORS_QUICK_START.rst | 12 +
Dockerfile | 78 +-
Dockerfile.ci | 65 +-
IMAGES.rst | 4 +-
INSTALL | 18 +-
INTHEWILD.md | 1 +
LICENSE | 1 -
LOCAL_VIRTUALENV.rst | 2 +-
README.md | 18 +-
STATIC_CODE_CHECKS.rst | 4 +-
TESTING.rst | 145 +-
airflow/__init__.py | 16 +-
airflow/__main__.py | 12 +
airflow/api/auth/backend/basic_auth.py | 2 +-
airflow/api/auth/backend/session.py | 4 +-
airflow/api_connexion/endpoints/dag_endpoint.py | 7 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 18 +-
.../endpoints/dag_warning_endpoint.py | 5 +-
.../api_connexion/endpoints/dataset_endpoint.py | 3 +-
.../endpoints/role_and_permission_endpoint.py | 2 +-
.../endpoints/task_instance_endpoint.py | 16 +-
airflow/api_connexion/endpoints/user_endpoint.py | 2 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 5 +-
airflow/api_connexion/openapi/v1.yaml | 5 +-
.../schemas/role_and_permission_schema.py | 2 +-
airflow/api_connexion/schemas/user_schema.py | 2 +-
airflow/auth/managers/base_auth_manager.py | 59 +-
.../{api => auth/managers/fab}/auth/__init__.py | 0
airflow/auth/managers/fab/auth/anonymous_user.py | 50 +
airflow/auth/managers/fab/fab_auth_manager.py | 44 +-
.../sqla => auth/managers/fab}/models.py | 5 +-
.../auth/managers/fab/security_manager_override.py | 274 ++++
.../managers/fab/views}/__init__.py | 0
airflow/auth/managers/fab/views/user_details.py | 55 +
.../{api/auth => auth/managers/model}/__init__.py | 0
.../managers/model/base_user.py} | 28 +-
airflow/cli/cli_config.py | 251 ++-
airflow/cli/cli_parser.py | 28 +-
airflow/cli/commands/celery_command.py | 12 +-
airflow/cli/commands/config_command.py | 20 +-
airflow/cli/commands/connection_command.py | 13 +
airflow/cli/commands/dag_command.py | 23 +-
airflow/cli/commands/dag_processor_command.py | 2 +
airflow/cli/commands/db_command.py | 29 +-
airflow/cli/commands/info_command.py | 2 +
airflow/cli/commands/internal_api_command.py | 2 +
airflow/cli/commands/jobs_command.py | 2 +
airflow/cli/commands/kerberos_command.py | 2 +
airflow/cli/commands/kubernetes_command.py | 13 +-
airflow/cli/commands/plugins_command.py | 2 +
airflow/cli/commands/pool_command.py | 7 +
airflow/cli/commands/provider_command.py | 52 +
airflow/cli/commands/role_command.py | 9 +-
airflow/cli/commands/rotate_fernet_key_command.py | 2 +
airflow/cli/commands/scheduler_command.py | 10 +-
airflow/cli/commands/standalone_command.py | 2 +
airflow/cli/commands/sync_perm_command.py | 2 +
airflow/cli/commands/task_command.py | 7 +
airflow/cli/commands/triggerer_command.py | 2 +
airflow/cli/commands/user_command.py | 7 +
airflow/cli/commands/variable_command.py | 7 +
airflow/cli/commands/webserver_command.py | 2 +
airflow/config_templates/__init__.py | 2 +-
airflow/config_templates/config.yml | 595 +------
airflow/config_templates/config.yml.schema.json | 5 +
airflow/config_templates/default_airflow.cfg | 1513 +-----------------
airflow/config_templates/default_test.cfg | 68 -
airflow/config_templates/pre_2_7_defaults.cfg | 93 ++
airflow/config_templates/unit_tests.cfg | 102 ++
airflow/configuration.py | 942 +++++++++---
airflow/decorators/__init__.pyi | 2 +-
airflow/decorators/base.py | 17 +-
airflow/decorators/branch_python.py | 29 +-
airflow/decorators/external_python.py | 34 +-
airflow/decorators/python_virtualenv.py | 31 +-
airflow/decorators/short_circuit.py | 31 +-
.../example_dags/example_kubernetes_executor.py | 5 +-
.../example_setup_teardown_taskflow.py | 85 +-
airflow/exceptions.py | 43 +-
airflow/executors/__init__.py | 33 +-
airflow/executors/base_executor.py | 10 +
airflow/executors/executor_loader.py | 24 +-
airflow/kubernetes/__init__.py | 138 ++
.../__init__.py} | 18 +-
.../{ => pre_7_4_0_compatibility}/k8s_model.py | 0
.../{ => pre_7_4_0_compatibility}/kube_client.py | 0
.../{ => pre_7_4_0_compatibility}/pod_generator.py | 33 +-
.../pod_generator_deprecated.py | 0
.../{ => pre_7_4_0_compatibility}/secret.py | 2 +-
airflow/kubernetes/volume.py | 33 -
airflow/kubernetes/volume_mount.py | 33 -
airflow/metrics/otel_logger.py | 6 +-
.../versions/0073_2_0_0_prefix_dag_permissions.py | 2 +-
airflow/models/__init__.py | 2 -
airflow/models/abstractoperator.py | 80 +-
airflow/models/base.py | 2 +-
airflow/models/baseoperator.py | 135 +-
airflow/models/connection.py | 23 +-
airflow/models/crypto.py | 5 +-
airflow/models/dag.py | 153 +-
airflow/models/dagbag.py | 22 +-
airflow/models/dagpickle.py | 11 +-
airflow/models/dagrun.py | 10 +-
airflow/models/errors.py | 5 +-
airflow/models/mappedoperator.py | 49 +-
airflow/models/param.py | 28 +-
airflow/models/pool.py | 10 +-
airflow/models/renderedtifields.py | 12 +-
airflow/models/serialized_dag.py | 73 +-
airflow/models/skipmixin.py | 5 +-
airflow/models/slamiss.py | 4 +-
airflow/models/taskinstance.py | 233 +--
airflow/models/taskmixin.py | 16 +-
airflow/models/taskreschedule.py | 3 +-
airflow/models/trigger.py | 104 +-
airflow/models/variable.py | 10 +-
airflow/models/xcom.py | 3 +-
airflow/models/xcom_arg.py | 8 +-
airflow/operators/python.py | 19 +-
airflow/plugins_manager.py | 32 +-
airflow/provider_info.schema.json | 12 +-
.../providers/CREATING_COMMUNITY_PROVIDERS.rst | 173 +--
airflow/providers/amazon/CHANGELOG.rst | 32 +
airflow/providers/amazon/__init__.py | 6 +-
airflow/providers/amazon/aws/hooks/base_aws.py | 41 +-
airflow/providers/amazon/aws/hooks/emr.py | 26 +-
airflow/providers/amazon/aws/hooks/eventbridge.py | 90 ++
.../providers/amazon/aws/hooks/redshift_data.py | 10 +
airflow/providers/amazon/aws/hooks/sagemaker.py | 38 +-
.../providers/amazon/aws/notifications/chime.py | 2 +-
airflow/providers/amazon/aws/operators/emr.py | 224 ++-
.../providers/amazon/aws/operators/eventbridge.py | 172 +++
airflow/providers/amazon/aws/operators/rds.py | 177 ++-
.../amazon/aws/operators/redshift_data.py | 7 +
.../providers/amazon/aws/operators/sagemaker.py | 192 ++-
.../amazon/aws/operators/step_function.py | 36 +-
.../providers/amazon/aws/transfers/gcs_to_s3.py | 32 +-
airflow/providers/amazon/aws/triggers/batch.py | 2 +-
airflow/providers/amazon/aws/triggers/ecs.py | 5 +-
airflow/providers/amazon/aws/triggers/eks.py | 6 +-
airflow/providers/amazon/aws/triggers/emr.py | 216 ++-
airflow/providers/amazon/aws/triggers/rds.py | 166 +-
airflow/providers/amazon/aws/triggers/sagemaker.py | 85 +-
.../providers/amazon/aws/triggers/step_function.py | 59 +
airflow/providers/amazon/aws/utils/__init__.py | 17 +-
airflow/providers/amazon/aws/utils/rds.py | 4 +-
.../providers/amazon/aws/waiters/sagemaker.json | 46 +
.../amazon/aws/waiters/stepfunctions.json | 36 +
airflow/providers/amazon/provider.yaml | 20 +-
airflow/providers/apache/beam/CHANGELOG.rst | 16 +
airflow/providers/apache/beam/__init__.py | 6 +-
airflow/providers/apache/beam/provider.yaml | 1 +
airflow/providers/apache/drill/CHANGELOG.rst | 15 +
airflow/providers/apache/drill/__init__.py | 6 +-
airflow/providers/apache/drill/provider.yaml | 1 +
airflow/providers/apache/druid/CHANGELOG.rst | 16 +
airflow/providers/apache/druid/__init__.py | 6 +-
airflow/providers/apache/druid/provider.yaml | 1 +
airflow/providers/apache/hive/CHANGELOG.rst | 14 +
airflow/providers/apache/hive/__init__.py | 6 +-
airflow/providers/apache/hive/hooks/hive.py | 2 +-
airflow/providers/apache/hive/provider.yaml | 21 +
airflow/providers/apache/pinot/CHANGELOG.rst | 17 +
airflow/providers/apache/pinot/__init__.py | 6 +-
airflow/providers/apache/pinot/provider.yaml | 1 +
airflow/providers/apache/spark/CHANGELOG.rst | 23 +
airflow/providers/apache/spark/__init__.py | 6 +-
airflow/providers/apache/spark/hooks/spark_jdbc.py | 2 +
.../providers/apache/spark/hooks/spark_submit.py | 2 +-
airflow/providers/apache/spark/provider.yaml | 6 +
airflow/providers/apache/sqoop/hooks/sqoop.py | 3 +
airflow/providers/apprise/CHANGELOG.rst | 8 +
airflow/providers/apprise/__init__.py | 6 +-
airflow/providers/apprise/hooks/apprise.py | 3 +-
airflow/providers/apprise/notifications/apprise.py | 3 +-
airflow/providers/apprise/provider.yaml | 1 +
airflow/providers/asana/provider.yaml | 6 +-
airflow/providers/celery/CHANGELOG.rst | 29 +-
airflow/providers/celery/__init__.py | 6 +-
.../providers/celery/executors/celery_executor.py | 165 +-
.../celery/executors/celery_executor_utils.py | 2 +-
.../celery/executors/celery_kubernetes_executor.py | 32 +-
.../providers/celery/executors/default_celery.py | 22 +-
airflow/providers/celery/provider.yaml | 258 ++++
airflow/providers/cncf/kubernetes/CHANGELOG.rst | 32 +-
airflow/providers/cncf/kubernetes/__init__.py | 6 +-
.../cncf/kubernetes/executors}/__init__.py | 0
.../kubernetes}/executors/kubernetes_executor.py | 133 +-
.../executors/kubernetes_executor_types.py | 0
.../executors/kubernetes_executor_utils.py | 38 +-
.../executors/local_kubernetes_executor.py | 6 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 2 +-
.../{ => providers/cncf}/kubernetes/k8s_model.py | 0
.../{ => providers/cncf}/kubernetes/kube_client.py | 8 +-
.../{ => providers/cncf}/kubernetes/kube_config.py | 1 -
.../kubernetes_executor_templates}/__init__.py | 0
.../basic_template.yaml | 0
.../kubernetes/kubernetes_helper_functions.py | 12 +-
airflow/providers/cncf/kubernetes/operators/pod.py | 11 +-
.../cncf/kubernetes/operators/spark_kubernetes.py | 30 +-
.../cncf}/kubernetes/pod_generator.py | 20 +-
.../cncf}/kubernetes/pod_generator_deprecated.py | 5 +-
.../cncf}/kubernetes/pod_launcher_deprecated.py | 4 +-
.../pod_template_file_examples}/__init__.py | 0
.../dags_in_image_template.yaml | 0
.../dags_in_volume_template.yaml | 0
.../git_sync_template.yaml | 0
airflow/providers/cncf/kubernetes/provider.yaml | 210 +++
airflow/{ => providers/cncf}/kubernetes/secret.py | 2 +-
.../cncf/kubernetes/template_rendering.py | 67 +
.../cncf/kubernetes/utils/k8s_hashlib_wrapper.py} | 10 +-
.../providers/cncf/kubernetes/utils/pod_manager.py | 4 +-
airflow/providers/common/sql/CHANGELOG.rst | 14 +
airflow/providers/common/sql/__init__.py | 6 +-
airflow/providers/common/sql/hooks/sql.py | 10 +-
airflow/providers/common/sql/provider.yaml | 1 +
.../{apprise => daskexecutor}/CHANGELOG.rst | 2 +-
.../{apache/pinot => daskexecutor}/__init__.py | 8 +-
.../daskexecutor/executors}/__init__.py | 0
.../daskexecutor}/executors/dask_executor.py | 0
airflow/providers/daskexecutor/provider.yaml | 79 +
airflow/providers/databricks/CHANGELOG.rst | 14 +
airflow/providers/databricks/__init__.py | 6 +-
.../providers/databricks/operators/databricks.py | 12 +-
airflow/providers/databricks/provider.yaml | 1 +
airflow/providers/exasol/CHANGELOG.rst | 15 +
airflow/providers/exasol/__init__.py | 6 +-
airflow/providers/exasol/provider.yaml | 1 +
airflow/providers/ftp/operators/ftp.py | 62 +
airflow/providers/google/CHANGELOG.rst | 36 +
airflow/providers/google/__init__.py | 6 +-
.../cloud/example_dags/example_postgres_to_gcs.py | 51 -
airflow/providers/google/cloud/hooks/bigquery.py | 31 +-
airflow/providers/google/cloud/hooks/datafusion.py | 23 +-
airflow/providers/google/cloud/hooks/dataproc.py | 12 +
airflow/providers/google/cloud/hooks/gcs.py | 52 +-
.../google/cloud/hooks/kubernetes_engine.py | 4 +
airflow/providers/google/cloud/hooks/spanner.py | 49 +-
.../providers/google/cloud/operators/bigquery.py | 34 +-
.../operators/cloud_storage_transfer_service.py | 5 +-
.../providers/google/cloud/operators/dataproc.py | 61 +-
airflow/providers/google/cloud/operators/gcs.py | 8 +-
.../providers/google/cloud/operators/mlengine.py | 3 +
.../providers/google/cloud/transfers/gcs_to_gcs.py | 29 +
.../google/cloud/transfers/postgres_to_gcs.py | 4 +
.../providers/google/cloud/triggers/bigquery.py | 63 +-
airflow/providers/google/provider.yaml | 5 +-
airflow/providers/jdbc/CHANGELOG.rst | 15 +
airflow/providers/jdbc/__init__.py | 6 +-
airflow/providers/jdbc/provider.yaml | 1 +
airflow/providers/microsoft/azure/CHANGELOG.rst | 9 +
airflow/providers/microsoft/azure/__init__.py | 6 +-
airflow/providers/microsoft/azure/hooks/wasb.py | 32 +-
.../microsoft/azure/log/wasb_task_handler.py | 2 +
airflow/providers/microsoft/azure/provider.yaml | 1 +
airflow/providers/microsoft/mssql/CHANGELOG.rst | 16 +
airflow/providers/microsoft/mssql/__init__.py | 6 +-
airflow/providers/microsoft/mssql/provider.yaml | 1 +
airflow/providers/mysql/CHANGELOG.rst | 22 +
airflow/providers/mysql/__init__.py | 6 +-
airflow/providers/mysql/hooks/mysql.py | 25 +
airflow/providers/mysql/provider.yaml | 1 +
airflow/providers/neo4j/CHANGELOG.rst | 16 +
airflow/providers/neo4j/__init__.py | 6 +-
airflow/providers/neo4j/provider.yaml | 1 +
airflow/providers/openlineage/__init__.py | 6 +-
airflow/providers/openlineage/extractors/base.py | 22 +-
airflow/providers/openlineage/extractors/bash.py | 3 +-
.../providers/openlineage/extractors/manager.py | 3 +-
airflow/providers/openlineage/extractors/python.py | 3 +-
airflow/providers/openlineage/plugins/adapter.py | 12 +-
airflow/providers/openlineage/plugins/facets.py | 1 -
airflow/providers/openlineage/provider.yaml | 4 +-
airflow/providers/openlineage/sqlparser.py | 8 +-
airflow/providers/openlineage/utils/sql.py | 3 +-
airflow/providers/openlineage/utils/utils.py | 6 +-
airflow/providers/oracle/CHANGELOG.rst | 16 +
airflow/providers/oracle/__init__.py | 6 +-
airflow/providers/oracle/provider.yaml | 1 +
airflow/providers/postgres/CHANGELOG.rst | 13 +
airflow/providers/postgres/__init__.py | 6 +-
airflow/providers/postgres/hooks/postgres.py | 48 +-
airflow/providers/postgres/provider.yaml | 1 +
airflow/providers/presto/CHANGELOG.rst | 16 +
airflow/providers/presto/__init__.py | 6 +-
airflow/providers/presto/provider.yaml | 1 +
airflow/providers/redis/CHANGELOG.rst | 15 +
airflow/providers/redis/__init__.py | 6 +-
.../{api/auth => providers/redis/log}/__init__.py | 0
airflow/providers/redis/log/redis_task_handler.py | 105 ++
airflow/providers/redis/provider.yaml | 4 +
airflow/providers/redis/sensors/redis_pub_sub.py | 9 +-
airflow/providers/sftp/CHANGELOG.rst | 8 +
airflow/providers/sftp/__init__.py | 6 +-
airflow/providers/sftp/operators/sftp.py | 82 +
airflow/providers/sftp/provider.yaml | 1 +
airflow/providers/slack/CHANGELOG.rst | 16 +
airflow/providers/slack/__init__.py | 6 +-
airflow/providers/slack/provider.yaml | 1 +
airflow/providers/snowflake/CHANGELOG.rst | 14 +
airflow/providers/snowflake/__init__.py | 6 +-
airflow/providers/snowflake/hooks/snowflake.py | 71 +-
airflow/providers/snowflake/operators/snowflake.py | 65 +-
airflow/providers/snowflake/provider.yaml | 1 +
airflow/providers/sqlite/CHANGELOG.rst | 15 +
airflow/providers/sqlite/__init__.py | 6 +-
airflow/providers/sqlite/provider.yaml | 1 +
airflow/providers/trino/CHANGELOG.rst | 13 +
airflow/providers/trino/__init__.py | 6 +-
airflow/providers/trino/provider.yaml | 1 +
airflow/providers/vertica/CHANGELOG.rst | 8 +
airflow/providers/vertica/__init__.py | 6 +-
airflow/providers/vertica/provider.yaml | 1 +
airflow/providers_manager.py | 89 +-
airflow/sensors/external_task.py | 43 +
airflow/serialization/serialized_objects.py | 9 +-
airflow/serialization/serializers/kubernetes.py | 7 +-
airflow/settings.py | 2 +-
airflow/stats.py | 7 +-
airflow/ti_deps/deps/prev_dagrun_dep.py | 116 +-
airflow/ti_deps/deps/trigger_rule_dep.py | 27 +-
airflow/timetables/_cron.py | 4 +-
airflow/triggers/external_task.py | 80 +-
airflow/utils/cli.py | 3 +-
airflow/utils/db.py | 53 +-
airflow/utils/db_cleanup.py | 6 +-
airflow/utils/deprecation_tools.py | 68 +-
.../amazon/aws/utils/rds.py => utils/empty_set.py} | 15 +-
airflow/utils/hashlib_wrapper.py | 5 +-
airflow/utils/helpers.py | 11 +-
airflow/utils/log/file_task_handler.py | 4 +-
airflow/utils/providers_configuration_loader.py | 58 +
airflow/utils/serve_logs.py | 25 +-
airflow/utils/setup_teardown.py | 246 ++-
airflow/utils/sqlalchemy.py | 8 +-
airflow/www/app.py | 5 +-
airflow/www/auth.py | 16 +-
airflow/www/decorators.py | 8 +-
airflow/www/extensions/init_appbuilder.py | 26 +-
.../extensions/init_auth_manager.py} | 30 +-
airflow/www/extensions/init_jinja_globals.py | 5 +-
airflow/www/extensions/init_security.py | 11 +-
airflow/www/fab_security/manager.py | 228 +--
airflow/www/fab_security/sqla/manager.py | 9 +-
airflow/www/fab_security/views.py | 34 -
airflow/www/package.json | 4 +-
airflow/www/security.py | 45 +-
airflow/www/static/css/gantt.css | 50 -
airflow/www/static/js/api/useDatasets.ts | 1 +
.../static/js/cluster-activity/nav/FilterBar.tsx | 2 +
.../www/static/js/cluster-activity/useFilters.tsx | 3 +
airflow/www/static/js/connection_form.js | 32 +-
airflow/www/static/js/dag/Main.tsx | 6 +
.../static/js/dag/details/gantt/GanttTooltip.tsx | 79 +
airflow/www/static/js/dag/details/gantt/Row.tsx | 139 ++
airflow/www/static/js/dag/details/gantt/index.tsx | 174 +++
airflow/www/static/js/dag/details/graph/Node.tsx | 1 +
airflow/www/static/js/dag/details/graph/index.tsx | 99 +-
airflow/www/static/js/dag/details/graph/utils.ts | 36 +-
airflow/www/static/js/dag/details/index.tsx | 53 +-
airflow/www/static/js/dag/grid/index.tsx | 37 +-
airflow/www/static/js/dag/nav/FilterBar.tsx | 1 +
airflow/www/static/js/dag/useFilters.test.tsx | 1 +
airflow/www/static/js/dag/useFilters.tsx | 1 +
airflow/www/static/js/gantt.js | 431 ------
airflow/www/static/js/types/api-generated.ts | 2 +
airflow/www/static/js/types/index.ts | 1 +
airflow/www/static/js/utils/graph.ts | 72 +-
airflow/www/templates/airflow/gantt.html | 83 -
airflow/www/templates/appbuilder/flash.html | 2 +-
airflow/www/templates/appbuilder/navbar_right.html | 11 +-
airflow/www/utils.py | 23 +-
airflow/www/views.py | 157 +-
airflow/www/webpack.config.js | 1 -
airflow/www/yarn.lock | 116 +-
chart/README.md | 2 +-
chart/templates/_helpers.yaml | 30 +-
.../templates/cleanup/cleanup-serviceaccount.yaml | 1 +
.../dag-processor/dag-processor-deployment.yaml | 5 +
.../dag-processor-serviceaccount.yaml | 1 +
chart/templates/flower/flower-serviceaccount.yaml | 1 +
.../jobs/create-user-job-serviceaccount.yaml | 1 +
.../jobs/migrate-database-job-serviceaccount.yaml | 1 +
.../pgbouncer/pgbouncer-serviceaccount.yaml | 1 +
chart/templates/redis/redis-serviceaccount.yaml | 1 +
.../scheduler/scheduler-serviceaccount.yaml | 1 +
chart/templates/statsd/statsd-serviceaccount.yaml | 1 +
.../triggerer/triggerer-serviceaccount.yaml | 1 +
.../webserver/webserver-serviceaccount.yaml | 1 +
chart/templates/workers/worker-deployment.yaml | 2 +-
chart/templates/workers/worker-serviceaccount.yaml | 1 +
chart/values.schema.json | 60 +
chart/values.yaml | 36 +
dev/README_RELEASE_AIRFLOW.md | 35 +-
dev/README_RELEASE_AIRFLOW_UPGRADE_CHECK.md | 636 --------
dev/README_RELEASE_PROVIDER_PACKAGES.md | 24 +-
dev/breeze/README.md | 2 +-
dev/breeze/pyproject.toml | 4 +
.../airflow_breeze/commands/ci_image_commands.py | 13 +-
.../commands/ci_image_commands_config.py | 2 +
.../airflow_breeze/commands/developer_commands.py | 3 +
.../airflow_breeze/commands/kubernetes_commands.py | 2 +-
.../commands/production_image_commands.py | 4 +
.../commands/production_image_commands_config.py | 2 +
.../commands/release_management_commands.py | 7 +
.../commands/release_management_commands_config.py | 1 +
.../airflow_breeze/commands/testing_commands.py | 3 +
dev/breeze/src/airflow_breeze/global_constants.py | 6 +-
.../src/airflow_breeze/params/build_ci_params.py | 2 +
.../src/airflow_breeze/params/build_prod_params.py | 2 +
.../airflow_breeze/params/common_build_params.py | 2 +
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 3 +-
.../airflow_breeze/utils/add_back_references.py | 1 +
.../src/airflow_breeze/utils/common_options.py | 7 +
dev/breeze/src/airflow_breeze/utils/docs_errors.py | 4 +-
.../airflow_breeze/utils/publish_docs_builder.py | 3 +-
.../airflow_breeze/utils/publish_docs_helpers.py | 2 +
.../src/airflow_breeze/utils/selective_checks.py | 2 +-
.../src/airflow_breeze/utils/spelling_checks.py | 4 +-
dev/breeze/tests/test_selective_checks.py | 47 +-
dev/example_dags/README.md | 15 +-
.../PROVIDER__INIT__PY_TEMPLATE.py.jinja2 | 4 +-
dev/provider_packages/prepare_provider_packages.py | 11 +-
dev/stats/get_important_pr_candidates.py | 228 ++-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-amazon/commits.rst | 28 +
docs/apache-airflow-providers-amazon/index.rst | 14 +-
.../installing-providers-from-sources.rst | 2 +-
.../operators/emr/emr_serverless.rst | 8 +
.../operators/eventbridge.rst | 71 +
.../operators/step_functions.rst | 1 +
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../apache-airflow-providers-apache-beam/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 16 +
.../index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../commits.rst | 16 +-
.../configurations-ref.rst} | 2 +-
.../apache-airflow-providers-apache-hive/index.rst | 7 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../index.rst | 30 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-apprise/commits.rst | 11 +
docs/apache-airflow-providers-apprise/index.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-celery/changelog.rst | 2 -
docs/apache-airflow-providers-celery/commits.rst | 25 +
.../configurations-ref.rst} | 2 +-
docs/apache-airflow-providers-celery/index.rst | 31 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../commits.rst | 17 +
.../configurations-ref.rst} | 2 +-
.../index.rst | 3 +-
.../installing-providers-from-sources.rst | 2 +-
.../operators.rst | 2 +-
.../commits.rst | 16 +-
docs/apache-airflow-providers-common-sql/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 4 +-
.../commits.rst | 20 +-
.../configurations-ref.rst} | 2 +-
.../index.rst | 32 +-
.../installing-providers-from-sources.rst | 2 +-
.../security.rst | 0
.../commits.rst | 16 +-
docs/apache-airflow-providers-databricks/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-exasol/changelog.rst | 2 -
docs/apache-airflow-providers-exasol/commits.rst | 16 +
docs/apache-airflow-providers-exasol/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-google/commits.rst | 31 +-
.../configurations-ref.rst | 71 -
docs/apache-airflow-providers-google/index.rst | 9 +-
.../installing-providers-from-sources.rst | 2 +-
.../cloud/cloud_storage_transfer_service.rst | 28 +
.../operators/cloud/dataproc.rst | 74 +-
.../operators/transfer/postgres_to_gcs.rst | 52 +
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-jdbc/changelog.rst | 2 -
docs/apache-airflow-providers-jdbc/commits.rst | 16 +
docs/apache-airflow-providers-jdbc/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../commits.rst | 13 +
.../index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../redirects.txt | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 17 +
.../index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-mysql/changelog.rst | 2 -
docs/apache-airflow-providers-mysql/commits.rst | 19 +
docs/apache-airflow-providers-mysql/index.rst | 25 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-neo4j/changelog.rst | 2 -
docs/apache-airflow-providers-neo4j/commits.rst | 17 +
docs/apache-airflow-providers-neo4j/index.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../changelog.rst | 2 -
.../commits.rst | 39 +
.../guides/developer.rst | 2 +-
.../apache-airflow-providers-openlineage/index.rst | 38 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-oracle/changelog.rst | 2 -
docs/apache-airflow-providers-oracle/commits.rst | 17 +
docs/apache-airflow-providers-oracle/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-postgres/commits.rst | 15 +-
docs/apache-airflow-providers-postgres/index.rst | 19 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-presto/changelog.rst | 2 -
docs/apache-airflow-providers-presto/commits.rst | 17 +
docs/apache-airflow-providers-presto/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-redis/changelog.rst | 2 -
docs/apache-airflow-providers-redis/commits.rst | 16 +
docs/apache-airflow-providers-redis/index.rst | 9 +-
.../installing-providers-from-sources.rst | 2 +-
.../logging/index.rst} | 7 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-sftp/commits.rst | 14 +-
docs/apache-airflow-providers-sftp/index.rst | 19 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-slack/changelog.rst | 2 -
docs/apache-airflow-providers-slack/commits.rst | 17 +
docs/apache-airflow-providers-slack/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../apache-airflow-providers-snowflake/commits.rst | 16 +-
docs/apache-airflow-providers-snowflake/index.rst | 19 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-sqlite/changelog.rst | 2 -
docs/apache-airflow-providers-sqlite/commits.rst | 16 +
docs/apache-airflow-providers-sqlite/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-trino/commits.rst | 16 +-
docs/apache-airflow-providers-trino/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
docs/apache-airflow-providers-vertica/commits.rst | 14 +-
docs/apache-airflow-providers-vertica/index.rst | 6 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../installing-providers-from-sources.rst | 2 +-
.../core-extensions/configurations.rst} | 16 +-
.../core-extensions/deferrable-operator-ref.rst} | 9 +-
.../howto/create-custom-providers.rst | 268 ++++
docs/apache-airflow-providers/index.rst | 241 +--
.../apache-airflow-providers/redirects.txt | 3 +
.../dag-serialization.rst | 2 +-
.../logging-monitoring/metrics.rst | 17 +
.../production-deployment.rst | 8 +-
.../administration-and-deployment/scheduler.rst | 7 +-
.../authoring-and-scheduling/plugins.rst | 1 -
docs/apache-airflow/configurations-ref.rst | 83 +-
docs/apache-airflow/core-concepts/dags.rst | 18 +-
.../core-concepts/executor/celery.rst | 9 +
.../core-concepts/executor/celery_kubernetes.rst | 10 +
.../apache-airflow/core-concepts/executor/dask.rst | 4 +-
.../core-concepts/executor/kubernetes.rst | 14 +-
.../core-concepts/executor/local_kubernetes.rst | 9 +-
docs/apache-airflow/extra-packages-ref.rst | 175 ++-
.../howto/create-custom-decorator.rst | 3 +-
.../howto/docker-compose/docker-compose.yaml | 2 +-
docs/apache-airflow/howto/index.rst | 2 +-
.../howto/operator/external_task_sensor.rst | 9 +
docs/apache-airflow/howto/set-config.rst | 27 +-
docs/apache-airflow/howto/set-up-database.rst | 2 +-
docs/apache-airflow/howto/setup-and-teardown.rst | 184 +++
.../howto/upgrading-from-1-10/index.rst | 8 +-
docs/apache-airflow/howto/usage-cli.rst | 6 +-
docs/apache-airflow/howto/use-test-config.rst | 36 -
docs/apache-airflow/img/airflow_erd.sha256 | 2 +-
docs/apache-airflow/img/airflow_erd.svg | 1619 +++++++++++++++++++-
docs/apache-airflow/img/gantt.png | Bin 70606 -> 352148 bytes
.../installation/setting-up-the-database.rst | 12 +-
docs/apache-airflow/installation/upgrading.rst | 22 +-
docs/apache-airflow/migrations-ref.rst | 2 +-
docs/apache-airflow/redirects.txt | 3 +
docs/apache-airflow/security/index.rst | 148 +-
.../security/releasing_security_patches.rst | 49 +
.../security/{index.rst => security_model.rst} | 61 +-
docs/apache-airflow/start.rst | 2 +-
docs/apache-airflow/templates-ref.rst | 24 +-
docs/apache-airflow/tutorial/fundamentals.rst | 2 +-
docs/apache-airflow/tutorial/taskflow.rst | 46 +-
docs/conf.py | 37 +-
docs/docker-stack/build-arg-ref.rst | 4 +-
docs/docker-stack/entrypoint.rst | 10 +-
.../installing-providers-from-sources.rst | 0
.../includes/providers-configurations-ref.rst} | 14 +-
.../includes/sections-and-options.rst} | 30 +-
docs/exts/operators_and_hooks_ref.py | 84 +-
docs/exts/{ => templates}/auth_backend.rst.jinja2 | 0
.../exts/templates/configuration.rst.jinja2 | 10 +-
docs/exts/{ => templates}/connections.rst.jinja2 | 0
.../deferrable_operators_list.rst.jinja2} | 16 +-
docs/exts/{ => templates}/executors.rst.jinja2 | 0
docs/exts/{ => templates}/extra_links.rst.jinja2 | 0
docs/exts/{ => templates}/logging.rst.jinja2 | 0
docs/exts/{ => templates}/notifications.rst.jinja2 | 0
.../operators_and_hooks_ref-transfers.rst.jinja2 | 0
.../operators_and_hooks_ref.rst.jinja2 | 0
.../exts/{ => templates}/secret_backend.rst.jinja2 | 0
docs/helm-chart/production-guide.rst | 2 +-
.../aws/Amazon-EventBridge_64.png | Bin 0 -> 3221 bytes
docs/integration-logos/dask/dask.png | Bin 0 -> 6134 bytes
docs/spelling_wordlist.txt | 14 +
generated/provider_dependencies.json | 42 +-
helm_tests/airflow_aux/test_airflow_common.py | 11 +-
helm_tests/airflow_aux/test_cleanup_pods.py | 20 +
helm_tests/airflow_aux/test_create_user_job.py | 22 +
.../airflow_aux/test_migrate_database_job.py | 26 +
helm_tests/airflow_core/test_dag_processor.py | 67 +
helm_tests/airflow_core/test_scheduler.py | 22 +
helm_tests/airflow_core/test_triggerer.py | 22 +
helm_tests/airflow_core/test_worker.py | 40 +
helm_tests/other/test_flower.py | 26 +
helm_tests/other/test_pgbouncer.py | 28 +
helm_tests/other/test_redis.py | 26 +
helm_tests/other/test_statsd.py | 26 +
helm_tests/security/test_security_context.py | 80 +-
helm_tests/webserver/test_webserver.py | 22 +
images/breeze/output-commands-hash.txt | 56 +-
images/breeze/output-commands.svg | 2 +-
images/breeze/output_build-docs.svg | 130 +-
images/breeze/output_ci-image_build.svg | 50 +-
images/breeze/output_k8s_configure-cluster.svg | 6 +-
images/breeze/output_k8s_create-cluster.svg | 6 +-
images/breeze/output_k8s_delete-cluster.svg | 4 +-
images/breeze/output_k8s_deploy-airflow.svg | 6 +-
images/breeze/output_k8s_k9s.svg | 4 +-
images/breeze/output_k8s_logs.svg | 4 +-
images/breeze/output_k8s_run-complete-tests.svg | 6 +-
images/breeze/output_k8s_shell.svg | 4 +-
images/breeze/output_k8s_status.svg | 4 +-
images/breeze/output_k8s_tests.svg | 6 +-
images/breeze/output_k8s_upload-k8s-image.svg | 6 +-
images/breeze/output_prod-image_build.svg | 74 +-
...management_generate-issue-content-providers.svg | 72 +-
...e-management_prepare-provider-documentation.svg | 66 +-
...elease-management_prepare-provider-packages.svg | 40 +-
.../output_release-management_publish-docs.svg | 110 +-
...tput_release-management_release-prod-images.svg | 22 +-
images/breeze/output_sbom.svg | 20 +-
.../output_sbom_generate-provider-requirements.svg | 54 +-
images/breeze/output_static-checks.svg | 146 +-
images/breeze/output_testing.svg | 24 +-
images/breeze/output_testing_tests.svg | 114 +-
.../conftest.py | 15 +-
kubernetes_tests/test_kubernetes_pod_operator.py | 10 +-
licenses/LICENSE-moment-strftime.txt | 9 -
newsfragments/32572.significant.rst | 10 +
newsfragments/32767.significant.rst | 7 +
pyproject.toml | 32 +-
scripts/ci/installed_providers.txt | 1 +
.../pre_commit_check_airflow_k8s_not_used.py | 81 +
...it_check_cncf_k8s_used_for_k8s_executor_only.py | 84 +
scripts/ci/pre_commit/pre_commit_insert_extras.py | 2 +-
scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py | 179 ---
scripts/docker/common.sh | 2 +-
scripts/docker/entrypoint_ci.sh | 8 +-
scripts/docker/entrypoint_prod.sh | 12 +-
scripts/docker/install_mssql.sh | 41 +
scripts/in_container/airflow_ci.cfg | 59 -
scripts/in_container/check_environment.sh | 2 +-
.../in_container/run_provider_yaml_files_check.py | 2 +
setup.py | 66 +-
tests/always/test_providers_manager.py | 30 +-
.../endpoints/test_role_and_permission_endpoint.py | 2 +-
.../api_connexion/endpoints/test_user_endpoint.py | 2 +-
tests/api_connexion/schemas/test_user_schema.py | 2 +-
tests/auh/managers/__init__.py | 16 -
tests/auh/managers/fab/__init__.py | 16 -
tests/auh/managers/fab/test_fab_auth_manager.py | 57 -
{airflow/api_connexion => tests/auth}/__init__.py | 0
.../auth/managers}/__init__.py | 0
.../auth/managers/fab}/__init__.py | 0
tests/auth/managers/fab/test_fab_auth_manager.py | 124 ++
.../managers/fab/test_security_manager_override.py | 124 ++
tests/auth/managers/test_base_auth_manager.py | 52 +
tests/cli/commands/test_celery_command.py | 14 +-
tests/cli/commands/test_config_command.py | 147 +-
tests/cli/commands/test_connection_command.py | 12 +-
tests/cli/commands/test_db_command.py | 14 +-
tests/cli/commands/test_kubernetes_command.py | 6 +-
tests/cli/commands/test_role_command.py | 2 +-
tests/cli/commands/test_scheduler_command.py | 24 +
tests/cli/commands/test_task_command.py | 8 +-
tests/cli/conftest.py | 12 +
tests/cli/test_cli_parser.py | 159 +-
tests/config_templates/deprecated.cfg | 9 -
tests/config_templates/deprecated_cmd.cfg | 10 -
tests/config_templates/deprecated_secret.cfg | 10 -
tests/config_templates/empty.cfg | 9 -
tests/conftest.py | 22 +-
tests/core/test_config_templates.py | 98 --
tests/core/test_configuration.py | 230 ++-
tests/decorators/test_setup_teardown.py | 44 +
.../kubernetes_executor_template_files/__init__.py | 16 -
tests/executors/test_base_executor.py | 4 +
.../cli/commands/test_celery_command.py | 21 +-
.../integration/executors/test_celery_executor.py | 16 +-
.../providers/redis/sensors/test_redis_pub_sub.py | 2 +-
tests/kubernetes/__init__.py | 17 -
tests/kubernetes/models/__init__.py | 17 -
tests/models/test_baseoperator.py | 74 +-
tests/models/test_dag.py | 122 +-
tests/models/test_mappedoperator.py | 841 ++++++++++
tests/models/test_renderedtifields.py | 48 -
tests/models/test_taskinstance.py | 81 -
tests/models/test_taskmixin.py | 55 -
tests/operators/test_python.py | 216 ++-
tests/plugins/test_plugins_manager.py | 33 +
tests/providers/amazon/aws/hooks/test_base_aws.py | 24 +-
.../providers/amazon/aws/hooks/test_eventbridge.py | 39 +-
.../amazon/aws/hooks/test_redshift_data.py | 67 +-
.../amazon/aws/operators/test_emr_serverless.py | 114 +-
.../amazon/aws/operators/test_eventbridge.py | 105 ++
tests/providers/amazon/aws/operators/test_rds.py | 29 +
.../amazon/aws/operators/test_redshift_data.py | 42 +
.../aws/operators/test_sagemaker_pipeline.py | 48 +-
.../amazon/aws/operators/test_step_function.py | 17 +
.../amazon/aws/transfers/test_gcs_to_s3.py | 6 +-
.../amazon/aws/triggers/test_emr_serverless.py | 92 ++
tests/providers/amazon/aws/triggers/test_rds.py | 201 +--
.../amazon/aws/triggers/test_step_function.py | 53 +
tests/providers/amazon/aws/utils/test_utils.py | 12 -
.../apache/spark/hooks/test_spark_jdbc.py | 20 +
.../apache/spark/hooks/test_spark_submit.py | 2 +-
tests/providers/apache/sqoop/hooks/test_sqoop.py | 15 +
.../celery/executors/test_celery_executor.py | 12 +-
.../executors/test_celery_kubernetes_executor.py | 7 +-
tests/providers/cncf/kubernetes/__init__.py | 1 +
.../{ => providers/cncf}/kubernetes/basic_pod.yaml | 0
.../cncf/kubernetes/executors}/__init__.py | 0
.../__init__.py | 0
.../basic_template.yaml | 0
.../executors/test_kubernetes_executor.py | 161 +-
.../executors/test_local_kubernetes_executor.py | 7 +-
tests/{ => providers/cncf}/kubernetes/kube_config | 0
.../providers/cncf/kubernetes/models}/__init__.py | 0
.../cncf}/kubernetes/models/test_secret.py | 10 +-
.../cncf/kubernetes/operators/test_pod.py | 19 +-
tests/{ => providers/cncf}/kubernetes/pod.yaml | 0
.../cncf}/kubernetes/pod_generator_base.yaml | 0
.../pod_generator_base_with_secrets.yaml | 0
.../{ => providers/cncf}/kubernetes/test_client.py | 18 +-
.../kubernetes/test_kubernetes_helper_functions.py | 2 +-
.../cncf}/kubernetes/test_pod_generator.py | 33 +-
.../cncf/kubernetes/test_template_rendering.py | 156 ++
.../common/sql/operators/test_sql_execute.py | 13 +-
.../providers/daskexecutor}/__init__.py | 0
.../daskexecutor}/test_dask_executor.py | 8 +-
.../databricks/operators/test_databricks.py | 6 +-
tests/providers/ftp/operators/test_ftp.py | 65 +-
.../providers/google/cloud/hooks/test_bigquery.py | 33 +-
.../providers/google/cloud/hooks/test_dataproc.py | 4 +
tests/providers/google/cloud/hooks/test_gcs.py | 6 +-
.../google/cloud/hooks/test_kubernetes_engine.py | 46 +
tests/providers/google/cloud/hooks/test_spanner.py | 35 +-
.../google/cloud/operators/test_bigquery.py | 13 +-
.../google/cloud/operators/test_dataproc.py | 10 +
tests/providers/google/cloud/operators/test_gcs.py | 5 +-
.../google/cloud/transfers/test_gcs_to_gcs.py | 73 +
.../cloud/transfers/test_postgres_to_gcs_system.py | 2 +-
.../google/cloud/triggers/test_bigquery.py | 20 +-
tests/providers/hashicorp/hooks/test_vault.py | 89 +-
tests/providers/http/hooks/test_http.py | 209 ++-
tests/providers/microsoft/azure/hooks/test_wasb.py | 326 ++--
tests/providers/mysql/operators/test_mysql.py | 78 +
.../providers/openlineage/utils/test_sqlparser.py | 8 +
tests/providers/postgres/hooks/test_postgres.py | 49 +
.../providers/postgres/operators/test_postgres.py | 79 +-
.../auth => tests/providers/redis/log}/__init__.py | 0
.../providers/redis/log/test_redis_task_handler.py | 94 ++
tests/providers/sftp/operators/test_sftp.py | 110 +-
tests/providers/snowflake/hooks/test_snowflake.py | 30 +
.../snowflake/operators/test_snowflake.py | 73 +
.../snowflake/operators/test_snowflake_sql.py | 68 +
tests/sensors/test_base.py | 6 +-
tests/sensors/test_external_task_sensor.py | 82 +-
tests/serialization/test_dag_serialization.py | 78 +-
tests/system/conftest.py | 2 +-
...example_gcs_to_s3.py => example_eventbridge.py} | 65 +-
.../providers/amazon/aws/example_gcs_to_s3.py | 52 +-
.../providers/amazon/aws/example_rds_export.py | 1 +
.../cncf/kubernetes/example_kubernetes.py | 2 +-
.../cncf/kubernetes/example_kubernetes_async.py | 2 +-
.../system/providers/core}/__init__.py | 0
.../core/example_external_task_child_deferrable.py | 34 +-
.../example_external_task_parent_deferrable.py} | 71 +-
.../cloud/dataflow/example_dataflow_template.py | 23 +-
.../dataflow/resources/{text.csv => input.csv} | 0
.../google/cloud/dataform/example_dataform.py | 19 +-
.../google/cloud/datafusion/example_datafusion.py | 60 +-
.../cloud/datafusion/example_datafusion_async.py | 68 +-
.../cloud/dataproc/example_dataproc_batch.py | 24 +-
.../dataproc/example_dataproc_batch_deferrable.py | 13 +-
.../dataproc/example_dataproc_batch_persistent.py | 18 +-
.../example_dataproc_cluster_deferrable.py | 24 +-
.../dataproc/example_dataproc_cluster_generator.py | 17 +-
.../dataproc/example_dataproc_cluster_update.py | 12 +-
.../google/cloud/dataproc/example_dataproc_gke.py | 17 +-
.../cloud/dataproc/example_dataproc_hadoop.py | 37 +-
.../google/cloud/dataproc/example_dataproc_hive.py | 27 +-
.../google/cloud/dataproc/example_dataproc_pig.py | 19 +-
...dataproc_hive.py => example_dataproc_presto.py} | 48 +-
.../cloud/dataproc/example_dataproc_pyspark.py | 36 +-
.../cloud/dataproc/example_dataproc_spark.py | 17 +-
.../cloud/dataproc/example_dataproc_spark_async.py | 17 +-
.../dataproc/example_dataproc_spark_deferrable.py | 17 +-
.../cloud/dataproc/example_dataproc_spark_sql.py | 12 +-
.../cloud/dataproc/example_dataproc_sparkr.py | 42 +-
...dataproc_spark.py => example_dataproc_trino.py} | 44 +-
.../cloud/dataproc/example_dataproc_workflow.py | 13 +-
.../example_dataproc_workflow_deferrable.py | 15 +-
...ple_dataproc_metastore_hive_partition_sensor.py | 200 ++-
.../google/cloud/ml_engine/example_mlengine.py | 18 +-
.../cloud/ml_engine/example_mlengine_async.py | 20 +-
...le_cloud_storage_transfer_service_gcp_to_gcs.py | 127 ++
.../cloud/transfers/example_postgres_to_gcs.py | 227 +++
.../microsoft/azure/example_local_to_wasb.py | 4 +-
tests/test_utils/db.py | 2 +-
tests/ti_deps/deps/test_prev_dagrun_dep.py | 123 +-
tests/triggers/test_external_task.py | 6 +
tests/utils/test_helpers.py | 4 +-
tests/utils/test_log_handlers.py | 6 +-
tests/utils/test_serve_logs.py | 40 +-
tests/www/test_security.py | 4 +-
tests/www/test_utils.py | 47 +-
tests/www/views/test_session.py | 8 +-
tests/www/views/test_views.py | 8 +-
tests/www/views/test_views_extra_links.py | 30 +-
...st_views_graph_gantt.py => test_views_graph.py} | 1 -
tests/www/views/test_views_grid.py | 10 +
tests/www/views/test_views_tasks.py | 30 +
913 files changed, 20525 insertions(+), 10149 deletions(-)
copy airflow/{api => auth/managers/fab}/auth/__init__.py (100%)
create mode 100644 airflow/auth/managers/fab/auth/anonymous_user.py
rename airflow/{www/fab_security/sqla => auth/managers/fab}/models.py (98%)
create mode 100644 airflow/auth/managers/fab/security_manager_override.py
copy airflow/{api_connexion => auth/managers/fab/views}/__init__.py (100%)
create mode 100644 airflow/auth/managers/fab/views/user_details.py
copy airflow/{api/auth => auth/managers/model}/__init__.py (100%)
copy airflow/{config_templates/__init__.py => auth/managers/model/base_user.py} (67%)
delete mode 100644 airflow/config_templates/default_test.cfg
create mode 100644 airflow/config_templates/pre_2_7_defaults.cfg
create mode 100644 airflow/config_templates/unit_tests.cfg
rename airflow/kubernetes/{pod_runtime_info_env.py => pre_7_4_0_compatibility/__init__.py} (57%)
copy airflow/kubernetes/{ => pre_7_4_0_compatibility}/k8s_model.py (100%)
copy airflow/kubernetes/{ => pre_7_4_0_compatibility}/kube_client.py (100%)
copy airflow/kubernetes/{ => pre_7_4_0_compatibility}/pod_generator.py (95%)
copy airflow/kubernetes/{ => pre_7_4_0_compatibility}/pod_generator_deprecated.py (100%)
copy airflow/kubernetes/{ => pre_7_4_0_compatibility}/secret.py (98%)
delete mode 100644 airflow/kubernetes/volume.py
delete mode 100644 airflow/kubernetes/volume_mount.py
rename docs/apache-airflow-providers/howto/create-update-providers.rst => airflow/providers/CREATING_COMMUNITY_PROVIDERS.rst (63%)
create mode 100644 airflow/providers/amazon/aws/hooks/eventbridge.py
create mode 100644 airflow/providers/amazon/aws/operators/eventbridge.py
create mode 100644 airflow/providers/amazon/aws/triggers/step_function.py
create mode 100644 airflow/providers/amazon/aws/waiters/stepfunctions.json
copy airflow/{api/auth => providers/cncf/kubernetes/executors}/__init__.py (100%)
rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor.py (85%)
rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_types.py (100%)
rename airflow/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_utils.py (93%)
rename airflow/{ => providers/cncf/kubernetes}/executors/local_kubernetes_executor.py (97%)
rename airflow/{ => providers/cncf}/kubernetes/k8s_model.py (100%)
rename airflow/{ => providers/cncf}/kubernetes/kube_client.py (96%)
rename airflow/{ => providers/cncf}/kubernetes/kube_config.py (99%)
copy airflow/{api_connexion => providers/cncf/kubernetes/kubernetes_executor_templates}/__init__.py (100%)
rename airflow/{ => providers/cncf/kubernetes}/kubernetes_executor_templates/basic_template.yaml (100%)
rename airflow/{ => providers/cncf}/kubernetes/kubernetes_helper_functions.py (94%)
rename airflow/{ => providers/cncf}/kubernetes/pod_generator.py (97%)
rename airflow/{ => providers/cncf}/kubernetes/pod_generator_deprecated.py (97%)
rename airflow/{ => providers/cncf}/kubernetes/pod_launcher_deprecated.py (98%)
copy airflow/{api_connexion => providers/cncf/kubernetes/pod_template_file_examples}/__init__.py (100%)
rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/dags_in_image_template.yaml (100%)
rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml (100%)
rename airflow/{ => providers/cncf}/kubernetes/pod_template_file_examples/git_sync_template.yaml (100%)
rename airflow/{ => providers/cncf}/kubernetes/secret.py (98%)
create mode 100644 airflow/providers/cncf/kubernetes/template_rendering.py
copy airflow/{utils/hashlib_wrapper.py => providers/cncf/kubernetes/utils/k8s_hashlib_wrapper.py} (76%)
copy airflow/providers/{apprise => daskexecutor}/CHANGELOG.rst (96%)
copy airflow/providers/{apache/pinot => daskexecutor}/__init__.py (86%)
copy airflow/{api_connexion => providers/daskexecutor/executors}/__init__.py (100%)
rename airflow/{ => providers/daskexecutor}/executors/dask_executor.py (100%)
create mode 100644 airflow/providers/daskexecutor/provider.yaml
delete mode 100644 airflow/providers/google/cloud/example_dags/example_postgres_to_gcs.py
copy airflow/{api/auth => providers/redis/log}/__init__.py (100%)
create mode 100644 airflow/providers/redis/log/redis_task_handler.py
copy airflow/{providers/amazon/aws/utils/rds.py => utils/empty_set.py} (68%)
create mode 100644 airflow/utils/providers_configuration_loader.py
copy airflow/{utils/hashlib_wrapper.py => www/extensions/init_auth_manager.py} (57%)
delete mode 100644 airflow/www/static/css/gantt.css
create mode 100644 airflow/www/static/js/dag/details/gantt/GanttTooltip.tsx
create mode 100644 airflow/www/static/js/dag/details/gantt/Row.tsx
create mode 100644 airflow/www/static/js/dag/details/gantt/index.tsx
delete mode 100644 airflow/www/static/js/gantt.js
delete mode 100644 airflow/www/templates/airflow/gantt.html
delete mode 100644 dev/README_RELEASE_AIRFLOW_UPGRADE_CHECK.md
create mode 100644 docs/apache-airflow-providers-amazon/operators/eventbridge.rst
copy docs/{apache-airflow-providers-apache-beam/installing-providers-from-sources.rst => apache-airflow-providers-apache-hive/configurations-ref.rst} (92%)
copy docs/{apache-airflow-providers-apache-beam/installing-providers-from-sources.rst => apache-airflow-providers-celery/configurations-ref.rst} (92%)
copy docs/{apache-airflow-providers-apache-beam/installing-providers-from-sources.rst => apache-airflow-providers-cncf-kubernetes/configurations-ref.rst} (92%)
copy docs/{apache-airflow-providers-celery => apache-airflow-providers-daskexecutor}/changelog.rst (92%)
copy docs/{apache-airflow-providers-apprise => apache-airflow-providers-daskexecutor}/commits.rst (57%)
copy docs/{apache-airflow-providers-apache-beam/installing-providers-from-sources.rst => apache-airflow-providers-daskexecutor/configurations-ref.rst} (92%)
copy docs/{apache-airflow-providers-celery => apache-airflow-providers-daskexecutor}/index.rst (69%)
copy docs/{apache-airflow-providers-apache-beam => apache-airflow-providers-daskexecutor}/installing-providers-from-sources.rst (92%)
copy docs/{apache-airflow-providers-airbyte => apache-airflow-providers-daskexecutor}/security.rst (100%)
delete mode 100644 docs/apache-airflow-providers-google/configurations-ref.rst
create mode 100644 docs/apache-airflow-providers-google/operators/transfer/postgres_to_gcs.rst
copy docs/{apache-airflow-providers-apache-beam/changelog.rst => apache-airflow-providers-redis/logging/index.rst} (59%)
copy docs/{apache-airflow-providers-openlineage/commits.rst => apache-airflow-providers/core-extensions/configurations.rst} (68%)
copy docs/{apache-airflow-providers-apache-drill/changelog.rst => apache-airflow-providers/core-extensions/deferrable-operator-ref.rst} (80%)
create mode 100644 docs/apache-airflow-providers/howto/create-custom-providers.rst
rename tests/auh/__init__.py => docs/apache-airflow-providers/redirects.txt (88%)
create mode 100644 docs/apache-airflow/howto/setup-and-teardown.rst
delete mode 100644 docs/apache-airflow/howto/use-test-config.rst
create mode 100644 docs/apache-airflow/security/releasing_security_patches.rst
copy docs/apache-airflow/security/{index.rst => security_model.rst} (70%)
rename docs/{ => exts/includes}/installing-providers-from-sources.rst (100%)
copy docs/{apache-airflow-providers-openlineage/commits.rst => exts/includes/providers-configurations-ref.rst} (67%)
copy docs/{apache-airflow/configurations-ref.rst => exts/includes/sections-and-options.rst} (64%)
rename docs/exts/{ => templates}/auth_backend.rst.jinja2 (100%)
copy dev/example_dags/README.md => docs/exts/templates/configuration.rst.jinja2 (82%)
rename docs/exts/{ => templates}/connections.rst.jinja2 (100%)
copy docs/exts/{executors.rst.jinja2 => templates/deferrable_operators_list.rst.jinja2} (75%)
rename docs/exts/{ => templates}/executors.rst.jinja2 (100%)
rename docs/exts/{ => templates}/extra_links.rst.jinja2 (100%)
rename docs/exts/{ => templates}/logging.rst.jinja2 (100%)
rename docs/exts/{ => templates}/notifications.rst.jinja2 (100%)
rename docs/exts/{ => templates}/operators_and_hooks_ref-transfers.rst.jinja2 (100%)
rename docs/exts/{ => templates}/operators_and_hooks_ref.rst.jinja2 (100%)
rename docs/exts/{ => templates}/secret_backend.rst.jinja2 (100%)
create mode 100644 docs/integration-logos/aws/Amazon-EventBridge_64.png
create mode 100644 docs/integration-logos/dask/dask.png
rename airflow/kubernetes/pod_launcher.py => kubernetes_tests/conftest.py (79%)
delete mode 100644 licenses/LICENSE-moment-strftime.txt
create mode 100644 newsfragments/32572.significant.rst
create mode 100644 newsfragments/32767.significant.rst
create mode 100755 scripts/ci/pre_commit/pre_commit_check_airflow_k8s_not_used.py
create mode 100755 scripts/ci/pre_commit/pre_commit_check_cncf_k8s_used_for_k8s_executor_only.py
delete mode 100755 scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
delete mode 100644 scripts/in_container/airflow_ci.cfg
delete mode 100644 tests/auh/managers/__init__.py
delete mode 100644 tests/auh/managers/fab/__init__.py
delete mode 100644 tests/auh/managers/fab/test_fab_auth_manager.py
copy {airflow/api_connexion => tests/auth}/__init__.py (100%)
copy {airflow/api_connexion => tests/auth/managers}/__init__.py (100%)
copy {airflow/api_connexion => tests/auth/managers/fab}/__init__.py (100%)
create mode 100644 tests/auth/managers/fab/test_fab_auth_manager.py
create mode 100644 tests/auth/managers/fab/test_security_manager_override.py
create mode 100644 tests/auth/managers/test_base_auth_manager.py
delete mode 100644 tests/core/test_config_templates.py
delete mode 100644 tests/executors/kubernetes_executor_template_files/__init__.py
delete mode 100644 tests/kubernetes/__init__.py
delete mode 100644 tests/kubernetes/models/__init__.py
copy airflow/api/auth/backend/session.py => tests/providers/amazon/aws/hooks/test_eventbridge.py (51%)
create mode 100644 tests/providers/amazon/aws/operators/test_eventbridge.py
create mode 100644 tests/providers/amazon/aws/triggers/test_emr_serverless.py
create mode 100644 tests/providers/amazon/aws/triggers/test_step_function.py
rename tests/{ => providers/cncf}/kubernetes/basic_pod.yaml (100%)
copy {airflow/api_connexion => tests/providers/cncf/kubernetes/executors}/__init__.py (100%)
copy {airflow/api_connexion => tests/providers/cncf/kubernetes/executors/kubernetes_executor_template_files}/__init__.py (100%)
rename tests/{ => providers/cncf/kubernetes}/executors/kubernetes_executor_template_files/basic_template.yaml (100%)
rename tests/{ => providers/cncf/kubernetes}/executors/test_kubernetes_executor.py (89%)
rename tests/{ => providers/cncf/kubernetes}/executors/test_local_kubernetes_executor.py (95%)
rename tests/{ => providers/cncf}/kubernetes/kube_config (100%)
copy {airflow/api/auth => tests/providers/cncf/kubernetes/models}/__init__.py (100%)
rename tests/{ => providers/cncf}/kubernetes/models/test_secret.py (93%)
rename tests/{ => providers/cncf}/kubernetes/pod.yaml (100%)
rename tests/{ => providers/cncf}/kubernetes/pod_generator_base.yaml (100%)
rename tests/{ => providers/cncf}/kubernetes/pod_generator_base_with_secrets.yaml (100%)
rename tests/{ => providers/cncf}/kubernetes/test_client.py (87%)
rename tests/{ => providers/cncf}/kubernetes/test_kubernetes_helper_functions.py (98%)
rename tests/{ => providers/cncf}/kubernetes/test_pod_generator.py (95%)
create mode 100644 tests/providers/cncf/kubernetes/test_template_rendering.py
copy {airflow/api_connexion => tests/providers/daskexecutor}/__init__.py (100%)
rename tests/{executors => providers/daskexecutor}/test_dask_executor.py (94%)
copy {airflow/api/auth => tests/providers/redis/log}/__init__.py (100%)
create mode 100644 tests/providers/redis/log/test_redis_task_handler.py
copy tests/system/providers/amazon/aws/{example_gcs_to_s3.py => example_eventbridge.py} (50%)
copy {airflow/api_connexion => tests/system/providers/core}/__init__.py (100%)
rename airflow/kubernetes/pod.py => tests/system/providers/core/example_external_task_child_deferrable.py (59%)
copy tests/system/providers/{google/cloud/dataproc_metastore/example_dataproc_metastore_hive_partition_sensor.py => core/example_external_task_parent_deferrable.py} (50%)
rename tests/system/providers/google/cloud/dataflow/resources/{text.csv => input.csv} (100%)
copy tests/system/providers/google/cloud/dataproc/{example_dataproc_hive.py => example_dataproc_presto.py} (76%)
copy tests/system/providers/google/cloud/dataproc/{example_dataproc_spark.py => example_dataproc_trino.py} (76%)
create mode 100644 tests/system/providers/google/cloud/storage_transfer/example_cloud_storage_transfer_service_gcp_to_gcs.py
create mode 100644 tests/system/providers/google/cloud/transfers/example_postgres_to_gcs.py
rename tests/www/views/{test_views_graph_gantt.py => test_views_graph.py} (99%)
[airflow] 01/01: openlineage, sagemaker: add OpenLineage support for SageMaker's Processing, Transform and Training operators
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mobuchowski pushed a commit to branch openlineage-sagemaker-operators
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 21da40d741a72b3e71a6ef9ff488f26a414c5057
Author: Maciej Obuchowski <ob...@gmail.com>
AuthorDate: Tue Aug 1 16:59:19 2023 +0200
openlineage, sagemaker: add OpenLineage support for SageMaker's Processing, Transform and Training operators
Signed-off-by: Maciej Obuchowski <ob...@gmail.com>
---
.../providers/amazon/aws/operators/sagemaker.py | 156 +++++++++++++++++++--
dev/breeze/tests/test_selective_checks.py | 9 +-
generated/provider_dependencies.json | 1 +
.../aws/operators/test_sagemaker_processing.py | 50 ++++++-
.../aws/operators/test_sagemaker_training.py | 32 ++++-
.../aws/operators/test_sagemaker_transform.py | 30 ++++
6 files changed, 251 insertions(+), 27 deletions(-)
diff --git a/airflow/providers/amazon/aws/operators/sagemaker.py b/airflow/providers/amazon/aws/operators/sagemaker.py
index 83a1e4f3d2..7e1a0a3567 100644
--- a/airflow/providers/amazon/aws/operators/sagemaker.py
+++ b/airflow/providers/amazon/aws/operators/sagemaker.py
@@ -40,13 +40,16 @@ from airflow.providers.amazon.aws.utils.tags import format_tags
from airflow.utils.json import AirflowJsonEncoder
if TYPE_CHECKING:
+ from openlineage.client.run import Dataset
+
+ from airflow.providers.openlineage.extractors.base import OperatorLineage
from airflow.utils.context import Context
DEFAULT_CONN_ID: str = "aws_default"
CHECK_INTERVAL_SECOND: int = 30
-def serialize(result: dict) -> str:
+def serialize(result: dict) -> dict:
return json.loads(json.dumps(result, cls=AirflowJsonEncoder))
@@ -158,6 +161,14 @@ class SageMakerBaseOperator(BaseOperator):
"""Return SageMakerHook."""
return SageMakerHook(aws_conn_id=self.aws_conn_id)
+ @staticmethod
+ def path_to_s3_dataset(path) -> Dataset:
+ from openlineage.client.run import Dataset
+
+ path = path.replace("s3://", "")
+ split_path = path.split("/")
+ return Dataset(namespace=f"s3://{split_path[0]}", name="/".join(split_path[1:]), facets={})
+
class SageMakerProcessingOperator(SageMakerBaseOperator):
"""
@@ -225,6 +236,7 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
self.max_attempts = max_attempts or 60
self.max_ingestion_time = max_ingestion_time
self.deferrable = deferrable
+ self.serialized_job: dict
def _create_integer_fields(self) -> None:
"""Set fields which should be cast to integers."""
@@ -282,14 +294,48 @@ class SageMakerProcessingOperator(SageMakerBaseOperator):
method_name="execute_complete",
)
- return {"Processing": serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))}
+ self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
+ return {"Processing": self.serialized_job}
def execute_complete(self, context, event=None):
if event["status"] != "success":
raise AirflowException(f"Error while running job: {event}")
else:
self.log.info(event["message"])
- return {"Processing": serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))}
+ self.serialized_job = serialize(self.hook.describe_processing_job(self.config["ProcessingJobName"]))
+ return {"Processing": self.serialized_job}
+
+ def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+ """Returns OpenLineage data gathered from SageMaker's API response saved by processing job."""
+ from airflow.providers.openlineage.extractors.base import OperatorLineage
+
+ inputs = []
+ outputs = []
+ try:
+ inputs, outputs = self._extract_s3_dataset_identifiers(
+ processing_inputs=self.serialized_job["ProcessingInputs"],
+ processing_outputs=self.serialized_job["ProcessingOutputConfig"]["Outputs"],
+ )
+ except KeyError:
+ self.log.exception("Could not find input/output information in Xcom.")
+
+ return OperatorLineage(inputs=inputs, outputs=outputs)
+
+ def _extract_s3_dataset_identifiers(self, processing_inputs, processing_outputs):
+ inputs = []
+ outputs = []
+ try:
+ for processing_input in processing_inputs:
+ inputs.append(self.path_to_s3_dataset(processing_input["S3Input"]["S3Uri"]))
+ except KeyError:
+ self.log.exception("Cannot find S3 input details", exc_info=True)
+
+ try:
+ for processing_output in processing_outputs:
+ outputs.append(self.path_to_s3_dataset(processing_output["S3Output"]["S3Uri"]))
+ except KeyError:
+ self.log.exception("Cannot find S3 output details.", exc_info=True)
+ return inputs, outputs
class SageMakerEndpointConfigOperator(SageMakerBaseOperator):
@@ -579,6 +625,8 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
Provided value: '{action_if_job_exists}'."
)
self.deferrable = deferrable
+ self.serialized_model: dict
+ self.serialized_tranform: dict
def _create_integer_fields(self) -> None:
"""Set fields which should be cast to integers."""
@@ -650,10 +698,11 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
method_name="execute_complete",
)
- return {
- "Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
- "Transform": serialize(self.hook.describe_transform_job(transform_config["TransformJobName"])),
- }
+ self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
+ self.serialized_tranform = serialize(
+ self.hook.describe_transform_job(transform_config["TransformJobName"])
+ )
+ return {"Model": self.serialized_model, "Transform": self.serialized_tranform}
def execute_complete(self, context, event=None):
if event["status"] != "success":
@@ -661,10 +710,62 @@ class SageMakerTransformOperator(SageMakerBaseOperator):
else:
self.log.info(event["message"])
transform_config = self.config.get("Transform", self.config)
- return {
- "Model": serialize(self.hook.describe_model(transform_config["ModelName"])),
- "Transform": serialize(self.hook.describe_transform_job(transform_config["TransformJobName"])),
- }
+ self.serialized_model = serialize(self.hook.describe_model(transform_config["ModelName"]))
+ self.serialized_tranform = serialize(
+ self.hook.describe_transform_job(transform_config["TransformJobName"])
+ )
+ return {"Model": self.serialized_model, "Transform": self.serialized_tranform}
+
+ def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+ """Returns OpenLineage data gathered from SageMaker's API response saved by transform job."""
+ from airflow.providers.openlineage.extractors import OperatorLineage
+
+ model_package_arn = None
+ transform_input = None
+ transform_output = None
+
+ try:
+ model_package_arn = self.serialized_model["PrimaryContainer"]["ModelPackageName"]
+ except KeyError:
+ self.log.error("Cannot find Model Package Name.", exc_info=True)
+
+ try:
+ transform_input = self.serialized_tranform["TransformInput"]["DataSource"]["S3DataSource"][
+ "S3Uri"
+ ]
+ transform_output = self.serialized_tranform["TransformOutput"]["S3OutputPath"]
+ except KeyError:
+ self.log.error("Cannot find some required input/output details.", exc_info=True)
+
+ inputs = []
+
+ if transform_input is not None:
+ inputs.append(self.path_to_s3_dataset(transform_input))
+
+ if model_package_arn is not None:
+ model_data_urls = self._get_model_data_urls(model_package_arn)
+ for model_data_url in model_data_urls:
+ inputs.append(self.path_to_s3_dataset(model_data_url))
+
+ outputs = []
+ if transform_output is not None:
+ outputs.append(self.path_to_s3_dataset(transform_output))
+
+ return OperatorLineage(inputs=inputs, outputs=outputs)
+
+ def _get_model_data_urls(self, model_package_arn) -> list:
+ model_data_urls = []
+ try:
+ model_containers = self.hook.get_conn().describe_model_package(
+ ModelPackageName=model_package_arn
+ )["InferenceSpecification"]["Containers"]
+
+ for container in model_containers:
+ model_data_urls.append(container["ModelDataUrl"])
+ except KeyError:
+ self.log.exception("Cannot retrieve model details.", exc_info=True)
+
+ return model_data_urls
class SageMakerTuningOperator(SageMakerBaseOperator):
@@ -891,6 +992,7 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
Provided value: '{action_if_job_exists}'."
)
self.deferrable = deferrable
+ self.serialized_training_data: dict
def expand_role(self) -> None:
"""Expands an IAM role name into an ARN."""
@@ -951,16 +1053,40 @@ class SageMakerTrainingOperator(SageMakerBaseOperator):
method_name="execute_complete",
)
- result = {"Training": serialize(self.hook.describe_training_job(self.config["TrainingJobName"]))}
- return result
+ self.serialized_training_data = serialize(
+ self.hook.describe_training_job(self.config["TrainingJobName"])
+ )
+ return {"Training": self.serialized_training_data}
def execute_complete(self, context, event=None):
if event["status"] != "success":
raise AirflowException(f"Error while running job: {event}")
else:
self.log.info(event["message"])
- result = {"Training": serialize(self.hook.describe_training_job(self.config["TrainingJobName"]))}
- return result
+ self.serialized_training_data = serialize(
+ self.hook.describe_training_job(self.config["TrainingJobName"])
+ )
+ return {"Training": self.serialized_training_data}
+
+ def get_openlineage_facets_on_complete(self, task_instance) -> OperatorLineage:
+ """Returns OpenLineage data gathered from SageMaker's API response saved by training job."""
+ from airflow.providers.openlineage.extractors import OperatorLineage
+
+ inputs = []
+ outputs = []
+ try:
+ for input_data in self.serialized_training_data["InputDataConfig"]:
+ inputs.append(self.path_to_s3_dataset(input_data["DataSource"]["S3DataSource"]["S3Uri"]))
+ except KeyError:
+ self.log.exception("Issues extracting inputs.")
+
+ try:
+ outputs.append(
+ self.path_to_s3_dataset(self.serialized_training_data["ModelArtifacts"]["S3ModelArtifacts"])
+ )
+ except KeyError:
+ self.log.exception("Issues extracting inputs.")
+ return OperatorLineage(inputs=inputs, outputs=outputs)
class SageMakerDeleteModelOperator(SageMakerBaseOperator):
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index e3f9bacb98..3cdf3dc7bb 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -312,7 +312,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
{
"affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.sql exasol ftp google http imap microsoft.azure "
- "mongo mysql postgres salesforce ssh",
+ "mongo mysql openlineage postgres salesforce ssh",
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
"python-versions": "['3.8']",
@@ -326,7 +326,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"run-amazon-tests": "true",
"parallel-test-types-list-as-string": "Providers[amazon] Always "
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,http,imap,microsoft.azure,"
- "mongo,mysql,postgres,salesforce,ssh] Providers[google]",
+ "mongo,mysql,openlineage,postgres,salesforce,ssh] Providers[google]",
},
id="Providers tests run including amazon tests if amazon provider files changed",
),
@@ -354,7 +354,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
{
"affected-providers-list-as-string": "amazon apache.hive cncf.kubernetes "
"common.sql exasol ftp google http imap microsoft.azure "
- "mongo mysql postgres salesforce ssh",
+ "mongo mysql openlineage postgres salesforce ssh",
"all-python-versions": "['3.8']",
"all-python-versions-list-as-string": "3.8",
"python-versions": "['3.8']",
@@ -368,7 +368,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
"upgrade-to-newer-dependencies": "false",
"parallel-test-types-list-as-string": "Providers[amazon] Always "
"Providers[apache.hive,cncf.kubernetes,common.sql,exasol,ftp,"
- "http,imap,microsoft.azure,mongo,mysql,postgres,salesforce,ssh] Providers[google]",
+ "http,imap,microsoft.azure,mongo,mysql,openlineage,postgres,salesforce,ssh] "
+ "Providers[google]",
},
id="Providers tests run including amazon tests if amazon provider files changed",
),
diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json
index 42146c2767..02b5d59a13 100644
--- a/generated/provider_dependencies.json
+++ b/generated/provider_dependencies.json
@@ -47,6 +47,7 @@
"imap",
"microsoft.azure",
"mongo",
+ "openlineage",
"salesforce",
"ssh"
],
diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py b/tests/providers/amazon/aws/operators/test_sagemaker_processing.py
index 1d73d44bdf..817761c014 100644
--- a/tests/providers/amazon/aws/operators/test_sagemaker_processing.py
+++ b/tests/providers/amazon/aws/operators/test_sagemaker_processing.py
@@ -20,12 +20,17 @@ from unittest import mock
import pytest
from botocore.exceptions import ClientError
+from openlineage.client.run import Dataset
from airflow.exceptions import AirflowException, TaskDeferred
from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook
from airflow.providers.amazon.aws.operators import sagemaker
-from airflow.providers.amazon.aws.operators.sagemaker import SageMakerProcessingOperator
+from airflow.providers.amazon.aws.operators.sagemaker import (
+ SageMakerBaseOperator,
+ SageMakerProcessingOperator,
+)
from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger
+from airflow.providers.openlineage.extractors import OperatorLineage
CREATE_PROCESSING_PARAMS: dict = {
"AppSpecification": {
@@ -238,14 +243,16 @@ class TestSageMakerProcessingOperator:
action_if_job_exists="not_fail_or_increment",
)
- @mock.patch.object(SageMakerHook, "create_processing_job")
- @mock.patch("airflow.providers.amazon.aws.operators.sagemaker.SageMakerBaseOperator._check_if_job_exists")
- def test_operator_defer(self, mock_job_exists, mock_processing):
- mock_processing.return_value = {
+ @mock.patch.object(
+ SageMakerHook,
+ "create_processing_job",
+ return_value={
"ProcessingJobArn": "test_arn",
"ResponseMetadata": {"HTTPStatusCode": 200},
- }
- mock_job_exists.return_value = False
+ },
+ )
+ @mock.patch.object(SageMakerBaseOperator, "_check_if_job_exists", return_value=False)
+ def test_operator_defer(self, mock_job_exists, mock_processing):
sagemaker_operator = SageMakerProcessingOperator(
**self.processing_config_kwargs,
config=CREATE_PROCESSING_PARAMS,
@@ -255,3 +262,32 @@ class TestSageMakerProcessingOperator:
with pytest.raises(TaskDeferred) as exc:
sagemaker_operator.execute(context=None)
assert isinstance(exc.value.trigger, SageMakerTrigger), "Trigger is not a SagemakerTrigger"
+
+ @mock.patch.object(
+ SageMakerHook,
+ "describe_processing_job",
+ return_value={
+ "ProcessingInputs": [{"S3Input": {"S3Uri": "s3://input-bucket/input-path"}}],
+ "ProcessingOutputConfig": {
+ "Outputs": [{"S3Output": {"S3Uri": "s3://output-bucket/output-path"}}]
+ },
+ },
+ )
+ @mock.patch.object(SageMakerHook, "count_processing_jobs_by_name", return_value=0)
+ @mock.patch.object(
+ SageMakerHook,
+ "create_processing_job",
+ return_value={"ProcessingJobArn": "test_arn", "ResponseMetadata": {"HTTPStatusCode": 200}},
+ )
+ @mock.patch.object(SageMakerBaseOperator, "_check_if_job_exists", return_value=False)
+ def test_operator_openlineage_data(self, check_job_exists, mock_processing, _, mock_desc):
+ sagemaker = SageMakerProcessingOperator(
+ **self.processing_config_kwargs,
+ config=CREATE_PROCESSING_PARAMS,
+ deferrable=True,
+ )
+ sagemaker.execute(context=None)
+ assert sagemaker.get_openlineage_facets_on_complete(None) == OperatorLineage(
+ inputs=[Dataset(namespace="s3://input-bucket", name="input-path")],
+ outputs=[Dataset(namespace="s3://output-bucket", name="output-path")],
+ )
diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_training.py b/tests/providers/amazon/aws/operators/test_sagemaker_training.py
index e551317d33..9cb50de7c8 100644
--- a/tests/providers/amazon/aws/operators/test_sagemaker_training.py
+++ b/tests/providers/amazon/aws/operators/test_sagemaker_training.py
@@ -20,12 +20,14 @@ from unittest import mock
import pytest
from botocore.exceptions import ClientError
+from openlineage.client.run import Dataset
from airflow.exceptions import AirflowException, TaskDeferred
from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook
from airflow.providers.amazon.aws.operators import sagemaker
-from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTrainingOperator
+from airflow.providers.amazon.aws.operators.sagemaker import SageMakerBaseOperator, SageMakerTrainingOperator
from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger
+from airflow.providers.openlineage.extractors import OperatorLineage
EXPECTED_INTEGER_FIELDS: list[list[str]] = [
["ResourceConfig", "InstanceCount"],
@@ -127,3 +129,31 @@ class TestSageMakerTrainingOperator:
with pytest.raises(TaskDeferred) as exc:
self.sagemaker.execute(context=None)
assert isinstance(exc.value.trigger, SageMakerTrigger), "Trigger is not a SagemakerTrigger"
+
+ @mock.patch.object(
+ SageMakerHook,
+ "describe_training_job",
+ return_value={
+ "InputDataConfig": [
+ {
+ "DataSource": {"S3DataSource": {"S3Uri": "s3://input-bucket/input-path"}},
+ }
+ ],
+ "ModelArtifacts": {"S3ModelArtifacts": "s3://model-bucket/model-path"},
+ },
+ )
+ @mock.patch.object(
+ SageMakerHook,
+ "create_training_job",
+ return_value={
+ "TrainingJobArn": "test_arn",
+ "ResponseMetadata": {"HTTPStatusCode": 200},
+ },
+ )
+ @mock.patch.object(SageMakerBaseOperator, "_check_if_job_exists", return_value=False)
+ def test_execute_openlineage_data(self, mock_exists, mock_training, mock_desc):
+ self.sagemaker.execute(None)
+ assert self.sagemaker.get_openlineage_facets_on_complete(None) == OperatorLineage(
+ inputs=[Dataset(namespace="s3://input-bucket", name="input-path")],
+ outputs=[Dataset(namespace="s3://model-bucket", name="model-path")],
+ )
diff --git a/tests/providers/amazon/aws/operators/test_sagemaker_transform.py b/tests/providers/amazon/aws/operators/test_sagemaker_transform.py
index 76a4d877b6..9a9af38b36 100644
--- a/tests/providers/amazon/aws/operators/test_sagemaker_transform.py
+++ b/tests/providers/amazon/aws/operators/test_sagemaker_transform.py
@@ -22,12 +22,14 @@ from unittest import mock
import pytest
from botocore.exceptions import ClientError
+from openlineage.client.run import Dataset
from airflow.exceptions import AirflowException, TaskDeferred
from airflow.providers.amazon.aws.hooks.sagemaker import SageMakerHook
from airflow.providers.amazon.aws.operators import sagemaker
from airflow.providers.amazon.aws.operators.sagemaker import SageMakerTransformOperator
from airflow.providers.amazon.aws.triggers.sagemaker import SageMakerTrigger
+from airflow.providers.openlineage.extractors import OperatorLineage
EXPECTED_INTEGER_FIELDS: list[list[str]] = [
["Transform", "TransformResources", "InstanceCount"],
@@ -178,3 +180,31 @@ class TestSageMakerTransformOperator:
with pytest.raises(TaskDeferred) as exc:
self.sagemaker.execute(context=None)
assert isinstance(exc.value.trigger, SageMakerTrigger), "Trigger is not a SagemakerTrigger"
+
+ @mock.patch.object(SageMakerHook, "describe_transform_job")
+ @mock.patch.object(SageMakerHook, "create_model")
+ @mock.patch.object(SageMakerHook, "describe_model")
+ @mock.patch.object(SageMakerHook, "get_conn")
+ @mock.patch.object(SageMakerHook, "create_transform_job")
+ def test_operator_lineage_data(self, mock_transform, mock_conn, mock_model, _, mock_desc):
+ self.sagemaker.check_if_job_exists = False
+ mock_conn.return_value.describe_model_package.return_value = {
+ "InferenceSpecification": {"Containers": [{"ModelDataUrl": "s3://model-bucket/model-path"}]},
+ }
+ mock_model.return_value = {"PrimaryContainer": {"ModelPackageName": "package-name"}}
+ mock_desc.return_value = {
+ "TransformInput": {"DataSource": {"S3DataSource": {"S3Uri": "s3://input-bucket/input-path"}}},
+ "TransformOutput": {"S3OutputPath": "s3://output-bucket/output-path"},
+ }
+ mock_transform.return_value = {
+ "TransformJobArn": "test_arn",
+ "ResponseMetadata": {"HTTPStatusCode": 200},
+ }
+ self.sagemaker.execute(None)
+ assert self.sagemaker.get_openlineage_facets_on_complete(None) == OperatorLineage(
+ inputs=[
+ Dataset(namespace="s3://input-bucket", name="input-path"),
+ Dataset(namespace="s3://model-bucket", name="model-path"),
+ ],
+ outputs=[Dataset(namespace="s3://output-bucket", name="output-path")],
+ )