You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/10/08 11:14:20 UTC
[airflow] branch v2-0-test updated (ff1ab97 -> 9dc32a3)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a change to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from ff1ab97 Add operator link to access DAG triggered by TriggerDagRunOperator (#11254)
add e89d384 The bats script for CI image is now placed in the docker folder (#11262)
add 1b9e59c Limits CodeQL workflow to run only in the Apache Airflow repo (#11264)
add 6dce7a6 Enable MySQL 8 CI jobs (#11247)
add a4478f5 Improve running and canceliling of the PR-triggered builds. (#11268)
add 1180f1b Fix link to static checks in CONTRIBUTING.rst (#11271)
add fd682fd fix job deletion (#11272)
add b93b6c5 Allow labels in KubernetesPodOperator to be templated (#10796)
add c9efa56 Access task type via the property, not dundervars (#11274)
add baa980f When sending tasks to celery from a sub-process, reset signal handlers (#11278)
add a33a919 Switched to Run Checks for Building Images. (#11276)
add 93475e9 Single/Multi-Namespace mode for helm chart (#11034)
add c51016b Add LocalToAzureDataLakeStorageOperator (#10814)
add 4847460 Add CeleryKubernetesExecutor to helm chart (#11288)
add 32b3cfb Strict type check for all hooks in amazon (#11250)
add 975558b Replaces depreated set-env with env file (#11292)
add 03e0ff2 Breeze start-airflow command wasn't able to initialize the db in 1.10.x (#11207)
add 03ff067 Add type annotations to ZendeskHook, update unit test (#10888)
add dd98b21 Add acl_policy parameter to GCSToS3Operator (#10804) (#10829)
add 9514cfe add releasing airflow docs to dev readme (#11245)
add c803a27 Prevent race condition in trying to collect result from DagFileProcessor (#11306)
add 4af7804 Bump tenacity to 6.2 (#11313)
add bbc3cea Move latest_only_operator.py to latest_only.py (#11178) (#11304)
add 22c6a84 Adds --no-rbac-ui flag for Breeze airflow 1.10 installation (#11315)
add 18dcac8 Add remaining community guidelines to CONTRIBUTING.rst (#11312)
add 47b05a8 Improve handling of job_id in BigQuery operators (#11287)
add e2655f6 Prints nicer message in case of git push errors (#11320)
add b0fcf67 Add AzureFileShareToGCSOperator (#10991)
add d86cf37 Automatically upgrade old default navbar color (#11322)
add fe59f26 Pin versions of "untrusted" 3rd-party GitHub Actions (#11319)
add d404cb0 Moves Commiter's guide to CONTRIBUTING.rst (#11314)
add b4baa2b Add environment variables documentation to cli-ref.rst. (#10970)
add 2bac481 Update link for Announcement Page (#11337)
add 5d007fd Strict type check for azure hooks (#11342)
add de07d13 Adds --install-wheels flag to breeze command line (#11317)
add 4d95d9c Improve code quality of SLA mechanism in SchedulerJob (#11257)
add 625afa2 Improve Committer's guide docs (#11338)
add 832a785 Add Azure Blob Storage to GCS transfer operator (#11321)
add 9dc32a3 Better message when Building Image fails or gets cancelled. (#11333)
No new revisions were added by this update.
Summary of changes:
.dockerignore | 1 -
.github/boring-cyborg.yml | 2 +-
.github/workflows/build-images-workflow-run.yml | 86 +++++-
.github/workflows/ci.yml | 10 +-
.github/workflows/codeql-analysis.yml | 1 +
.github/workflows/codeql-cancel.yml | 21 ++
.github/workflows/delete_old_artifacts.yml | 2 +-
.github/workflows/scheduled_quarantined.yml | 6 +-
BREEZE.rst | 20 +-
CONTRIBUTING.rst | 36 ++-
Dockerfile.ci | 2 +-
README.md | 2 +-
airflow/configuration.py | 3 +
airflow/example_dags/example_latest_only.py | 2 +-
.../example_latest_only_with_trigger.py | 2 +-
airflow/executors/celery_executor.py | 10 +-
airflow/jobs/scheduler_job.py | 60 +++-
airflow/models/baseoperator.py | 2 +-
airflow/models/dagrun.py | 2 +-
airflow/models/taskinstance.py | 4 +-
.../{latest_only_operator.py => latest_only.py} | 0
airflow/operators/latest_only_operator.py | 54 +---
airflow/providers/amazon/aws/hooks/aws_dynamodb.py | 2 +-
airflow/providers/amazon/aws/hooks/batch_client.py | 10 +-
.../providers/amazon/aws/hooks/batch_waiters.py | 4 +-
.../providers/amazon/aws/hooks/cloud_formation.py | 6 +-
airflow/providers/amazon/aws/hooks/datasync.py | 45 +--
airflow/providers/amazon/aws/hooks/glue_catalog.py | 17 +-
airflow/providers/amazon/aws/hooks/kinesis.py | 2 +-
airflow/providers/amazon/aws/hooks/sagemaker.py | 101 ++++--
airflow/providers/amazon/aws/hooks/sns.py | 11 +-
.../providers/amazon/aws/transfers/gcs_to_s3.py | 9 +-
.../cncf/kubernetes/operators/kubernetes_pod.py | 8 +-
airflow/providers/dependencies.json | 1 +
.../example_azure_fileshare_to_gcs.py} | 44 ++-
airflow/providers/google/cloud/hooks/bigquery.py | 14 +-
airflow/providers/google/cloud/hooks/gcs.py | 10 +
.../providers/google/cloud/operators/bigquery.py | 5 +-
.../cloud/transfers/azure_fileshare_to_gcs.py | 182 +++++++++++
.../providers/google/cloud/transfers/s3_to_gcs.py | 12 +-
.../example_dags/example_azure_blob_to_gcs.py | 68 +++++
.../azure/example_dags/example_local_to_adls.py} | 26 +-
airflow/providers/microsoft/azure/hooks/adx.py | 4 +-
.../providers/microsoft/azure/hooks/azure_batch.py | 43 ++-
.../azure/hooks/azure_container_instance.py | 18 +-
.../azure/hooks/azure_container_registry.py | 4 +-
.../azure/hooks/azure_container_volume.py | 8 +-
.../microsoft/azure/hooks/azure_cosmos.py | 47 +--
.../microsoft/azure/hooks/azure_data_lake.py | 38 ++-
.../microsoft/azure/hooks/azure_fileshare.py | 85 +++++-
airflow/providers/microsoft/azure/hooks/wasb.py | 27 +-
.../microsoft/azure/operators/azure_batch.py | 8 +-
.../azure/transfers/azure_blob_to_gcs.py} | 118 +++++---
.../microsoft/azure/transfers/local_to_adls.py | 103 +++++++
airflow/providers/zendesk/hooks/zendesk.py | 16 +-
breeze | 30 +-
breeze-complete | 3 +-
chart/README.md | 2 +-
chart/templates/NOTES.txt | 4 +-
chart/templates/_helpers.yaml | 6 +-
chart/templates/configmap.yaml | 2 +-
chart/templates/flower/flower-deployment.yaml | 2 +-
chart/templates/flower/flower-ingress.yaml | 2 +-
chart/templates/flower/flower-service.yaml | 2 +-
chart/templates/rbac/pod-launcher-role.yaml | 7 +
chart/templates/rbac/pod-launcher-rolebinding.yaml | 11 +-
chart/templates/redis/redis-networkpolicy.yaml | 2 +-
chart/templates/redis/redis-service.yaml | 2 +-
chart/templates/redis/redis-statefulset.yaml | 2 +-
.../templates/scheduler/scheduler-deployment.yaml | 2 -
chart/templates/workers/worker-deployment.yaml | 2 +-
chart/templates/workers/worker-kedaautoscaler.yaml | 2 +-
chart/templates/workers/worker-networkpolicy.yaml | 2 +-
chart/templates/workers/worker-service.yaml | 2 +-
...t.yaml => celery-kubernetes-executor_test.yaml} | 33 +-
.../tests/celery-kubernetes-pod-launcher-role.yaml | 21 +-
chart/values.schema.json | 4 +
chart/values.yaml | 3 +
dev/README.md | 337 +++++++++++++++++++++
docs/cli-and-env-variables-ref.rst | 85 ++++++
docs/cli-ref.rst | 37 ---
docs/concepts.rst | 2 +-
docs/conf.py | 1 +
.../google/transfer/azure_fileshare_to_gcs.rst | 54 ++++
docs/howto/operator/index.rst | 1 +
.../_partials/prerequisite_tasks.rst | 10 +-
.../howto/operator/{apache => microsoft}/index.rst | 8 +-
.../microsoft/transfer/blob_storage_to_gcs.rst | 61 ++++
.../{amazon/aws => microsoft}/transfer/index.rst | 4 +-
.../operator/microsoft/transfer/local_to_adls.rst | 58 ++++
docs/howto/variable.rst | 2 +
docs/index.rst | 2 +-
docs/operators-and-hooks-ref.rst | 17 +-
docs/redirects.txt | 4 +
docs/usage-cli.rst | 2 +-
scripts/ci/docker-compose/base.yml | 2 +
scripts/ci/libraries/_initialization.sh | 14 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 29 ++
scripts/ci/mysql/conf.d/airflow.cnf | 4 +-
scripts/docker/install_mysql.sh | 4 +-
scripts/{ci/dockerfiles/bats => docker}/load.bash | 0
scripts/in_container/check_environment.sh | 9 +-
scripts/in_container/entrypoint_ci.sh | 14 +
setup.py | 2 +-
tests/api/common/experimental/test_mark_tasks.py | 2 +-
tests/deprecated_classes.py | 4 +
tests/operators/test_latest_only_operator.py | 2 +-
.../amazon/aws/transfers/test_gcs_to_s3.py | 32 ++
...s3_to_gcs.py => test_azure_fileshare_to_gcs.py} | 85 +++---
.../test_azure_fileshare_to_gcs_system.py | 80 +++++
.../microsoft/azure/hooks/test_azure_batch.py | 15 +
.../microsoft/azure/hooks/test_azure_fileshare.py | 29 ++
.../microsoft/azure/operators/test_azure_batch.py | 22 ++
.../azure/transfers/test_azure_blob_to_gcs.py | 103 +++++++
.../azure/transfers/test_local_to_adls.py | 74 +++++
tests/providers/mysql/hooks/test_mysql.py | 8 +
tests/providers/zendesk/hooks/test_zendesk.py | 2 +-
tests/test_utils/azure_system_helpers.py | 89 ++++++
tests/test_utils/gcp_system_helpers.py | 8 +
119 files changed, 2368 insertions(+), 522 deletions(-)
create mode 100644 .github/workflows/codeql-cancel.yml
copy airflow/operators/{latest_only_operator.py => latest_only.py} (100%)
copy airflow/providers/{microsoft/azure/example_dags/example_azure_container_instances.py => google/cloud/example_dags/example_azure_fileshare_to_gcs.py} (55%)
create mode 100644 airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py
create mode 100644 airflow/providers/microsoft/azure/example_dags/example_azure_blob_to_gcs.py
copy airflow/providers/{google/cloud/example_dags/example_postgres_to_gcs.py => microsoft/azure/example_dags/example_local_to_adls.py} (62%)
copy airflow/providers/{google/suite/transfers/gcs_to_sheets.py => microsoft/azure/transfers/azure_blob_to_gcs.py} (56%)
create mode 100644 airflow/providers/microsoft/azure/transfers/local_to_adls.py
copy chart/tests/{git-sync-worker_test.yaml => celery-kubernetes-executor_test.yaml} (60%)
copy scripts/ci/spectral_rules/connexion.yml => chart/tests/celery-kubernetes-pod-launcher-role.yaml (71%)
create mode 100644 docs/cli-and-env-variables-ref.rst
delete mode 100644 docs/cli-ref.rst
create mode 100644 docs/howto/operator/google/transfer/azure_fileshare_to_gcs.rst
copy docs/howto/operator/{amazon/aws => microsoft}/_partials/prerequisite_tasks.rst (78%)
copy docs/howto/operator/{apache => microsoft}/index.rst (93%)
create mode 100644 docs/howto/operator/microsoft/transfer/blob_storage_to_gcs.rst
copy docs/howto/operator/{amazon/aws => microsoft}/transfer/index.rst (93%)
create mode 100644 docs/howto/operator/microsoft/transfer/local_to_adls.rst
copy scripts/{ci/dockerfiles/bats => docker}/load.bash (100%)
copy tests/providers/google/cloud/transfers/{test_s3_to_gcs.py => test_azure_fileshare_to_gcs.py} (56%)
create mode 100644 tests/providers/google/cloud/transfers/test_azure_fileshare_to_gcs_system.py
create mode 100644 tests/providers/microsoft/azure/transfers/test_azure_blob_to_gcs.py
create mode 100644 tests/providers/microsoft/azure/transfers/test_local_to_adls.py
create mode 100644 tests/test_utils/azure_system_helpers.py