You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by gi...@apache.org on 2020/06/17 02:58:44 UTC
[airflow] tag nightly-master updated (7c44518 -> bbdbe81)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 7c44518 (commit)
to bbdbe81 (commit)
from 7c44518 Make experimental/endpoints.py Pylint compatible (#9287)
add 12d39cd Make airflow/models/errors.py Pylint compatible (#9288)
add e5ce87a Send Celery tasks from main process when sync_parallelism is 1 (#9253)
add a8cd23c Fix tree view if config contains " (#9250)
add aee6ab9 Wait for pipeline state in Data Fusion operators (#8954)
add 431ea32 Resolve upstream tasks when template field is XComArg (#8805)
add c44f1b1 Include some missing RBAC roles on User and Viewer roles (#9133)
add 9312a04 Remove duplicated log line from `db upgrade` (#9305)
add 832593a Add /version endpoint (#9296)
add d0e06ef Keep consistent system test names (#9272)
add 2c18a3f Fix failing tests from #9250 (#9307)
add 1459970 Rename CloudBuildCreateBuildOperator to CloudBuildCreateOperator (#9314)
add 8b0073d Get all pod logs on k8s launching failure (#9317)
add a80cd25 Close/Flush byte stream in s3 hook load_string and load_bytes (#9211)
add e96eac1 Add nullable and required to some fields in OpenAPI spec #9315 (#9315)
add caa2a75 Add event log endpoints (#9227)
add 3de6850 Add schema and read-only endpoints for Pools (#9097)
add 696e745 Fix broken CI image optimisation (#9313)
add 58a8ec0 AWSBatchOperator <> ClientHook relation changed to composition (#9306)
add 47bddf7 Add schema and read-only endpoints for Import errors (#9217)
add 639972d Add support for latest Apache Beam SDK in Dataflow operators (#9323)
add 7c12a9d Improve production image iteration speed (#9162)
add 2fc13f0 Fixes unbound variable on MacOS (#9335)
add d6e5e7c clarify breeze initialize virtualenv instructions (#9319)
add c78e2a5 Make hive macros py3 compatible (#8598)
add f6bd817 Introduce 'transfers' packages (#9320)
add bbdbe81 Add GrowthSimple to list of Airflow users (#9337)
No new revisions were added by this update.
Summary of changes:
.dockerignore | 2 +-
.github/workflows/ci.yml | 2 +-
.pre-commit-config.yaml | 2 +
BREEZE.rst | 411 +++++++--------------
CONTRIBUTING.rst | 68 +++-
Dockerfile | 35 +-
Dockerfile.ci | 34 +-
IMAGES.rst | 350 ++++++++++++++----
LOCAL_VIRTUALENV.rst | 2 +-
README.md | 1 +
TESTING.rst | 10 +-
UPDATING.md | 6 +-
.../api_connexion/endpoints/event_log_endpoint.py | 31 +-
.../endpoints/import_error_endpoint.py | 62 ++++
airflow/api_connexion/endpoints/pool_endpoint.py | 31 +-
...port_errror_endpoint.py => version_endpoint.py} | 34 +-
airflow/api_connexion/exceptions.py | 31 +-
airflow/api_connexion/openapi/v1.yaml | 36 +-
airflow/api_connexion/parameters.py | 4 +
airflow/api_connexion/schemas/error_schema.py | 56 +++
airflow/api_connexion/schemas/event_log_schema.py | 57 +++
airflow/api_connexion/schemas/pool_schema.py | 87 +++++
.../{exceptions.py => schemas/version_schema.py} | 13 +-
airflow/contrib/operators/adls_to_gcs.py | 10 +-
airflow/contrib/operators/bigquery_to_bigquery.py | 6 +-
airflow/contrib/operators/bigquery_to_gcs.py | 10 +-
.../operators/bigquery_to_mysql_operator.py | 6 +-
airflow/contrib/operators/cassandra_to_gcs.py | 10 +-
airflow/contrib/operators/dynamodb_to_s3.py | 6 +-
airflow/contrib/operators/file_to_gcs.py | 10 +-
airflow/contrib/operators/file_to_wasb.py | 6 +-
.../contrib/operators/gcp_cloud_build_operator.py | 17 +-
airflow/contrib/operators/gcp_transfer_operator.py | 4 +-
airflow/contrib/operators/gcs_download_operator.py | 8 +-
airflow/contrib/operators/gcs_to_bq.py | 10 +-
airflow/contrib/operators/gcs_to_gcs.py | 10 +-
.../contrib/operators/gcs_to_gdrive_operator.py | 6 +-
airflow/contrib/operators/gcs_to_s3.py | 10 +-
airflow/contrib/operators/hive_to_dynamodb.py | 6 +-
.../operators/imap_attachment_to_s3_operator.py | 6 +-
airflow/contrib/operators/mongo_to_s3.py | 6 +-
airflow/contrib/operators/mssql_to_gcs.py | 10 +-
airflow/contrib/operators/mysql_to_gcs.py | 10 +-
.../oracle_to_azure_data_lake_transfer.py | 8 +-
.../contrib/operators/oracle_to_oracle_transfer.py | 12 +-
.../contrib/operators/postgres_to_gcs_operator.py | 10 +-
airflow/contrib/operators/s3_to_gcs_operator.py | 6 +-
airflow/contrib/operators/s3_to_sftp_operator.py | 6 +-
airflow/contrib/operators/sftp_to_s3_operator.py | 6 +-
airflow/contrib/operators/sql_to_gcs.py | 10 +-
airflow/contrib/operators/vertica_to_hive.py | 12 +-
airflow/contrib/operators/vertica_to_mysql.py | 12 +-
.../example_xcomargs.py} | 48 ++-
airflow/executors/celery_executor.py | 9 +-
airflow/models/baseoperator.py | 96 ++++-
airflow/models/errors.py | 6 +-
airflow/models/taskinstance.py | 5 +-
airflow/operators/gcs_to_s3.py | 6 +-
airflow/operators/google_api_to_s3_transfer.py | 14 +-
airflow/operators/hive_to_druid.py | 12 +-
airflow/operators/hive_to_mysql.py | 12 +-
airflow/operators/hive_to_samba_operator.py | 6 +-
airflow/operators/mssql_to_hive.py | 12 +-
airflow/operators/mysql_to_hive.py | 12 +-
airflow/operators/presto_to_mysql.py | 12 +-
airflow/operators/redshift_to_s3_operator.py | 12 +-
airflow/operators/s3_to_hive_operator.py | 12 +-
airflow/operators/s3_to_redshift_operator.py | 10 +-
.../amazon/PROVIDERS_CHANGES_2020.05.20.md | 8 +
airflow/providers/amazon/README.md | 57 +--
.../example_google_api_to_s3_transfer_advanced.py | 6 +-
.../example_google_api_to_s3_transfer_basic.py | 4 +-
.../example_dags/example_imap_attachment_to_s3.py | 2 +-
.../aws/example_dags/example_s3_to_redshift.py | 6 +-
airflow/providers/amazon/aws/hooks/batch_client.py | 3 +-
airflow/providers/amazon/aws/hooks/s3.py | 2 +
airflow/providers/amazon/aws/operators/batch.py | 24 +-
.../providers/amazon/aws/transfers}/__init__.py | 0
.../aws/{operators => transfers}/dynamodb_to_s3.py | 0
.../aws/{operators => transfers}/gcs_to_s3.py | 0
.../google_api_to_s3.py} | 2 +-
.../{operators => transfers}/hive_to_dynamodb.py | 2 +-
.../imap_attachment_to_s3.py | 0
.../aws/{operators => transfers}/mongo_to_s3.py | 0
.../aws/{operators => transfers}/redshift_to_s3.py | 2 +-
.../aws/{operators => transfers}/s3_to_redshift.py | 4 +-
.../aws/{operators => transfers}/s3_to_sftp.py | 0
.../aws/{operators => transfers}/sftp_to_s3.py | 0
.../cassandra/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/cassandra/README.md | 3 +
.../apache/druid/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/druid/README.md | 24 +-
.../providers/apache/druid/transfers}/__init__.py | 0
.../{operators => transfers}/hive_to_druid.py | 2 +-
.../apache/hdfs/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/hdfs/README.md | 3 +
.../apache/hive/PROVIDERS_CHANGES_2020.05.20.md | 3 +
airflow/providers/apache/hive/README.md | 38 +-
airflow/providers/apache/hive/hooks/hive.py | 3 +-
.../providers/apache/hive/transfers}/__init__.py | 0
.../hive/{operators => transfers}/hive_to_mysql.py | 2 +-
.../hive/{operators => transfers}/hive_to_samba.py | 2 +-
.../hive/{operators => transfers}/mssql_to_hive.py | 2 +-
.../hive/{operators => transfers}/mysql_to_hive.py | 2 +-
.../hive/{operators => transfers}/s3_to_hive.py | 2 +-
.../{operators => transfers}/vertica_to_hive.py | 2 +-
.../apache/livy/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/livy/README.md | 3 +
.../apache/pig/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/pig/README.md | 3 +
.../apache/pinot/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/pinot/README.md | 3 +
.../apache/spark/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/spark/README.md | 3 +
.../apache/sqoop/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/apache/sqoop/README.md | 3 +
.../celery/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/celery/README.md | 3 +
.../cloudant/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/cloudant/README.md | 3 +
.../databricks/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/databricks/README.md | 3 +
.../datadog/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/datadog/README.md | 3 +
.../dingding/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/dingding/README.md | 3 +
.../discord/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/discord/README.md | 3 +
.../docker/PROVIDERS_CHANGES_2020.05.20.md | 3 +
airflow/providers/docker/README.md | 5 +
.../elasticsearch/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/elasticsearch/README.md | 3 +
.../email/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/email/README.md | 3 +
.../exasol/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/exasol/README.md | 3 +
.../facebook/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/facebook/README.md | 3 +
.../providers/ftp/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/ftp/README.md | 3 +
.../google/PROVIDERS_CHANGES_2020.05.20.md | 20 +
airflow/providers/google/README.md | 110 ++++--
.../google/ads/example_dags/example_ads.py | 3 +-
airflow/providers/google/ads/operators/ads.py | 96 +----
.../providers/google/ads/transfers}/__init__.py | 0
.../{operators/ads.py => transfers/ads_to_gcs.py} | 85 +----
.../example_dags/example_bigquery_to_bigquery.py | 2 +-
.../cloud/example_dags/example_bigquery_to_gcs.py | 2 +-
.../example_dags/example_bigquery_transfer.py | 4 +-
.../cloud/example_dags/example_cloud_build.py | 8 +-
.../google/cloud/example_dags/example_dataflow.py | 25 +-
.../cloud/example_dags/example_datafusion.py | 51 +--
.../example_dags/example_facebook_ads_to_gcs.py | 4 +-
.../google/cloud/example_dags/example_gcs.py | 11 +-
.../cloud/example_dags/example_gcs_to_bigquery.py | 2 +-
.../cloud/example_dags/example_gcs_to_gcs.py | 5 +-
.../cloud/example_dags/example_gcs_to_sftp.py | 2 +-
.../cloud/example_dags/example_local_to_gcs.py | 2 +-
.../cloud/example_dags/example_postgres_to_gcs.py | 2 +-
.../cloud/example_dags/example_presto_to_gcs.py | 2 +-
.../cloud/example_dags/example_sftp_to_gcs.py | 2 +-
.../cloud/example_dags/example_sheets_to_gcs.py | 2 +-
airflow/providers/google/cloud/hooks/dataflow.py | 85 +++--
airflow/providers/google/cloud/hooks/datafusion.py | 162 ++++++--
.../google/cloud/operators/cloud_build.py | 4 +-
.../providers/google/cloud/operators/datafusion.py | 42 ++-
airflow/providers/google/cloud/operators/gcs.py | 188 +++++-----
.../providers/google/cloud/transfers}/__init__.py | 0
.../cloud/{operators => transfers}/adls_to_gcs.py | 0
.../bigquery_to_bigquery.py | 0
.../{operators => transfers}/bigquery_to_gcs.py | 0
.../{operators => transfers}/bigquery_to_mysql.py | 0
.../{operators => transfers}/cassandra_to_gcs.py | 0
.../facebook_ads_to_gcs.py | 0
.../{operators => transfers}/gcs_to_bigquery.py | 0
.../cloud/{operators => transfers}/gcs_to_gcs.py | 92 -----
.../google/cloud/transfers/gcs_to_local.py | 122 ++++++
.../cloud/{operators => transfers}/gcs_to_sftp.py | 0
.../cloud/{operators => transfers}/local_to_gcs.py | 0
.../cloud/{operators => transfers}/mssql_to_gcs.py | 2 +-
.../cloud/{operators => transfers}/mysql_to_gcs.py | 2 +-
.../{operators => transfers}/postgres_to_gcs.py | 2 +-
.../{operators => transfers}/presto_to_gcs.py | 2 +-
.../cloud/{operators => transfers}/s3_to_gcs.py | 0
.../cloud/{operators => transfers}/sftp_to_gcs.py | 0
.../{operators => transfers}/sheets_to_gcs.py | 0
.../cloud/{operators => transfers}/sql_to_gcs.py | 2 +-
.../example_dags/example_display_video.py | 2 +-
.../suite/example_dags/example_gcs_to_gdrive.py | 2 +-
.../suite/example_dags/example_gcs_to_sheets.py | 4 +-
.../google/suite/example_dags/example_sheets.py | 4 +-
.../providers/google/suite/transfers}/__init__.py | 0
.../{operators => transfers}/gcs_to_gdrive.py | 0
.../{operators => transfers}/gcs_to_sheets.py | 0
.../providers/grpc/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/grpc/README.md | 3 +
.../hashicorp/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/hashicorp/README.md | 3 +
.../providers/http/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/http/README.md | 3 +
.../providers/imap/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/imap/README.md | 3 +
.../providers/jdbc/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/jdbc/README.md | 3 +
.../jenkins/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/jenkins/README.md | 3 +
.../providers/jira/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/jira/README.md | 3 +
.../azure/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/microsoft/azure/README.md | 34 +-
.../microsoft/azure/transfers}/__init__.py | 0
.../azure/{operators => transfers}/file_to_wasb.py | 0
.../oracle_to_azure_data_lake.py} | 2 +-
.../mssql/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/microsoft/mssql/README.md | 3 +
.../winrm/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/microsoft/winrm/README.md | 3 +
.../mongo/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/mongo/README.md | 3 +
.../mysql/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/mysql/README.md | 36 +-
.../providers/mysql/transfers}/__init__.py | 0
.../{operators => transfers}/presto_to_mysql.py | 2 +-
.../mysql/{operators => transfers}/s3_to_mysql.py | 2 +-
.../{operators => transfers}/vertica_to_mysql.py | 2 +-
.../providers/odbc/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/odbc/README.md | 3 +
.../openfaas/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/openfaas/README.md | 3 +
.../opsgenie/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/opsgenie/README.md | 3 +
.../oracle/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/oracle/README.md | 22 +-
.../providers/oracle/transfers}/__init__.py | 0
.../oracle_to_oracle.py} | 2 +-
.../pagerduty/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/pagerduty/README.md | 3 +
.../postgres/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/postgres/README.md | 3 +
.../presto/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/presto/README.md | 3 +
.../qubole/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/qubole/README.md | 3 +
.../redis/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/redis/README.md | 3 +
.../salesforce/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/salesforce/README.md | 3 +
.../samba/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/samba/README.md | 3 +
.../segment/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/segment/README.md | 3 +
.../providers/sftp/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/sftp/README.md | 3 +
.../singularity/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/singularity/README.md | 3 +
.../slack/PROVIDERS_CHANGES_2020.05.20.md | 2 +
airflow/providers/slack/README.md | 11 +
.../snowflake/PROVIDERS_CHANGES_2020.05.20.md | 2 +
airflow/providers/snowflake/README.md | 40 +-
.../snowflake/example_dags/example_snowflake.py | 6 +-
.../providers/snowflake/transfers}/__init__.py | 0
.../{operators => transfers}/s3_to_snowflake.py | 2 +-
.../{operators => transfers}/snowflake_to_slack.py | 0
.../sqlite/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/sqlite/README.md | 3 +
.../providers/ssh/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/ssh/README.md | 3 +
.../vertica/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/vertica/README.md | 3 +
.../yandex/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/yandex/README.md | 3 +
.../zendesk/PROVIDERS_CHANGES_2020.05.20.md | 1 +
airflow/providers/zendesk/README.md | 3 +
airflow/serialization/serialized_objects.py | 4 +-
airflow/utils/db.py | 2 -
airflow/utils/platform.py | 15 +
airflow/www/security.py | 2 +
airflow/www/templates/airflow/tree.html | 2 +-
airflow/www/views.py | 12 +-
.../PROVIDERS_CLASSES_TEMPLATE.md.jinja2 | 27 +-
.../PROVIDERS_README_TEMPLATE.md.jinja2 | 18 +-
backport_packages/refactor_backport_packages.py | 26 +-
backport_packages/setup_backport_packages.py | 245 +++++++-----
breeze | 263 +++++--------
breeze-complete | 13 +-
dev/BACKPORT_PACKAGES.md | 22 +-
docs/autoapi_templates/index.rst | 20 +
docs/build | 3 +-
docs/concepts.rst | 2 +-
docs/howto/define_extra_link.rst | 4 +-
.../amazon/aws/google_api_to_s3_transfer.rst | 2 +-
.../operator/amazon/aws/imap_attachment_to_s3.rst | 2 +-
docs/howto/operator/amazon/aws/s3_to_redshift.rst | 8 +-
docs/howto/operator/gcp/ads.rst | 8 +-
docs/howto/operator/gcp/cloud_build.rst | 6 +-
docs/howto/operator/gcp/facebook_ads_to_gcs.rst | 2 +-
docs/howto/operator/gcp/gcs.rst | 2 +-
docs/howto/operator/gcp/gcs_to_gcs.rst | 4 +-
docs/howto/operator/gcp/gcs_to_gdrive.rst | 4 +-
.../gcp/{local_to_gcs.rst => gcs_to_local.rst} | 21 +-
docs/howto/operator/gcp/gcs_to_sftp.rst | 4 +-
docs/howto/operator/gcp/gcs_to_sheets.rst | 4 +-
docs/howto/operator/gcp/local_to_gcs.rst | 2 +-
docs/howto/operator/gcp/presto_to_gcs.rst | 4 +-
docs/howto/operator/gcp/sftp_to_gcs.rst | 4 +-
docs/howto/operator/gcp/sheets_to_gcs.rst | 4 +-
docs/installation.rst | 98 ++---
docs/operators-and-hooks-ref.rst | 153 ++++----
docs/plugins.rst | 2 +-
requirements/requirements-python3.6.txt | 42 +--
requirements/requirements-python3.7.txt | 42 +--
requirements/requirements-python3.8.txt | 44 +--
scripts/ci/ci_run_airflow_testing.sh | 6 +-
scripts/ci/docker-compose/base.yml | 7 -
scripts/ci/docker-compose/local-prod.yml | 2 +-
scripts/ci/docker-compose/local.yml | 4 +-
scripts/ci/in_container/_in_container_utils.sh | 2 +-
scripts/ci/in_container/entrypoint_ci.sh | 42 +--
scripts/ci/in_container/entrypoint_exec.sh | 2 +-
scripts/ci/in_container/run_ci_tests.sh | 9 +-
scripts/ci/libraries/_build_images.sh | 92 ++---
scripts/ci/libraries/_initialization.sh | 12 +-
scripts/ci/libraries/_kind.sh | 12 +-
scripts/ci/libraries/_local_mounts.sh | 2 +-
scripts/ci/libraries/_md5sum.sh | 4 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 5 +-
scripts/ci/libraries/_start_end.sh | 4 +
scripts/ci/libraries/_verbosity.sh | 11 +
scripts/ci/pylint_todo.txt | 1 -
scripts/docker/entrypoint.sh | 110 ------
scripts/{include => prod}/clean-logs.sh | 0
entrypoint.sh => scripts/prod/entrypoint_prod.sh | 0
.../endpoints/test_event_log_endpoint.py | 177 ++++++++-
.../endpoints/test_import_error_endpoint.py | 169 +++++++++
.../api_connexion/endpoints/test_pool_endpoint.py | 129 +++++--
...errror_endpoint.py => test_version_endpoint.py} | 37 +-
tests/api_connexion/schemas/test_error_schema.py | 106 ++++++
.../api_connexion/schemas/test_event_log_schema.py | 126 +++++++
tests/api_connexion/schemas/test_pool_schemas.py | 93 +++++
.../api_connexion/schemas/test_version_schema.py | 34 +-
tests/models/test_baseoperator.py | 61 ++-
tests/providers/amazon/aws/operators/test_batch.py | 19 +-
.../hooks => amazon/aws/transfers}/__init__.py | 0
.../test_dynamodb_to_s3.py | 6 +-
.../aws/{operators => transfers}/test_gcs_to_s3.py | 12 +-
.../test_google_api_to_s3.py} | 30 +-
.../test_google_api_to_s3_system.py} | 0
.../test_hive_to_dynamodb.py | 8 +-
.../test_imap_attachment_to_s3.py | 6 +-
.../test_imap_attachment_to_s3_system.py | 0
.../{operators => transfers}/test_mongo_to_s3.py | 6 +-
.../test_redshift_to_s3.py | 4 +-
.../test_s3_to_redshift.py | 4 +-
.../test_s3_to_redshift_system.py | 0
.../{operators => transfers}/test_s3_to_sftp.py | 2 +-
.../{operators => transfers}/test_sftp_to_s3.py | 2 +-
.../hooks => apache/druid/transfers}/__init__.py | 0
.../{operators => transfers}/test_hive_to_druid.py | 4 +-
tests/providers/apache/hive/hooks/test_hive.py | 15 +-
.../hooks => apache/hive/transfers}/__init__.py | 0
.../{operators => transfers}/test_hive_to_mysql.py | 40 +-
.../{operators => transfers}/test_hive_to_samba.py | 16 +-
.../{operators => transfers}/test_mssql_to_hive.py | 30 +-
.../{operators => transfers}/test_mysql_to_hive.py | 14 +-
.../{operators => transfers}/test_s3_to_hive.py | 24 +-
.../apache/hive/transfers/test_vertica_to_hive.py | 68 ++++
tests/providers/google/ads/operators/test_ads.py | 29 +-
.../hooks => google/ads/transfers}/__init__.py | 0
.../google/ads/transfers/test_ads_to_gcs.py | 50 +++
.../providers/google/cloud/hooks/test_dataflow.py | 82 +++-
.../google/cloud/hooks/test_datafusion.py | 27 +-
.../google/cloud/operators/test_cloud_build.py | 14 +-
...erator_system.py => test_cloud_build_system.py} | 0
.../test_cloud_sql_operatorquery_system.py | 128 -------
.../cloud/operators/test_cloud_sql_system.py | 101 ++++-
.../google/cloud/operators/test_dataflow_system.py | 12 +-
.../google/cloud/operators/test_datafusion.py | 8 +-
..._to_gcs_system.py => test_datafusion_system.py} | 27 +-
tests/providers/google/cloud/operators/test_gcs.py | 50 ++-
.../hooks => google/cloud/transfers}/__init__.py | 0
.../{operators => transfers}/test_adls_to_gcs.py | 10 +-
.../test_bigquery_to_bigquery.py | 4 +-
.../test_bigquery_to_bigquery_system.py | 0
.../test_bigquery_to_gcs.py | 4 +-
.../test_bigquery_to_gcs_system.py | 0
.../test_bigquery_to_mysql.py | 4 +-
.../test_cassandra_to_gcs.py | 8 +-
.../test_facebook_ads_to_gcs.py | 6 +-
.../test_facebook_ads_to_gcs_system.py | 0
.../test_gcs_to_bigquery.py | 6 +-
.../test_gcs_to_bigquery_system.py | 0
.../{operators => transfers}/test_gcs_to_gcs.py | 92 ++---
.../test_gcs_to_gcs_system.py} | 0
.../google/cloud/transfers/test_gcs_to_local.py | 47 +++
.../{operators => transfers}/test_gcs_to_sftp.py | 22 +-
.../test_gcs_to_sftp_system.py | 0
.../{operators => transfers}/test_local_to_gcs.py | 4 +-
.../test_local_to_gcs_system.py | 0
.../{operators => transfers}/test_mssql_to_gcs.py | 14 +-
.../{operators => transfers}/test_mysql_to_gcs.py | 38 +-
.../test_postgres_to_gcs.py | 8 +-
.../test_postgres_to_gcs_system.py | 0
.../{operators => transfers}/test_presto_to_gcs.py | 26 +-
.../test_presto_to_gcs_system.py | 0
.../{operators => transfers}/test_s3_to_gcs.py | 10 +-
.../{operators => transfers}/test_sftp_to_gcs.py | 22 +-
.../test_sftp_to_gcs_system.py | 0
.../{operators => transfers}/test_sheets_to_gcs.py | 14 +-
.../test_sheets_to_gcs_system.py | 0
.../google/cloud/utils/gcp_authenticator.py | 1 +
.../hooks => google/suite/transfers}/__init__.py | 0
.../{operators => transfers}/test_gcs_to_gdrive.py | 4 +-
.../{operators => transfers}/test_gcs_to_sheets.py | 10 +-
.../test_gcs_to_sheets_system.py | 0
.../azure/transfers}/__init__.py | 0
.../{operators => transfers}/test_file_to_wasb.py | 4 +-
.../test_oracle_to_azure_data_lake.py} | 10 +-
.../{zendesk/hooks => mysql/transfers}/__init__.py | 0
.../test_presto_to_mysql.py | 16 +-
.../{operators => transfers}/test_s3_to_mysql.py | 18 +-
.../test_vertica_to_mysql.py | 44 +--
.../oracle/{operators => transfers}/__init__.py | 0
.../test_oracle_to_oracle.py} | 4 +-
.../hooks => snowflake/transfers}/__init__.py | 0
.../test_s3_to_snowflake.py | 6 +-
.../test_snowflake_to_slack.py | 6 +-
tests/serialization/test_dag_serialization.py | 3 +-
tests/test_core_to_contrib.py | 368 +++++++++---------
tests/test_project_structure.py | 41 +-
tests/test_utils/db.py | 5 +
tests/www/test_views.py | 13 +-
431 files changed, 4820 insertions(+), 2931 deletions(-)
create mode 100644 airflow/api_connexion/endpoints/import_error_endpoint.py
rename airflow/api_connexion/endpoints/{import_errror_endpoint.py => version_endpoint.py} (56%)
create mode 100644 airflow/api_connexion/schemas/error_schema.py
create mode 100644 airflow/api_connexion/schemas/event_log_schema.py
create mode 100644 airflow/api_connexion/schemas/pool_schema.py
copy airflow/api_connexion/{exceptions.py => schemas/version_schema.py} (75%)
copy airflow/{providers/google/cloud/example_dags/example_postgres_to_gcs.py => example_dags/example_xcomargs.py} (56%)
copy {tests/providers/zendesk/hooks => airflow/providers/amazon/aws/transfers}/__init__.py (100%)
rename airflow/providers/amazon/aws/{operators => transfers}/dynamodb_to_s3.py (100%)
rename airflow/providers/amazon/aws/{operators => transfers}/gcs_to_s3.py (100%)
rename airflow/providers/amazon/aws/{operators/google_api_to_s3_transfer.py => transfers/google_api_to_s3.py} (99%)
rename airflow/providers/amazon/aws/{operators => transfers}/hive_to_dynamodb.py (98%)
rename airflow/providers/amazon/aws/{operators => transfers}/imap_attachment_to_s3.py (100%)
rename airflow/providers/amazon/aws/{operators => transfers}/mongo_to_s3.py (100%)
rename airflow/providers/amazon/aws/{operators => transfers}/redshift_to_s3.py (99%)
rename airflow/providers/amazon/aws/{operators => transfers}/s3_to_redshift.py (98%)
rename airflow/providers/amazon/aws/{operators => transfers}/s3_to_sftp.py (100%)
rename airflow/providers/amazon/aws/{operators => transfers}/sftp_to_s3.py (100%)
copy {tests/providers/zendesk/hooks => airflow/providers/apache/druid/transfers}/__init__.py (100%)
rename airflow/providers/apache/druid/{operators => transfers}/hive_to_druid.py (99%)
copy {tests/providers/zendesk/hooks => airflow/providers/apache/hive/transfers}/__init__.py (100%)
rename airflow/providers/apache/hive/{operators => transfers}/hive_to_mysql.py (99%)
rename airflow/providers/apache/hive/{operators => transfers}/hive_to_samba.py (98%)
rename airflow/providers/apache/hive/{operators => transfers}/mssql_to_hive.py (99%)
rename airflow/providers/apache/hive/{operators => transfers}/mysql_to_hive.py (99%)
rename airflow/providers/apache/hive/{operators => transfers}/s3_to_hive.py (99%)
rename airflow/providers/apache/hive/{operators => transfers}/vertica_to_hive.py (99%)
copy {tests/providers/zendesk/hooks => airflow/providers/google/ads/transfers}/__init__.py (100%)
copy airflow/providers/google/ads/{operators/ads.py => transfers/ads_to_gcs.py} (60%)
copy {tests/providers/zendesk/hooks => airflow/providers/google/cloud/transfers}/__init__.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/adls_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/bigquery_to_bigquery.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/bigquery_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/bigquery_to_mysql.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/cassandra_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/facebook_ads_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/gcs_to_bigquery.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/gcs_to_gcs.py (82%)
create mode 100644 airflow/providers/google/cloud/transfers/gcs_to_local.py
rename airflow/providers/google/cloud/{operators => transfers}/gcs_to_sftp.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/local_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/mssql_to_gcs.py (97%)
rename airflow/providers/google/cloud/{operators => transfers}/mysql_to_gcs.py (98%)
rename airflow/providers/google/cloud/{operators => transfers}/postgres_to_gcs.py (98%)
rename airflow/providers/google/cloud/{operators => transfers}/presto_to_gcs.py (99%)
rename airflow/providers/google/cloud/{operators => transfers}/s3_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/sftp_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/sheets_to_gcs.py (100%)
rename airflow/providers/google/cloud/{operators => transfers}/sql_to_gcs.py (99%)
copy {tests/providers/zendesk/hooks => airflow/providers/google/suite/transfers}/__init__.py (100%)
rename airflow/providers/google/suite/{operators => transfers}/gcs_to_gdrive.py (100%)
rename airflow/providers/google/suite/{operators => transfers}/gcs_to_sheets.py (100%)
copy {tests/providers/zendesk/hooks => airflow/providers/microsoft/azure/transfers}/__init__.py (100%)
rename airflow/providers/microsoft/azure/{operators => transfers}/file_to_wasb.py (100%)
rename airflow/providers/microsoft/azure/{operators/oracle_to_azure_data_lake_transfer.py => transfers/oracle_to_azure_data_lake.py} (98%)
copy {tests/providers/zendesk/hooks => airflow/providers/mysql/transfers}/__init__.py (100%)
rename airflow/providers/mysql/{operators => transfers}/presto_to_mysql.py (98%)
rename airflow/providers/mysql/{operators => transfers}/s3_to_mysql.py (98%)
rename airflow/providers/mysql/{operators => transfers}/vertica_to_mysql.py (99%)
copy {tests/providers/zendesk/hooks => airflow/providers/oracle/transfers}/__init__.py (100%)
rename airflow/providers/oracle/{operators/oracle_to_oracle_transfer.py => transfers/oracle_to_oracle.py} (98%)
copy {tests/providers/zendesk/hooks => airflow/providers/snowflake/transfers}/__init__.py (100%)
rename airflow/providers/snowflake/{operators => transfers}/s3_to_snowflake.py (98%)
rename airflow/providers/snowflake/{operators => transfers}/snowflake_to_slack.py (100%)
copy docs/howto/operator/gcp/{local_to_gcs.rst => gcs_to_local.rst} (68%)
delete mode 100755 scripts/docker/entrypoint.sh
rename scripts/{include => prod}/clean-logs.sh (100%)
rename entrypoint.sh => scripts/prod/entrypoint_prod.sh (100%)
create mode 100644 tests/api_connexion/endpoints/test_import_error_endpoint.py
rename tests/api_connexion/endpoints/{test_import_errror_endpoint.py => test_version_endpoint.py} (56%)
create mode 100644 tests/api_connexion/schemas/test_error_schema.py
create mode 100644 tests/api_connexion/schemas/test_event_log_schema.py
create mode 100644 tests/api_connexion/schemas/test_pool_schemas.py
copy scripts/ci/in_container/run_ci_tests.sh => tests/api_connexion/schemas/test_version_schema.py (55%)
mode change 100755 => 100644
copy tests/providers/{zendesk/hooks => amazon/aws/transfers}/__init__.py (100%)
rename tests/providers/amazon/aws/{operators => transfers}/test_dynamodb_to_s3.py (92%)
rename tests/providers/amazon/aws/{operators => transfers}/test_gcs_to_s3.py (95%)
rename tests/providers/amazon/aws/{operators/test_google_api_to_s3_transfer.py => transfers/test_google_api_to_s3.py} (81%)
rename tests/providers/amazon/aws/{operators/test_google_api_to_s3_transfer_system.py => transfers/test_google_api_to_s3_system.py} (100%)
rename tests/providers/amazon/aws/{operators => transfers}/test_hive_to_dynamodb.py (93%)
rename tests/providers/amazon/aws/{operators => transfers}/test_imap_attachment_to_s3.py (92%)
rename tests/providers/amazon/aws/{operators => transfers}/test_imap_attachment_to_s3_system.py (100%)
rename tests/providers/amazon/aws/{operators => transfers}/test_mongo_to_s3.py (95%)
rename tests/providers/amazon/aws/{operators => transfers}/test_redshift_to_s3.py (95%)
rename tests/providers/amazon/aws/{operators => transfers}/test_s3_to_redshift.py (94%)
rename tests/providers/amazon/aws/{operators => transfers}/test_s3_to_redshift_system.py (100%)
rename tests/providers/amazon/aws/{operators => transfers}/test_s3_to_sftp.py (98%)
rename tests/providers/amazon/aws/{operators => transfers}/test_sftp_to_s3.py (98%)
copy tests/providers/{zendesk/hooks => apache/druid/transfers}/__init__.py (100%)
rename tests/providers/apache/druid/{operators => transfers}/test_hive_to_druid.py (97%)
copy tests/providers/{zendesk/hooks => apache/hive/transfers}/__init__.py (100%)
rename tests/providers/apache/hive/{operators => transfers}/test_hive_to_mysql.py (83%)
rename tests/providers/apache/hive/{operators => transfers}/test_hive_to_samba.py (87%)
rename tests/providers/apache/hive/{operators => transfers}/test_mssql_to_hive.py (83%)
rename tests/providers/apache/hive/{operators => transfers}/test_mysql_to_hive.py (98%)
rename tests/providers/apache/hive/{operators => transfers}/test_s3_to_hive.py (93%)
create mode 100644 tests/providers/apache/hive/transfers/test_vertica_to_hive.py
copy tests/providers/{zendesk/hooks => google/ads/transfers}/__init__.py (100%)
create mode 100644 tests/providers/google/ads/transfers/test_ads_to_gcs.py
rename tests/providers/google/cloud/operators/{test_cloud_build_operator_system.py => test_cloud_build_system.py} (100%)
delete mode 100644 tests/providers/google/cloud/operators/test_cloud_sql_operatorquery_system.py
copy tests/providers/google/cloud/operators/{test_sheets_to_gcs_system.py => test_datafusion_system.py} (65%)
copy tests/providers/{zendesk/hooks => google/cloud/transfers}/__init__.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_adls_to_gcs.py (94%)
rename tests/providers/google/cloud/{operators => transfers}/test_bigquery_to_bigquery.py (95%)
rename tests/providers/google/cloud/{operators => transfers}/test_bigquery_to_bigquery_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_bigquery_to_gcs.py (94%)
rename tests/providers/google/cloud/{operators => transfers}/test_bigquery_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_bigquery_to_mysql.py (93%)
rename tests/providers/google/cloud/{operators => transfers}/test_cassandra_to_gcs.py (93%)
rename tests/providers/google/cloud/{operators => transfers}/test_facebook_ads_to_gcs.py (93%)
rename tests/providers/google/cloud/{operators => transfers}/test_facebook_ads_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_gcs_to_bigquery.py (93%)
rename tests/providers/google/cloud/{operators => transfers}/test_gcs_to_bigquery_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_gcs_to_gcs.py (87%)
rename tests/providers/google/cloud/{operators/test_gcs_to_gcs_operator_system.py => transfers/test_gcs_to_gcs_system.py} (100%)
create mode 100644 tests/providers/google/cloud/transfers/test_gcs_to_local.py
rename tests/providers/google/cloud/{operators => transfers}/test_gcs_to_sftp.py (90%)
rename tests/providers/google/cloud/{operators => transfers}/test_gcs_to_sftp_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_local_to_gcs.py (95%)
rename tests/providers/google/cloud/{operators => transfers}/test_local_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_mssql_to_gcs.py (92%)
rename tests/providers/google/cloud/{operators => transfers}/test_mysql_to_gcs.py (91%)
rename tests/providers/google/cloud/{operators => transfers}/test_postgres_to_gcs.py (95%)
rename tests/providers/google/cloud/{operators => transfers}/test_postgres_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_presto_to_gcs.py (92%)
rename tests/providers/google/cloud/{operators => transfers}/test_presto_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_s3_to_gcs.py (93%)
rename tests/providers/google/cloud/{operators => transfers}/test_sftp_to_gcs.py (91%)
rename tests/providers/google/cloud/{operators => transfers}/test_sftp_to_gcs_system.py (100%)
rename tests/providers/google/cloud/{operators => transfers}/test_sheets_to_gcs.py (90%)
rename tests/providers/google/cloud/{operators => transfers}/test_sheets_to_gcs_system.py (100%)
copy tests/providers/{zendesk/hooks => google/suite/transfers}/__init__.py (100%)
rename tests/providers/google/suite/{operators => transfers}/test_gcs_to_gdrive.py (98%)
rename tests/providers/google/suite/{operators => transfers}/test_gcs_to_sheets.py (87%)
rename tests/providers/google/suite/{operators => transfers}/test_gcs_to_sheets_system.py (100%)
copy tests/providers/{zendesk/hooks => microsoft/azure/transfers}/__init__.py (100%)
rename tests/providers/microsoft/azure/{operators => transfers}/test_file_to_wasb.py (95%)
rename tests/providers/microsoft/azure/{operators/test_oracle_to_azure_data_lake_transfer.py => transfers/test_oracle_to_azure_data_lake.py} (93%)
copy tests/providers/{zendesk/hooks => mysql/transfers}/__init__.py (100%)
rename tests/providers/mysql/{operators => transfers}/test_presto_to_mysql.py (83%)
rename tests/providers/mysql/{operators => transfers}/test_s3_to_mysql.py (86%)
rename tests/providers/mysql/{operators => transfers}/test_vertica_to_mysql.py (58%)
rename tests/providers/oracle/{operators => transfers}/__init__.py (100%)
rename tests/providers/oracle/{operators/test_oracle_to_oracle_transfer.py => transfers/test_oracle_to_oracle.py} (94%)
copy tests/providers/{zendesk/hooks => snowflake/transfers}/__init__.py (100%)
rename tests/providers/snowflake/{operators => transfers}/test_s3_to_snowflake.py (95%)
rename tests/providers/snowflake/{operators => transfers}/test_snowflake_to_slack.py (94%)