You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by gi...@apache.org on 2020/10/12 01:56:55 UTC
[airflow] tag nightly-master updated (11eb649 -> d305876)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 11eb649 (commit)
to d305876 (commit)
from 11eb649 Fix to make y-axis of Tries chart visible (#10071)
add 27e637f Bugfix: Error in SSHOperator when command is None (#11361)
add 7541c88 Allways use Airlfow db in FAB (#11364)
add 7f674c6 Use only-if-needed upgrade strategy for PRs (#11363)
add 5605d10 Fix DagBag bug when a dag has invalid schedule_interval (#11344)
add 422b61a Adding ElastiCache Hook for creating, describing and deleting replication groups (#8701)
add 8baf657 Fix regression in DataflowTemplatedJobStartOperator (#11167)
add d2754ef Strict type check for Microsoft (#11359)
add fe0bf6e Reduce "start-up" time for tasks in CeleryExecutor (#11372)
add ff1a2aa Set start_date, end_date & duration for tasks failing without DagRun (#11358)
add eb5fea7 Replace nuke with useful information on error page (#11346)
add 49aad02 Users can specify sub-secrets and paths k8spodop (#11369)
add 29a145c Add capability of adding service account annotations to Helm Chart (#11387)
add e198077 Add pypirc initialization (#11386)
add 73b9163 Fully support running more than one scheduler concurrently (#10956)
add d752575 Revert "Revert "Adds --install-wheels flag to breeze command line (#11317)" (#11348)" (#11356)
add 7b0a2f5 Replaced basestring with str in the Exasol hook (#11360)
add 298052f [airflow/providers/cncf/kubernetes] correct hook methods name (#11008)
add 3164025 Fix airflow_local_settings.py showing up as directory (#10999)
add 39fc961 Fix case of JavaScript. (#10957)
add 6fe020e Add tests for Custom cluster policy (#11381)
add b7404b0 KubernetesPodOperator should retry log tailing in case of interruption (#11325)
add 8640fb6 fix tests (#11368)
add 0497390 Constraints and PIP packages can be installed from local sources (#11382)
add 401a579 Push and schedule duplicates are not cancelled. (#11397)
add 7507183 Remove redundant parentheses from Python files (#10967)
add a34f5ee Fixes automated upgrade to latest constraints. (#11399)
add 076fe88 Fixes cancelling of too many workflows. (#11403)
add 0620aaa Fix spelling (#11401)
add 7959df9 Fix spelling (#11404)
add f9dddd5 Workarounds "unknown blob" issue by introducing retries (#11411)
add 45d33db Add capability of customising PyPI sources (#11385)
add 9416bed Moving the test to quarantine. (#11405)
add bd204bb Optionally set null marker in csv exports in BaseSQLToGCSOperator (#11409)
add 4de8f85 Fixes SHA used for cancel-workflow-action (#11400)
add 5bc5994 Split tests to more sub-types (#11402)
add 686e0ee Fix incorrect typing, remove hardcoded argument values and improve code in AzureContainerInstancesOperator (#11408)
add ce2f19d Fix constraints generation script (#11412)
add b786327 Fix spelling in CeleryExecutor (#11407)
add 7e40a32 Add more info about dag_concurrency (#11300)
add 42a23d1 Update MySQLToS3Operator's s3_bucket to template_fields (#10778)
add c3e3405 Change prefix of AwsDynamoDB hook module (#11209)
add 1845cd1 Strict type check for google ads and cloud hooks (#11390)
add 9a01ce0 Mutual SSL added in PGBouncer configuration in the Chart (#11384)
add fb8b1e5 Merge Airflow and Backport Packages preparation instructions (#11310)
add 2c7e343 Fix syntax highlightling for concurrency in configurations doc (#11438)
add a155db1 Fix typo in airflow/utils/dag_processing.py (#11445)
add 369bbf0 Selective tests - depends on files changed in the commit. (#11417)
add eaaad31 Fix correct Sphinx return type for DagFileProcessorProcess.result (#11444)
add d8d13fa Use augmented assignment (#11449)
add d305876 Remove redundant None provided as default to dict.get() (#11448)
No new revisions were added by this update.
Summary of changes:
.dockerignore | 5 +
.github/workflows/build-images-workflow-run.yml | 55 +-
.github/workflows/ci.yml | 144 +-
.github/workflows/codeql-cancel.yml | 3 +-
.gitignore | 2 +
BREEZE.rst | 77 +-
CONTRIBUTING.rst | 4 +-
Dockerfile | 32 +-
Dockerfile.ci | 5 +-
IMAGES.rst | 25 +-
TESTING.rst | 57 +-
airflow/api/common/experimental/trigger_dag.py | 1 +
.../api_connexion/endpoints/connection_endpoint.py | 2 +-
airflow/api_connexion/endpoints/log_endpoint.py | 2 +-
airflow/api_connexion/exceptions.py | 2 +-
airflow/api_connexion/security.py | 2 +-
airflow/cli/cli_parser.py | 2 +-
airflow/cli/commands/scheduler_command.py | 1 -
airflow/config_templates/config.yml | 45 +
airflow/config_templates/default_airflow.cfg | 29 +
.../config_templates/default_webserver_config.py | 5 -
airflow/contrib/hooks/aws_dynamodb_hook.py | 6 +-
airflow/executors/base_executor.py | 33 +-
airflow/executors/celery_executor.py | 41 +
airflow/executors/celery_kubernetes_executor.py | 22 +-
airflow/jobs/scheduler_job.py | 1225 +++++-----
airflow/kubernetes/pod_launcher.py | 56 +-
airflow/kubernetes/secret.py | 17 +-
...606e2_add_scheduling_decision_to_dagrun_and_.py | 88 +
airflow/models/dag.py | 301 ++-
airflow/models/dagbag.py | 38 +-
airflow/models/dagrun.py | 124 +-
airflow/models/pool.py | 19 +-
airflow/models/serialized_dag.py | 34 +-
airflow/models/taskinstance.py | 35 +-
airflow/operators/python.py | 2 +-
.../cloud => amazon/aws}/ADDITIONAL_INFO.md | 11 +-
airflow/providers/amazon/aws/hooks/aws_dynamodb.py | 56 +-
.../aws/hooks/{aws_dynamodb.py => dynamodb.py} | 0
.../aws/hooks/elasticache_replication_group.py | 282 +++
.../providers/amazon/aws/hooks/step_function.py | 2 +-
.../amazon/aws/transfers/dynamodb_to_s3.py | 2 +-
.../amazon/aws/transfers/hive_to_dynamodb.py | 2 +-
.../providers/amazon/aws/transfers/mysql_to_s3.py | 7 +-
.../providers/apache/spark/hooks/spark_submit.py | 6 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 12 +-
.../cncf/kubernetes/operators/kubernetes_pod.py | 24 +-
.../cncf/kubernetes/operators/spark_kubernetes.py | 2 +-
.../cncf/kubernetes/sensors/spark_kubernetes.py | 2 +-
airflow/providers/exasol/hooks/exasol.py | 5 +-
airflow/providers/google/ads/hooks/ads.py | 5 +-
airflow/providers/google/ads/operators/ads.py | 4 +-
.../providers/google/ads/transfers/ads_to_gcs.py | 4 +-
.../_internal_client/secret_manager_client.py | 2 +-
airflow/providers/google/cloud/hooks/bigquery.py | 53 +-
.../providers/google/cloud/hooks/cloud_build.py | 6 +-
.../google/cloud/hooks/cloud_memorystore.py | 6 +-
airflow/providers/google/cloud/hooks/cloud_sql.py | 10 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 38 +-
airflow/providers/google/cloud/hooks/compute.py | 18 +-
.../providers/google/cloud/hooks/datacatalog.py | 56 +-
airflow/providers/google/cloud/hooks/dataflow.py | 79 +-
airflow/providers/google/cloud/hooks/datafusion.py | 4 +-
airflow/providers/google/cloud/hooks/dataproc.py | 8 +-
airflow/providers/google/cloud/hooks/datastore.py | 36 +-
airflow/providers/google/cloud/hooks/functions.py | 12 +-
airflow/providers/google/cloud/hooks/gcs.py | 113 +-
airflow/providers/google/cloud/hooks/gdm.py | 6 +-
.../providers/google/cloud/hooks/life_sciences.py | 8 +-
airflow/providers/google/cloud/hooks/mlengine.py | 26 +-
.../google/cloud/hooks/natural_language.py | 16 +-
airflow/providers/google/cloud/hooks/pubsub.py | 12 +-
.../providers/google/cloud/hooks/stackdriver.py | 4 +-
airflow/providers/google/cloud/hooks/tasks.py | 4 +-
airflow/providers/google/cloud/hooks/translate.py | 4 +-
airflow/providers/google/cloud/hooks/vision.py | 36 +-
.../providers/google/cloud/operators/dataflow.py | 13 +-
.../providers/google/cloud/operators/functions.py | 4 +-
.../providers/google/cloud/operators/mlengine.py | 6 +-
.../providers/google/cloud/transfers/sql_to_gcs.py | 6 +
.../google/suite/transfers/gcs_to_sheets.py | 2 +-
.../microsoft/azure/log/wasb_task_handler.py | 22 +-
.../microsoft/azure/operators/adls_list.py | 4 +-
airflow/providers/microsoft/azure/operators/adx.py | 6 +-
.../microsoft/azure/operators/azure_batch.py | 4 +-
.../azure/operators/azure_container_instances.py | 89 +-
.../microsoft/azure/operators/azure_cosmos.py | 3 +-
.../microsoft/azure/operators/wasb_delete_blob.py | 4 +-
.../microsoft/azure/secrets/azure_key_vault.py | 4 +-
.../microsoft/azure/sensors/azure_cosmos.py | 3 +-
airflow/providers/microsoft/azure/sensors/wasb.py | 8 +-
.../microsoft/azure/transfers/azure_blob_to_gcs.py | 2 +-
.../microsoft/azure/transfers/file_to_wasb.py | 4 +-
.../microsoft/azure/transfers/local_to_adls.py | 2 +-
.../azure/transfers/oracle_to_azure_data_lake.py | 4 +-
airflow/providers/microsoft/mssql/hooks/mssql.py | 18 +-
.../providers/microsoft/mssql/operators/mssql.py | 14 +-
airflow/providers/microsoft/winrm/hooks/winrm.py | 43 +-
.../providers/microsoft/winrm/operators/winrm.py | 28 +-
airflow/providers/oracle/hooks/oracle.py | 8 +-
airflow/providers/postgres/hooks/postgres.py | 2 +-
airflow/providers/redis/hooks/redis.py | 2 +-
airflow/providers/salesforce/hooks/salesforce.py | 2 +-
airflow/providers/sendgrid/utils/emailer.py | 2 +-
airflow/providers/sftp/hooks/sftp.py | 2 +-
airflow/providers/snowflake/hooks/snowflake.py | 2 +-
airflow/providers/ssh/operators/ssh.py | 2 +-
airflow/sensors/smart_sensor_operator.py | 12 +-
airflow/settings.py | 14 +
airflow/stats.py | 58 +
airflow/ti_deps/deps/not_previously_skipped_dep.py | 2 +-
airflow/utils/asciiart.py | 59 -
airflow/utils/callback_requests.py | 101 +
airflow/utils/dag_processing.py | 166 +-
airflow/utils/decorators.py | 2 +-
airflow/utils/sqlalchemy.py | 124 ++
airflow/www/app.py | 1 +
airflow/www/templates/airflow/traceback.html | 9 +-
airflow/www/views.py | 21 +-
backport_packages/README.md | 173 ++
backport_packages/remove_old_releases.py | 96 +
breeze | 102 +-
breeze-complete | 16 +-
chart/README.md | 4 +-
chart/templates/_helpers.yaml | 18 +
chart/templates/configmap.yaml | 2 +-
.../templates/pgbouncer/pgbouncer-deployment.yaml | 22 +
.../scheduler/scheduler-serviceaccount.yaml | 6 +
...ret.yaml => pgbouncer-certificates-secret.yaml} | 16 +-
.../webserver/webserver-serviceaccount.yaml | 6 +
chart/templates/workers/worker-serviceaccount.yaml | 8 +-
chart/values.schema.json | 39 +
chart/values.yaml | 17 +-
dev/README.md | 1332 ++++++++---
docker-context-files/README.md | 31 +
docs/logging-monitoring/metrics.rst | 15 +-
docs/operators-and-hooks-ref.rst | 8 +-
docs/production-deployment.rst | 79 +
docs/spelling_wordlist.txt | 1 +
docs/templates/layout.html | 2 +-
kubernetes_tests/test_kubernetes_pod_operator.py | 41 +-
scripts/ci/docker-compose/base.yml | 8 +-
scripts/ci/libraries/_build_images.sh | 22 +-
scripts/ci/libraries/_docker.env | 5 +
scripts/ci/libraries/_initialization.sh | 46 +-
scripts/ci/libraries/_parameters.sh | 28 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 67 +-
scripts/ci/selective_tests.sh | 350 +++
scripts/ci/testing/ci_run_airflow_testing.sh | 9 +-
.../ci/tools/ci_check_if_tests_should_be_run.sh | 56 -
scripts/ci/tools/ci_count_changed_files.sh | 63 -
scripts/in_container/entrypoint_ci.sh | 148 +-
scripts/in_container/run_ci_tests.sh | 57 +-
scripts/in_container/run_generate_constraints.sh | 5 +-
tests/api/client/test_local_client.py | 14 +-
tests/api/common/experimental/test_trigger_dag.py | 4 +
tests/api_connexion/schemas/test_error_schema.py | 2 +-
tests/bats/test_breeze_complete.bats | 8 +
tests/cli/commands/test_cheat_sheet_command.py | 2 +-
tests/cli/commands/test_connection_command.py | 22 +-
tests/cli/test_cli_parser.py | 8 +-
tests/{ => core}/test_config_templates.py | 0
tests/{ => core}/test_configuration.py | 0
tests/{ => core}/test_core.py | 0
tests/{ => core}/test_core_to_contrib.py | 0
tests/{ => core}/test_example_dags.py | 0
tests/{ => core}/test_example_dags_system.py | 0
.../test_impersonation_tests.py} | 0
tests/{ => core}/test_local_settings.py | 24 +
tests/{ => core}/test_logging_config.py | 0
tests/{ => core}/test_project_structure.py | 38 +-
tests/{ => core}/test_sentry.py | 0
tests/{ => core}/test_sqlalchemy_config.py | 3 +-
tests/{ => core}/test_stats.py | 18 +-
tests/dags/test_zip.zip | Bin 2204 -> 2977 bytes
tests/deprecated_classes.py | 2 +-
tests/executors/test_celery_executor.py | 22 +-
tests/jobs/test_backfill_job.py | 24 +-
tests/jobs/test_local_task_job.py | 3 +-
tests/jobs/test_scheduler_job.py | 2355 ++++++++++----------
tests/kubernetes/models/test_secret.py | 23 +
tests/kubernetes/test_pod_launcher.py | 42 +-
tests/models/test_dag.py | 566 +++--
tests/models/test_dagbag.py | 1 +
tests/models/test_dagrun.py | 131 +-
.../{test_aws_dynamodb.py => test_dynamodb.py} | 2 +-
.../hooks/test_elasticache_replication_group.py | 210 ++
tests/providers/amazon/aws/hooks/test_s3.py | 4 +-
.../amazon/aws/hooks/test_step_function.py | 4 +-
.../amazon/aws/transfers/test_hive_to_dynamodb.py | 2 +-
.../providers/google/cloud/hooks/test_dataflow.py | 47 +
.../providers/google/cloud/hooks/test_mlengine.py | 4 +-
.../google/cloud/operators/test_dataflow.py | 2 +
.../google/cloud/transfers/test_sql_to_gcs.py | 23 +
.../operators/test_azure_container_instances.py | 114 +
.../providers/salesforce/hooks/test_salesforce.py | 2 +-
tests/providers/ssh/operators/test_ssh.py | 1 +
tests/sensors/test_external_task_sensor.py | 2 +-
tests/serialization/test_dag_serialization.py | 6 +-
.../task/task_runner/test_standard_task_runner.py | 7 +-
tests/test_utils/mock_executor.py | 3 +-
tests/test_utils/mock_plugins.py | 2 +-
.../operators/postgres_local_executor.cfg | 2 +-
tests/test_utils/perf/perf_kit/__init__.py | 10 +-
tests/test_utils/perf/perf_kit/python.py | 2 +-
tests/test_utils/perf/perf_kit/sqlalchemy.py | 2 +-
tests/test_utils/perf/sql_queries.py | 2 +-
tests/ti_deps/deps/test_runnable_exec_date_dep.py | 19 +-
tests/utils/test_dag_processing.py | 119 +-
tests/utils/test_sqlalchemy.py | 14 +-
tests/www/test_views.py | 23 +-
211 files changed, 7652 insertions(+), 3594 deletions(-)
create mode 100644 airflow/migrations/versions/98271e7606e2_add_scheduling_decision_to_dagrun_and_.py
copy airflow/providers/{google/cloud => amazon/aws}/ADDITIONAL_INFO.md (61%)
copy airflow/providers/amazon/aws/hooks/{aws_dynamodb.py => dynamodb.py} (100%)
create mode 100644 airflow/providers/amazon/aws/hooks/elasticache_replication_group.py
delete mode 100644 airflow/utils/asciiart.py
create mode 100644 airflow/utils/callback_requests.py
create mode 100644 backport_packages/README.md
create mode 100644 backport_packages/remove_old_releases.py
copy chart/templates/secrets/{pgbouncer-stats-secret.yaml => pgbouncer-certificates-secret.yaml} (69%)
create mode 100644 docker-context-files/README.md
create mode 100755 scripts/ci/selective_tests.sh
delete mode 100755 scripts/ci/tools/ci_check_if_tests_should_be_run.sh
delete mode 100755 scripts/ci/tools/ci_count_changed_files.sh
rename tests/{ => core}/test_config_templates.py (100%)
rename tests/{ => core}/test_configuration.py (100%)
rename tests/{ => core}/test_core.py (100%)
rename tests/{ => core}/test_core_to_contrib.py (100%)
rename tests/{ => core}/test_example_dags.py (100%)
rename tests/{ => core}/test_example_dags_system.py (100%)
rename tests/{test_impersonation.py => core/test_impersonation_tests.py} (100%)
rename tests/{ => core}/test_local_settings.py (85%)
rename tests/{ => core}/test_logging_config.py (100%)
rename tests/{ => core}/test_project_structure.py (82%)
rename tests/{ => core}/test_sentry.py (100%)
rename tests/{ => core}/test_sqlalchemy_config.py (96%)
rename tests/{ => core}/test_stats.py (93%)
rename tests/providers/amazon/aws/hooks/{test_aws_dynamodb.py => test_dynamodb.py} (96%)
create mode 100644 tests/providers/amazon/aws/hooks/test_elasticache_replication_group.py