You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by gi...@apache.org on 2021/04/04 02:39:21 UTC
[airflow] tag nightly-master updated (9ca49b6 -> 437850b)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 9ca49b6 (commit)
to 437850b (commit)
from 9ca49b6 Add query mutations to new UI (#15068)
add daa3f3c Add test to guard how DAG/Operator params work together (#15075)
add 78d9feb Removes references to directory that no longer exists (#15083)
add 375d26d Pass queue to BaseExecutor.execute_async like in airflow 1.10 (#14861)
add 398ac7b Make skip_exit_code configurable in BashOperator (#14963)
add 6b9b067 Assign TS type to fix linting (#15090)
add 3585b3c Remove 'conf' from search_columns in DagRun View (#15099)
add 1521b96 Fixed deprecated code example in Concepts doc (#15098)
add 390015d fix broken link in experimental API deprecation headers (#13547)
add b5aac82 Allow celery workers without gossip or mingle modes (#13880)
add 9c5c36d Added Pretius to the list of companies using Apache Airflow (#14367)
add 6c1bbd8 Fix file permission issue when running git-sync in the KubernetesExecutor mode. (#12441)
add 741a545 Parallelize build of documentation. (#15062)
add a4aee3f Add timeout to test jobs to prevent hanging docker containers (#15078)
add 88a7444 add timezone context in new ui (#15096)
add 7757fe3 Doc: Update kubernetes.rst to clarify Kubernetes Connection (#14954)
add 1d0c168 Move celery.default_queue to operators.default_queue to re-use between executors (#14699)
add 441b4ef Add workers extraVolumes to Kubernetes pod template for Helm Chart (#14743)
add ca4c4f3 Re-introduce dagrun.schedule_delay metric (#15105)
add 2578657 Fix deprecated import of `@task` in example DAG (#15111)
add 6e99ae0 Allow pathlib.Path in DagBag and various util fns (#15110)
add 1bec3b2 Fix doc link permission name (#14972)
add 0e024c4 Fixed all remaining code usage of old task import (#15118)
add a2d9929 Add REST API query sort and order to some endpoints (#14895)
add 7417f81 Run UI tests selectively when UI files have changed (#15009)
add b0e68eb Avoid scheduler/parser manager deadlock by using non-blocking IO (#15112)
add 116a8a0 Remove duplicate call to sync_metadata inside DagFileProcessorManager (#15121)
add a607002 Little clarification in Aws connection docs (#14290)
add 1d641d8 update remaining old import paths of operators (#15127)
add 3a80b70 Document Airflow's versioning and release policy for users (#14132)
add b7cd2df Fix bug in airflow.stats timing that broke dogstatsd mode (#15132)
add b672faa Don't run UI tests when python files have changed (#15138)
add ee2d6c5 Make app creation session- or class-wide when possible to make tests faster (#14878)
add da897c9 Taskgroup decorator (#15034)
add 6822665 Send region_name into parant class of AwsGlueJobHook (#14251)
add 486b764 Fix password masking in CLI action_logging (#15143)
add 535e1a8 Better handling of docker command (#15080)
add 4d1b2e9 Mark the test_scheduler_task_start_date as quarantined (#15086)
add 83d702c Fixes failing docs upload on master (#15148)
add 56d4311 fix year in CONTRIBUTING.rst (#15166)
add 9b76b94 A bunch of template_fields_renderers additions (#15130)
add eda538f AWS: Do not log info when SSM & SecretsManager secret not found (#15120)
add 2ac4638 Minor Helm Chart doc enhancements (#15124)
add 8cc8d11 Store inventories in GitHub Action cache (#15109)
add 730037a Add core-operators label to boring cyborg automation (#15167)
add 610aea9 Fix documentation error in `git_sync_template.yaml` (#13197)
add 3a38134 Deactivate trigger, refresh, and delete controls on dag detail view. (#14144)
add 77ab986 Add Guidelines about releasing Providers (#15168)
add 9ac1d0a Close issues that are pending response from the issue author (#15170)
add af2d11e Restore base lineage backend (#14146)
add 95ae24a Increase timeout for building the docs (#15157)
add 2e8aa0d Merges prepare/test provider packages into two jobs (#15152)
add de22fc7 Finish quarantine for test_should_force_kill_process (#15081)
add 932f8c2e9 Add documentation create/update community providers (#15061)
add 437850b Adds Blinker dependency which is missing after recent changes (#15182)
No new revisions were added by this update.
Summary of changes:
.coveragerc | 1 -
.dockerignore | 4 -
.github/boring-cyborg.yml | 12 +-
.github/workflows/ci.yml | 127 +++++--
.github/workflows/stale.yml | 15 +-
.gitignore | 4 +-
.pre-commit-config.yaml | 7 +
BREEZE.rst | 2 +-
CONTRIBUTING.rst | 2 +-
INTHEWILD.md | 1 +
PULL_REQUEST_WORKFLOW.rst | 2 +-
STATIC_CODE_CHECKS.rst | 2 +
TESTING.rst | 2 +-
UPDATING.md | 4 +
.../api_connexion/endpoints/connection_endpoint.py | 10 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 26 +-
.../api_connexion/endpoints/event_log_endpoint.py | 19 +-
.../endpoints/import_error_endpoint.py | 16 +-
airflow/api_connexion/endpoints/pool_endpoint.py | 10 +-
.../endpoints/role_and_permission_endpoint.py | 10 +-
airflow/api_connexion/endpoints/task_endpoint.py | 11 +-
airflow/api_connexion/endpoints/user_endpoint.py | 19 +-
.../api_connexion/endpoints/variable_endpoint.py | 17 +-
airflow/api_connexion/openapi/v1.yaml | 28 +-
airflow/api_connexion/parameters.py | 19 +
airflow/api_connexion/schemas/dag_run_schema.py | 1 +
airflow/cli/cli_parser.py | 16 +-
airflow/cli/commands/celery_command.py | 2 +
airflow/config_templates/config.yml | 14 +-
airflow/config_templates/default_airflow.cfg | 6 +-
airflow/config_templates/default_celery.py | 4 +-
airflow/config_templates/default_test.cfg | 1 -
airflow/configuration.py | 1 +
airflow/contrib/sensors/weekday_sensor.py | 4 +-
airflow/decorators/__init__.py | 1 +
airflow/decorators/task_group.py | 72 ++++
airflow/example_dags/example_bash_operator.py | 3 +-
.../example_dags/example_task_group_decorator.py | 73 ++++
airflow/example_dags/example_xcomargs.py | 3 +-
airflow/executors/base_executor.py | 4 +-
airflow/hooks/subprocess.py | 2 -
airflow/jobs/scheduler_job.py | 10 +-
.../git_sync_template.yaml | 7 +-
airflow/lineage/__init__.py | 22 ++
airflow/lineage/backend.py | 47 +++
airflow/models/baseoperator.py | 2 +-
airflow/models/dagbag.py | 20 +-
airflow/operators/bash.py | 23 +-
airflow/operators/generic_transfer.py | 1 +
airflow/operators/sql.py | 1 +
airflow/operators/trigger_dagrun.py | 1 +
airflow/providers/amazon/aws/hooks/glue.py | 2 -
airflow/providers/amazon/aws/operators/athena.py | 1 +
airflow/providers/amazon/aws/operators/batch.py | 1 +
airflow/providers/amazon/aws/operators/datasync.py | 7 +
airflow/providers/amazon/aws/operators/ecs.py | 1 +
.../amazon/aws/operators/emr_create_job_flow.py | 1 +
airflow/providers/amazon/aws/operators/glue.py | 1 +
.../amazon/aws/operators/sagemaker_base.py | 1 +
airflow/providers/amazon/aws/operators/sns.py | 1 +
.../amazon/aws/secrets/secrets_manager.py | 7 +-
.../amazon/aws/secrets/systems_manager.py | 6 +-
.../providers/amazon/aws/transfers/mongo_to_s3.py | 1 +
.../providers/amazon/aws/transfers/mysql_to_s3.py | 1 +
.../cloud/example_dags/example_datacatalog.py | 2 +-
airflow/providers/grpc/operators/grpc.py | 1 +
.../azure/operators/azure_container_instances.py | 1 +
.../azure/transfers/oracle_to_azure_data_lake.py | 1 +
.../providers/microsoft/winrm/operators/winrm.py | 1 +
.../providers/mysql/transfers/presto_to_mysql.py | 1 +
.../providers/mysql/transfers/vertica_to_mysql.py | 4 +
.../providers/oracle/transfers/oracle_to_oracle.py | 1 +
.../providers/singularity/operators/singularity.py | 1 +
.../snowflake/transfers/snowflake_to_slack.py | 1 +
airflow/providers/ssh/operators/ssh.py | 1 +
airflow/security/permissions.py | 2 +-
airflow/stats.py | 2 +-
airflow/ui/.neutrinorc.js | 2 +-
airflow/ui/src/App.tsx | 2 +-
airflow/ui/src/api/defaults.ts | 2 +
airflow/ui/src/api/index.ts | 8 +-
.../ui/src/components/AppContainer/AppHeader.tsx | 9 +-
airflow/ui/src/components/AppContainer/AppNav.tsx | 2 +-
airflow/ui/src/index.tsx | 2 +-
airflow/ui/src/interfaces/index.ts | 9 +
airflow/ui/src/providers/TimezoneProvider.tsx | 67 ++++
.../ui/src/{ => providers}/auth/AuthProvider.tsx | 1 +
.../ui/src/{ => providers}/auth/PrivateRoute.tsx | 4 +-
airflow/ui/src/{ => providers}/auth/context.ts | 6 +-
airflow/ui/src/views/Login.tsx | 2 +-
airflow/ui/test/Login.test.tsx | 8 +-
airflow/ui/test/Pipelines.test.tsx | 14 +-
airflow/utils/cli.py | 20 +-
airflow/utils/dag_processing.py | 48 ++-
airflow/utils/file.py | 13 +-
airflow/utils/task_group.py | 28 +-
airflow/www/api/experimental/endpoints.py | 2 +-
airflow/www/extensions/init_appbuilder_links.py | 10 +-
airflow/www/security.py | 2 +-
airflow/www/templates/airflow/dag.html | 6 +-
airflow/www/utils.py | 2 +
airflow/www/views.py | 30 +-
breeze | 2 +-
breeze-complete | 1 +
chart/files/pod-template-file.kubernetes-helm-yaml | 6 +
chart/tests/test_pod_template_file.py | 28 ++
chart/values.yaml | 3 +-
dev/PROJECT_GUIDELINES.md | 38 +-
.../enter_breeze_provider_package_tests.sh | 2 +-
.../connections/aws.rst | 7 +-
.../connections/kubernetes.rst | 2 +-
.../howto/create-update-providers.rst | 301 +++++++++++++++
docs/apache-airflow-providers/index.rst | 14 +-
docs/apache-airflow/concepts.rst | 11 +-
docs/apache-airflow/executor/celery.rst | 2 +-
docs/apache-airflow/howto/operator/bash.rst | 3 +-
docs/apache-airflow/index.rst | 1 +
docs/apache-airflow/lineage.rst | 21 ++
docs/apache-airflow/release-process.rst | 83 ++++
docs/build_docs.py | 417 ++++++++++++++++++---
docs/exts/docs_build/code_utils.py | 32 ++
docs/exts/docs_build/docs_builder.py | 310 +++++++++++----
docs/exts/docs_build/errors.py | 39 +-
docs/exts/docs_build/fetch_inventories.py | 11 +-
docs/exts/docs_build/github_action_utils.py | 1 +
docs/exts/docs_build/spelling_checks.py | 47 ++-
docs/helm-chart/index.rst | 4 +-
docs/helm-chart/quick-start.rst | 2 +-
docs/spelling_wordlist.txt | 1 +
pylintrc-tests | 4 +-
scripts/ci/docs/ci_docs.sh | 14 +-
.../ci_docs_prepare.sh} | 5 +-
scripts/ci/libraries/_build_images.sh | 42 +--
scripts/ci/libraries/_docker_engine_resources.sh | 12 +-
scripts/ci/libraries/_initialization.sh | 2 +
scripts/ci/libraries/_kind.sh | 2 +-
scripts/ci/libraries/_push_pull_remove_images.sh | 28 +-
scripts/ci/libraries/_runs.sh | 10 +-
scripts/ci/libraries/_start_end.sh | 2 +-
scripts/ci/libraries/_verbosity.sh | 6 +-
scripts/ci/libraries/_verify_image.sh | 8 +-
.../ci_install_and_test_provider_packages.sh | 2 +-
scripts/ci/selective_ci_checks.sh | 51 ++-
scripts/ci/static_checks/bats_tests.sh | 2 +-
scripts/ci/static_checks/check_license.sh | 2 +-
.../{refresh_pylint_todo.sh => eslint.sh} | 14 +-
scripts/ci/static_checks/flake8.sh | 4 +-
.../ci/static_checks/in_container_bats_tests.sh | 4 +-
scripts/ci/static_checks/lint_dockerfile.sh | 4 +-
scripts/ci/static_checks/mypy.sh | 2 +-
scripts/ci/static_checks/pylint.sh | 4 +-
scripts/ci/static_checks/refresh_pylint_todo.sh | 2 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 5 +-
scripts/ci/tools/ci_clear_tmp.sh | 2 +-
scripts/ci/tools/ci_fix_ownership.sh | 2 +-
scripts/ci/tools/ci_free_space_on_ci.sh | 2 +-
scripts/docker/compile_www_assets.sh | 10 +-
scripts/in_container/_in_container_utils.sh | 12 +-
.../in_container/run_anything.sh | 2 +
scripts/in_container/run_fix_ownership.sh | 4 +-
scripts/in_container/run_pylint.sh | 1 -
setup.cfg | 1 +
.../endpoints/test_connection_endpoint.py | 49 +++
.../endpoints/test_dag_run_endpoint.py | 93 +++++
.../endpoints/test_event_log_endpoint.py | 65 +++-
.../endpoints/test_import_error_endpoint.py | 58 +++
.../api_connexion/endpoints/test_pool_endpoint.py | 43 +++
.../endpoints/test_role_and_permission_endpoint.py | 8 +
.../api_connexion/endpoints/test_task_endpoint.py | 75 +++-
.../api_connexion/endpoints/test_user_endpoint.py | 9 +
.../endpoints/test_variable_endpoint.py | 10 +
.../schemas/test_role_and_permission_schema.py | 58 ++-
tests/api_connexion/test_error_handling.py | 39 +-
tests/cli/commands/test_celery_command.py | 4 +
tests/cli/commands/test_role_command.py | 24 +-
tests/cli/commands/test_user_command.py | 27 +-
airflow/www/blueprints.py => tests/cli/conftest.py | 17 +-
tests/conftest.py | 7 +
tests/core/test_core.py | 37 ++
tests/core/test_stats.py | 193 +++++-----
tests/jobs/test_scheduler_job.py | 20 +-
tests/lineage/test_lineage.py | 49 ++-
tests/models/test_dag.py | 2 +-
tests/models/test_dagparam.py | 2 +-
tests/operators/test_bash.py | 24 +-
tests/plugins/test_plugins_manager.py | 47 +--
.../common/auth_backend/test_google_openid.py | 56 +--
tests/test_utils/perf/dags/perf_dag_2.py | 2 +-
tests/test_utils/perf/dags/sql_perf_dag.py | 2 +-
tests/utils/test_cli_util.py | 10 +
tests/utils/test_dag_processing.py | 99 ++++-
tests/utils/test_process_utils.py | 12 +-
tests/utils/test_task_group.py | 373 +++++++++++++++++-
.../api/experimental}/conftest.py | 30 +-
.../www/api/experimental/test_dag_runs_endpoint.py | 24 +-
tests/www/api/experimental/test_endpoints.py | 60 ++-
tests/www/test_security.py | 2 +-
197 files changed, 3493 insertions(+), 853 deletions(-)
create mode 100644 airflow/decorators/task_group.py
create mode 100644 airflow/example_dags/example_task_group_decorator.py
create mode 100644 airflow/lineage/backend.py
create mode 100644 airflow/ui/src/providers/TimezoneProvider.tsx
rename airflow/ui/src/{ => providers}/auth/AuthProvider.tsx (98%)
rename airflow/ui/src/{ => providers}/auth/PrivateRoute.tsx (83%)
rename airflow/ui/src/{ => providers}/auth/context.ts (89%)
create mode 100644 docs/apache-airflow-providers/howto/create-update-providers.rst
create mode 100644 docs/apache-airflow/release-process.rst
copy scripts/ci/{build_airflow/ci_build_airflow_package.sh => docs/ci_docs_prepare.sh} (83%)
copy scripts/ci/static_checks/{refresh_pylint_todo.sh => eslint.sh} (80%)
copy airflow/api_connexion/__init__.py => scripts/in_container/run_anything.sh (96%)
mode change 100644 => 100755
copy airflow/www/blueprints.py => tests/cli/conftest.py (75%)
copy tests/{api_connexion => www/api/experimental}/conftest.py (64%)