You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by je...@apache.org on 2021/09/07 22:44:15 UTC
[airflow] branch v2-2-test updated (fe34582 -> 79d8557)
This is an automated email from the ASF dual-hosted git repository.
jedcunningham pushed a change to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from fe34582 New google operator: SQLToGoogleSheetsOperator (#17887)
add 0791719 Add support for kinit options [-f|-F] and [-a|-A] (#17816)
add 683fbd4 Fix Clear task instances endpoint resets all DAG runs bug (#17961)
add fdbb798 Making spelling of "TaskFlow" consistent in docs (#17968)
add 6914091 Update max_tis_per_query to better render on the webpage (#17971)
add 9a0c10b deduplicate running jobs on BigQueryInsertJobOperator (#17496)
add 4d52207 Add missing menu access for dag dependencies and configurations pages (#17450)
add 617fed9 Chart: Make cleanup cronjob cmd/args configurable (#17970)
add 7c4d317 Delete unnecessary parameters in EKSPodOperator (#17960)
add d67133c Remove default_args pattern + added get_current_context() use for Core Airflow example DAGs (#16866)
add bec006e Fix constraint generation properly (#17964)
add 430976c Fix max_active_runs not allowing moving of queued dagruns to running (#17945)
add ab76581 Add DAG run endpoint for marking a dagrun success or failed(#17839)
add f727635 Fix blank dag dependencies view (#17990)
add ca4f99d Serialize the template_ext attribute to show it in UI (#17985)
add 1e1b3de Fix grammar in local.rst (#18001)
add 7b3a5f9 Hide variable import form if user lacks permission (#18000)
add 6db73a6 Reduce number of lines in "monitoring job" parallel group (#17995)
add 48316b9 Make EMR cluster visible to all users note (#17557)
add 31b15c9 Applied permissions to self._error_file (#15947)
add be7efb1 Require timetable class be registered via plugin (#17989)
add bff5806 Summarize test failures at the end of CI run (#18008)
add b6a962c Logs task launch exception in StandardTaskRunner (#17967)
add 2cace94 Implement API endpoint for DAG deletion (#17980)
add bcd81f2 Add Spark to the EMR cluster for the job flow examples (#17563)
add ace2374 Fix failed tests from run_as_user+error_file changes (#18013)
add a88115e Allow specifying multiple URLs via the CORS config option (#17941)
add 9c64419 Fix unexpected bug in exiting hook context manager (#18014)
add 0fe0b06 Adding missing init file in example_dags directory (#18019)
add decaaeb Fix spelling error causing tests to fail on main (#18021)
add 867e930 AwsBaseHook make `client_type` & `resource_type` optional params for `get_client_type` & `get_resource_type` (#17987)
add 21348c1 fix(CloudSqlProxyRunner): don't query connections from Airflow DB (#18006)
add fd5d657 Update release notes for 3 extra providers released (#18018)
add b486a0e Adds capability of Warnings for incompatible community providers (#18020)
add a0a05ff Adds example showing the ES_hook (#17944)
add 16a17f7 Fix building docs in `main` builds (#18035)
add 6898a2f Add Python2 to installed packages (#18037)
add fe7efca Opmitise LoggingMixin.log (#17280)
add 1be3ef6 Improve the description of how to handle dynamic task generation (#17963)
add bfad233 Fix providers tests in main branch with eager upgrades (#18040)
add 952fa57 Suppress stat stderr output when we have fallback (#18044)
add 0c2f71e Revert "Opmitise LoggingMixin.log (#17280)" (#18047)
add 022b4e0 Fix building documentation broken by upgrade of dnspython (#18046)
add 944dcfb Change TaskInstance and TaskReschedule PK from execution_date to run_id (#17719)
add 873d4d1 Optimize imports of Providers Manager (#18052)
add 64d2f54 Do not fail KubernetesPodOperator tasks if log reading fails (#17649)
add 1bfb572 Build CI images for the merge result of a PR, not the tip of the PR (#18060)
add 0dba2e0 Revert "Build CI images for the merge result of a PR, not the tip of the PR (#18060)" (#18063)
add 5d2b056 Fix Neo4jHook to get the query response (#18007)
add 77d50a1 Add missing "be" in documentation (#18059)
add 79d8557 Added Jagex to the list of companies using Apache Airflow (#18058)
No new revisions were added by this update.
Summary of changes:
CI.rst | 8 +-
Dockerfile | 4 +
Dockerfile.ci | 5 +
INTHEWILD.md | 1 +
UPDATING.md | 10 +
airflow/api/common/experimental/mark_tasks.py | 5 +-
airflow/api_connexion/endpoints/dag_endpoint.py | 25 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 31 +-
airflow/api_connexion/endpoints/log_endpoint.py | 18 +-
.../endpoints/task_instance_endpoint.py | 41 +-
airflow/api_connexion/openapi/v1.yaml | 65 +-
airflow/api_connexion/schemas/dag_run_schema.py | 10 +-
.../api_connexion/schemas/task_instance_schema.py | 10 +-
airflow/cli/commands/dag_command.py | 4 +-
airflow/cli/commands/kubernetes_command.py | 6 +-
airflow/cli/commands/task_command.py | 106 ++--
airflow/config_templates/config.yml | 34 +-
airflow/config_templates/default_airflow.cfg | 22 +-
airflow/configuration.py | 5 +-
airflow/dag_processing/processor.py | 16 +-
airflow/example_dags/example_bash_operator.py | 5 -
.../example_branch_datetime_operator.py | 6 -
.../example_branch_day_of_week_operator.py | 5 -
airflow/example_dags/example_branch_operator.py | 5 -
.../example_branch_python_dop_operator_3.py | 7 +-
airflow/example_dags/example_dag_decorator.py | 4 +-
.../example_dags/example_kubernetes_executor.py | 5 -
.../example_kubernetes_executor_config.py | 5 -
.../example_passing_params_via_test_command.py | 3 -
airflow/example_dags/example_python_operator.py | 5 -
.../example_dags/example_short_circuit_operator.py | 5 -
airflow/example_dags/example_skip_dag.py | 6 +-
.../example_dags/example_trigger_controller_dag.py | 1 -
airflow/example_dags/example_trigger_target_dag.py | 9 +-
airflow/example_dags/example_xcom.py | 43 +-
airflow/example_dags/example_xcomargs.py | 2 -
airflow/example_dags/tutorial_taskflow_api_etl.py | 10 +-
.../tutorial_taskflow_api_etl_virtualenv.py | 14 +-
airflow/exceptions.py | 5 +
airflow/executors/kubernetes_executor.py | 41 +-
airflow/jobs/backfill_job.py | 46 +-
airflow/jobs/local_task_job.py | 2 +-
airflow/jobs/scheduler_job.py | 128 +---
airflow/kubernetes/kubernetes_helper_functions.py | 26 +-
airflow/kubernetes/pod_generator.py | 40 +-
.../7b2661a43ba3_taskinstance_keyed_to_dagrun.py | 287 +++++++++
airflow/models/baseoperator.py | 36 +-
airflow/models/dag.py | 41 +-
airflow/models/dagrun.py | 56 +-
airflow/models/skipmixin.py | 85 ++-
airflow/models/taskinstance.py | 292 +++++----
airflow/models/taskreschedule.py | 26 +-
airflow/plugins_manager.py | 27 +
.../alibaba/cloud/example_dags}/__init__.py | 0
.../example_emr_job_flow_automatic_steps.py | 1 +
.../example_emr_job_flow_manual_steps.py | 1 +
airflow/providers/amazon/aws/hooks/base_aws.py | 23 +-
airflow/providers/amazon/aws/hooks/eks.py | 40 +-
airflow/providers/amazon/aws/operators/eks.py | 40 +-
airflow/providers/celery/CHANGELOG.rst | 11 +
airflow/providers/celery/provider.yaml | 2 +
.../cncf/kubernetes/utils/pod_launcher.py | 27 +-
.../elasticsearch/example_dags}/__init__.py | 0
.../example_dags/example_elasticsearch_query.py | 59 ++
airflow/providers/google/cloud/hooks/cloud_sql.py | 9 +-
.../providers/google/cloud/hooks/datacatalog.py | 2 +-
.../providers/google/cloud/operators/bigquery.py | 16 +-
airflow/providers/hashicorp/CHANGELOG.rst | 7 +
airflow/providers/microsoft/psrp/CHANGELOG.rst | 11 +
airflow/providers/microsoft/psrp/hooks/psrp.py | 2 +-
airflow/providers/microsoft/psrp/provider.yaml | 1 +
airflow/providers/neo4j/hooks/neo4j.py | 11 +-
airflow/providers_manager.py | 31 +-
airflow/security/kerberos.py | 12 +
airflow/sensors/base.py | 2 +-
airflow/sensors/smart_sensor.py | 12 +-
airflow/sentry.py | 8 +-
airflow/serialization/serialized_objects.py | 52 +-
airflow/task/task_runner/base_task_runner.py | 8 +
airflow/task/task_runner/standard_task_runner.py | 5 +
airflow/ti_deps/dep_context.py | 22 +-
airflow/ti_deps/deps/dagrun_exists_dep.py | 28 +-
airflow/ti_deps/deps/dagrun_id_dep.py | 11 +-
airflow/ti_deps/deps/not_previously_skipped_dep.py | 2 +-
airflow/ti_deps/deps/runnable_exec_date_dep.py | 13 +-
airflow/ti_deps/deps/trigger_rule_dep.py | 2 +-
airflow/utils/callback_requests.py | 7 +-
airflow/www/auth.py | 2 +
airflow/www/decorators.py | 1 +
airflow/www/extensions/init_views.py | 9 +-
airflow/www/security.py | 2 +
.../www/templates/airflow/dag_dependencies.html | 2 +-
airflow/www/templates/airflow/variable_list.html | 2 +
airflow/www/utils.py | 21 +-
airflow/www/views.py | 209 ++++---
breeze | 2 +-
chart/templates/cleanup/cleanup-cronjob.yaml | 8 +-
chart/tests/test_cleanup_pods.py | 47 ++
chart/values.schema.json | 26 +
chart/values.yaml | 5 +
.../operators/emr.rst | 4 +
docs/apache-airflow-providers-celery/commits.rst | 3 +-
docs/apache-airflow-providers-celery/index.rst | 9 +-
.../index.rst | 6 +
.../apache-airflow-providers-hashicorp/commits.rst | 5 +-
.../commits.rst | 11 +
.../index.rst | 2 +-
docs/apache-airflow/best-practices.rst | 150 ++++-
docs/apache-airflow/concepts/scheduler.rst | 11 -
docs/apache-airflow/concepts/taskflow.rst | 2 +-
docs/apache-airflow/howto/custom-operator.rst | 2 +-
docs/apache-airflow/logging-monitoring/metrics.rst | 1 -
docs/apache-airflow/migrations-ref.rst | 4 +-
docs/apache-airflow/security/api.rst | 4 +-
docs/apache-airflow/security/kerberos.rst | 22 +
docs/apache-airflow/start/local.rst | 2 +-
docs/apache-airflow/tutorial_taskflow_api.rst | 20 +-
docs/build_docs.py | 1 +
docs/conf.py | 6 +
docs/exts/exampleinclude.py | 6 +
docs/spelling_wordlist.txt | 3 +
kubernetes_tests/test_kubernetes_pod_operator.py | 6 +-
.../test_kubernetes_pod_operator_backcompat.py | 6 +-
scripts/ci/libraries/_docker_engine_resources.sh | 18 +-
scripts/ci/libraries/_parallel.sh | 21 +-
scripts/ci/selective_ci_checks.sh | 21 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 4 +
.../ci_run_single_airflow_test_in_docker.sh | 4 +-
scripts/ci/testing/summarize_junit_failures.py | 125 ++++
scripts/in_container/run_generate_constraints.sh | 48 +-
setup.py | 6 +
tests/api/common/experimental/test_delete_dag.py | 36 +-
tests/api/common/experimental/test_mark_tasks.py | 41 +-
tests/api_connexion/conftest.py | 8 +
tests/api_connexion/endpoints/test_dag_endpoint.py | 2 +
.../endpoints/test_dag_run_endpoint.py | 88 +++
.../endpoints/test_event_log_endpoint.py | 194 +++---
tests/api_connexion/endpoints/test_log_endpoint.py | 54 +-
.../endpoints/test_task_instance_endpoint.py | 105 +---
tests/api_connexion/schemas/test_dag_run_schema.py | 9 +-
.../api_connexion/schemas/test_event_log_schema.py | 74 +--
.../schemas/test_task_instance_schema.py | 60 +-
tests/cli/commands/test_dag_command.py | 4 +-
tests/cli/commands/test_task_command.py | 195 +-----
tests/conftest.py | 93 ++-
tests/core/test_core.py | 79 +--
tests/core/test_providers_manager.py | 24 +-
tests/core/test_sentry.py | 55 +-
tests/dag_processing/test_manager.py | 34 +-
tests/dag_processing/test_processor.py | 89 +--
tests/executors/test_base_executor.py | 87 ++-
tests/executors/test_celery_executor.py | 21 +-
tests/executors/test_kubernetes_executor.py | 34 +-
tests/jobs/test_backfill_job.py | 73 +--
tests/jobs/test_local_task_job.py | 58 +-
tests/jobs/test_scheduler_job.py | 524 +++++-----------
tests/jobs/test_triggerer_job.py | 22 +-
tests/lineage/test_lineage.py | 40 +-
tests/models/test_baseoperator.py | 10 +
tests/models/test_cleartasks.py | 190 +++---
tests/models/test_dag.py | 96 +--
tests/models/test_dagrun.py | 48 +-
tests/models/test_renderedtifields.py | 64 +-
tests/models/test_skipmixin.py | 3 +-
tests/models/test_taskinstance.py | 685 ++++++++++-----------
tests/models/test_trigger.py | 27 +-
tests/operators/test_latest_only_operator.py | 3 +-
tests/operators/test_python.py | 115 +---
tests/operators/test_subdag_operator.py | 88 +--
tests/providers/amazon/aws/hooks/test_base_aws.py | 80 ++-
tests/providers/amazon/aws/hooks/test_eks.py | 36 ++
.../amazon/aws/log/test_cloudwatch_task_handler.py | 10 +-
.../amazon/aws/log/test_s3_task_handler.py | 10 +-
.../providers/amazon/aws/operators/test_athena.py | 12 +-
.../amazon/aws/operators/test_datasync.py | 60 +-
.../aws/operators/test_dms_describe_tasks.py | 9 +-
tests/providers/amazon/aws/operators/test_eks.py | 30 +
.../amazon/aws/operators/test_emr_add_steps.py | 61 +-
.../aws/operators/test_emr_create_job_flow.py | 12 +-
tests/providers/amazon/aws/operators/test_sqs.py | 9 +-
tests/providers/amazon/aws/sensors/test_s3_key.py | 7 +-
tests/providers/amazon/aws/sensors/test_sqs.py | 39 +-
.../amazon/aws/transfers/test_mongo_to_s3.py | 7 +-
.../amazon/aws/transfers/test_s3_to_sftp.py | 13 +-
.../amazon/aws/transfers/test_sftp_to_s3.py | 6 +-
.../providers/apache/druid/operators/test_druid.py | 98 ++-
tests/providers/apache/hive/operators/test_hive.py | 6 +-
.../apache/kylin/operators/test_kylin_cube.py | 5 +-
.../apache/spark/operators/test_spark_submit.py | 5 +-
.../kubernetes/operators/test_kubernetes_pod.py | 8 +-
.../cncf/kubernetes/utils/test_pod_launcher.py | 25 +-
.../elasticsearch/log/test_es_task_handler.py | 156 ++---
.../google/cloud/hooks/test_datacatalog.py | 3 +-
.../google/cloud/log/test_gcs_task_handler.py | 75 ++-
.../cloud/log/test_stackdriver_task_handler.py | 165 ++---
.../google/cloud/operators/test_bigquery.py | 152 +++--
.../google/cloud/operators/test_cloud_build.py | 48 +-
.../test_cloud_storage_transfer_service.py | 208 +++----
.../google/cloud/operators/test_compute.py | 78 ++-
.../google/cloud/operators/test_dataproc.py | 424 ++++++-------
.../google/cloud/operators/test_mlengine.py | 44 +-
tests/providers/http/sensors/test_http.py | 56 +-
.../microsoft/azure/log/test_wasb_task_handler.py | 51 +-
.../microsoft/azure/operators/test_adx.py | 24 +-
tests/providers/microsoft/psrp/hooks/test_psrp.py | 2 +
tests/providers/neo4j/hooks/test_neo4j.py | 109 +++-
tests/providers/neo4j/operators/test_neo4j.py | 31 +-
tests/providers/qubole/operators/test_qubole.py | 87 ++-
tests/providers/sftp/operators/test_sftp.py | 417 ++++++-------
tests/providers/ssh/operators/test_ssh.py | 124 ++--
tests/security/test_kerberos.py | 43 ++
tests/sensors/test_base.py | 179 +++---
tests/sensors/test_external_task_sensor.py | 189 +++---
tests/sensors/test_smart_sensor_operator.py | 11 +-
tests/serialization/test_dag_serialization.py | 206 +++++--
tests/task/task_runner/test_task_runner.py | 5 +-
tests/test_utils/mock_executor.py | 5 +-
tests/test_utils/mock_operators.py | 12 +-
tests/test_utils/timetables.py | 6 +
tests/ti_deps/deps/test_dagrun_exists_dep.py | 3 +-
tests/ti_deps/deps/test_dagrun_id_dep.py | 7 -
.../deps/test_not_previously_skipped_dep.py | 177 +++---
tests/ti_deps/deps/test_ready_to_reschedule_dep.py | 4 +-
tests/ti_deps/deps/test_runnable_exec_date_dep.py | 70 ++-
tests/ti_deps/deps/test_trigger_rule_dep.py | 160 +++--
tests/utils/log/test_log_reader.py | 108 ++--
tests/utils/test_dot_renderer.py | 87 ++-
tests/utils/test_helpers.py | 40 +-
tests/utils/test_log_handlers.py | 61 +-
.../test_task_handler_with_custom_formatter.py | 14 +-
tests/www/test_security.py | 1 +
tests/www/views/test_views_dagrun.py | 16 +-
tests/www/views/test_views_extra_links.py | 34 +-
tests/www/views/test_views_log.py | 75 ++-
tests/www/views/test_views_rendered.py | 109 ++--
tests/www/views/test_views_variable.py | 35 +-
236 files changed, 5957 insertions(+), 4985 deletions(-)
create mode 100644 airflow/migrations/versions/7b2661a43ba3_taskinstance_keyed_to_dagrun.py
copy {tests/www/api/experimental => airflow/providers/alibaba/cloud/example_dags}/__init__.py (100%)
copy {tests/providers/zendesk/hooks => airflow/providers/elasticsearch/example_dags}/__init__.py (100%)
create mode 100644 airflow/providers/elasticsearch/example_dags/example_elasticsearch_query.py
create mode 100755 scripts/ci/testing/summarize_junit_failures.py