You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/10/22 09:07:33 UTC
[airflow] tag nightly-master updated (3970bfa -> eba1d91)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 3970bfa (commit)
to eba1d91 (commit)
from 3970bfa Pod template file uses custom custom env variable (#11480)
add dd1c07b Optimizes CI builds heavily with selective checks (#11656)
add 0373089 Brings back fixup to CI optimisation (#11671)
add b5d1ab9 Introduced deterministic order in connection export (#11670)
add 9a90ebe Bats tests should be much faster now for pre-commits. (#11662)
add cd0ed7d Fix Start Date tooltip on DAGs page (#10637)
add 3ee6186 Switch PagerdutyHook from pypd to use pdpyras instead (#11151)
add 26ae8e9 StreamLogWriter: Provide (no-op) close method. (#10884)
add 318f18f Removes separate cancel codeql workflow (#11672)
add 9c4f0a9 Add missing doc dependency in CI (#11678)
add e3a0839 Security scans are also selective now (#11674)
add 1543923 Add Kerberos Auth for PrestoHook (#10488)
add c568c88 Fixing problem with missing output in pre-commits in some cases (#11684)
add 72b644b Add missing template in pip package - python_virtualenv_script.jinja2 (#11677)
add 080a470 Improve example DAGs data by diversifying "tags" value (#11665)
add 782b8b3 Simplify import check in CLI (#11668)
add 4e352f9 Add datumo.io to Airflow family (#11692)
add cb7c67d Fix example DAGs in pip packages (#11687)
add 349b081 Add D200 pydocstyle check (#11688)
add 5df2842 Improve legibility with greater contrast ratio in footer and pagination (#11690)
add 3391c90 fix pod launcher rolebinding in helm chart (#11675)
add 3a45f1f Extract Kubernetes command to separate file (#11669)
add f468fc5 Fix documentation for PythonVirtualenvOperator (#11700)
add 3caa539 Update TIs with a proper lock (#11683)
add 7ef0b3c Revert "Refactor celery worker command (#11336)" (#11698)
add 282b084 Make table header text nowrap (#11689)
add 883eadf Consistent use images in Helm Chart (#11701)
add ffc9aeb Better file extension for Helm template (#11702)
add 53e6062 Enforce strict rules for yamllint (#11709)
add 2bfc53b Fix doc errors in google provider files. (#11713)
add 172820d Fix case of GitHub (#11398)
add 4fb5c01 Check response status in slack webhook hook. (#11620)
add b946b44 Remove unused value in Helm Chart - podMutation (#11703)
add 68ea94a Also cancel duplicated Build Image runs for master pushes (#11655)
add a1822919 Fix case of JavaScript (#11718)
add 950c16d Retry requests in case of error in Google ML Engine Hook (#11712)
add 1fb3c28 Add support for setting ciphers for SFTPHook (#11720)
add 6c3f435 Refactor "loading" state of DAGs view to remove visual jank (#11725)
add ae791e1 Fix formatting errors introduced in #11720 (#11733)
add b9d677c Add type hints to aws provider (#11531)
add 9150330 Add Google Cloud Memorystore Memcached Operators (#10121)
add 1da8379 Fix static checks after merging #10121 (#11737)
add eba1d91 Fixes ROVIDERS -> PROVIDERS typo in Dockerfile (#11738)
No new revisions were added by this update.
Summary of changes:
.github/ISSUE_TEMPLATE/config.yml | 1 +
.github/workflows/build-images-workflow-run.yml | 79 +--
.github/workflows/ci.yml | 169 ++++--
.github/workflows/codeql-analysis.yml | 53 +-
.github/workflows/codeql-cancel.yml | 22 -
.github/workflows/delete_old_artifacts.yml | 9 +-
.github/workflows/repo_sync.yml | 2 +-
.github/workflows/scheduled_quarantined.yml | 2 +-
.pre-commit-config.yaml | 86 ++-
BREEZE.rst | 8 +-
CI.rst | 168 +++++-
CONTRIBUTING.rst | 6 +-
Dockerfile | 4 +-
IMAGES.rst | 6 +-
INTHEWILD.md | 1 +
MANIFEST.in | 13 +-
airflow/api/client/__init__.py | 8 +-
airflow/api/common/experimental/get_lineage.py | 8 +-
airflow/api_connexion/endpoints/config_endpoint.py | 4 +-
.../api_connexion/endpoints/connection_endpoint.py | 20 +-
airflow/api_connexion/endpoints/dag_endpoint.py | 16 +-
.../api_connexion/endpoints/dag_run_endpoint.py | 20 +-
.../api_connexion/endpoints/dag_source_endpoint.py | 4 +-
.../api_connexion/endpoints/event_log_endpoint.py | 8 +-
.../api_connexion/endpoints/extra_link_endpoint.py | 4 +-
airflow/api_connexion/endpoints/health_endpoint.py | 4 +-
.../endpoints/import_error_endpoint.py | 8 +-
airflow/api_connexion/endpoints/log_endpoint.py | 4 +-
airflow/api_connexion/endpoints/pool_endpoint.py | 20 +-
airflow/api_connexion/endpoints/task_endpoint.py | 8 +-
.../endpoints/task_instance_endpoint.py | 12 +-
.../api_connexion/endpoints/variable_endpoint.py | 20 +-
airflow/api_connexion/endpoints/xcom_endpoint.py | 8 +-
airflow/api_connexion/openapi/v1.yaml | 4 +-
airflow/api_connexion/schemas/common_schema.py | 4 +-
airflow/api_connexion/schemas/connection_schema.py | 8 +-
airflow/api_connexion/schemas/dag_run_schema.py | 8 +-
airflow/api_connexion/schemas/pool_schema.py | 16 +-
airflow/api_connexion/schemas/xcom_schema.py | 8 +-
airflow/api_connexion/security.py | 4 +-
airflow/cli/cli_parser.py | 23 +-
airflow/cli/commands/celery_command.py | 9 +-
airflow/cli/commands/connection_command.py | 2 +-
airflow/cli/commands/dag_command.py | 51 +-
airflow/cli/commands/info_command.py | 4 +-
airflow/cli/commands/kubernetes_command.py | 63 +++
airflow/cli/commands/pool_command.py | 4 +-
airflow/cli/commands/task_command.py | 4 +-
airflow/cli/commands/webserver_command.py | 8 +-
airflow/configuration.py | 4 +-
airflow/contrib/hooks/aws_hook.py | 4 +-
airflow/contrib/hooks/gcp_api_base_hook.py | 3 +-
airflow/contrib/hooks/gcp_container_hook.py | 4 +-
airflow/contrib/hooks/gcp_dataflow_hook.py | 4 +-
airflow/contrib/hooks/gcp_function_hook.py | 3 +-
airflow/contrib/hooks/gcp_kms_hook.py | 4 +-
airflow/contrib/hooks/gcp_natural_language_hook.py | 4 +-
airflow/contrib/hooks/gcp_spanner_hook.py | 4 +-
airflow/contrib/hooks/gcp_speech_to_text_hook.py | 4 +-
airflow/contrib/hooks/gcp_sql_hook.py | 8 +-
airflow/contrib/hooks/gcp_tasks_hook.py | 4 +-
airflow/contrib/hooks/gcp_text_to_speech_hook.py | 4 +-
airflow/contrib/hooks/gcp_transfer_hook.py | 3 +-
airflow/contrib/hooks/gcp_translate_hook.py | 4 +-
.../contrib/hooks/gcp_video_intelligence_hook.py | 4 +-
airflow/contrib/hooks/gcp_vision_hook.py | 4 +-
airflow/contrib/hooks/gcs_hook.py | 8 +-
airflow/contrib/hooks/sftp_hook.py | 4 +-
airflow/contrib/operators/awsbatch_operator.py | 6 +-
.../azure_container_instances_operator.py | 4 +-
airflow/contrib/operators/cassandra_to_gcs.py | 4 +-
airflow/contrib/operators/ecs_operator.py | 4 +-
airflow/contrib/operators/file_to_gcs.py | 4 +-
.../contrib/operators/gcp_container_operator.py | 4 +-
.../operators/gcp_natural_language_operator.py | 4 +-
.../operators/gcp_speech_to_text_operator.py | 4 +-
airflow/contrib/operators/gcp_sql_operator.py | 7 +-
airflow/contrib/operators/gcp_tasks_operator.py | 4 +-
.../operators/gcp_text_to_speech_operator.py | 4 +-
.../contrib/operators/gcp_translate_operator.py | 4 +-
.../operators/gcp_video_intelligence_operator.py | 4 +-
airflow/contrib/operators/gcp_vision_operator.py | 4 +-
airflow/contrib/operators/gcs_acl_operator.py | 4 +-
airflow/contrib/operators/gcs_delete_operator.py | 4 +-
airflow/contrib/operators/gcs_download_operator.py | 4 +-
airflow/contrib/operators/gcs_list_operator.py | 4 +-
airflow/contrib/operators/gcs_operator.py | 4 +-
airflow/contrib/operators/gcs_to_bq.py | 4 +-
airflow/contrib/operators/gcs_to_gcs.py | 4 +-
airflow/contrib/operators/gcs_to_s3.py | 7 +-
airflow/contrib/operators/sftp_operator.py | 4 +-
airflow/contrib/sensors/sftp_sensor.py | 4 +-
airflow/contrib/utils/gcp_field_sanitizer.py | 4 +-
airflow/contrib/utils/gcp_field_validator.py | 4 +-
airflow/contrib/utils/log/__init__.py | 4 +-
.../log/task_handler_with_custom_formatter.py | 4 +-
airflow/contrib/utils/weekday.py | 4 +-
airflow/dag/base_dag.py | 8 +-
airflow/example_dags/example_bash_operator.py | 2 +-
airflow/example_dags/example_branch_operator.py | 2 +-
airflow/example_dags/example_complex.py | 2 +-
.../example_external_task_marker_dag.py | 4 +-
.../example_dags/example_kubernetes_executor.py | 2 +-
.../example_kubernetes_executor_config.py | 2 +-
airflow/example_dags/example_latest_only.py | 2 +-
.../example_latest_only_with_trigger.py | 2 +-
airflow/executors/base_executor.py | 20 +-
airflow/executors/celery_executor.py | 7 +-
airflow/executors/celery_kubernetes_executor.py | 20 +-
airflow/executors/dask_executor.py | 4 +-
airflow/executors/debug_executor.py | 8 +-
airflow/executors/executor_loader.py | 8 +-
airflow/executors/local_executor.py | 16 +-
airflow/hooks/dbapi_hook.py | 19 +-
airflow/jobs/base_job.py | 16 +-
airflow/jobs/local_task_job.py | 4 +-
airflow/jobs/scheduler_job.py | 27 +-
airflow/kubernetes/k8s_model.py | 4 +-
airflow/kubernetes/pod_generator.py | 4 +-
airflow/kubernetes/pod_generator_deprecated.py | 4 +-
airflow/kubernetes/pod_runtime_info_env.py | 8 +-
airflow/kubernetes/refresh_config.py | 4 +-
airflow/kubernetes/secret.py | 4 +-
.../basic_template.yaml | 2 +-
airflow/lineage/__init__.py | 20 +-
airflow/lineage/entities.py | 20 +-
.../004c1210f153_increase_queue_name_size_limit.py | 4 +-
...0_fix_mssql_exec_date_rendered_task_instance.py | 6 +-
...a59344a4_make_taskinstance_pool_not_nullable.py | 8 +-
...867_change_datetime_to_datetime2_6_on_mssql_.py | 8 +-
airflow/models/baseoperator.py | 57 +-
airflow/models/dag.py | 40 +-
airflow/models/dagbag.py | 12 +-
airflow/models/dagrun.py | 4 +-
airflow/models/log.py | 4 +-
airflow/models/pool.py | 4 +-
airflow/models/renderedtifields.py | 4 +-
airflow/models/skipmixin.py | 4 +-
airflow/models/taskfail.py | 4 +-
airflow/models/taskinstance.py | 28 +-
airflow/models/taskmixin.py | 24 +-
airflow/models/taskreschedule.py | 4 +-
airflow/models/variable.py | 12 +-
airflow/models/xcom.py | 4 +-
airflow/models/xcom_arg.py | 12 +-
airflow/operators/dagrun_operator.py | 4 +-
airflow/operators/latest_only_operator.py | 4 +-
airflow/operators/python.py | 2 +-
airflow/operators/sql.py | 4 +-
airflow/operators/subdag_operator.py | 8 +-
airflow/plugins_manager.py | 4 +-
airflow/providers/amazon/aws/hooks/athena.py | 4 +-
.../providers/amazon/aws/hooks/cloud_formation.py | 8 +-
airflow/providers/amazon/aws/hooks/datasync.py | 4 +-
airflow/providers/amazon/aws/hooks/dynamodb.py | 8 +-
.../aws/hooks/elasticache_replication_group.py | 45 +-
airflow/providers/amazon/aws/hooks/glacier.py | 4 +-
airflow/providers/amazon/aws/hooks/glue.py | 12 +-
airflow/providers/amazon/aws/hooks/glue_catalog.py | 4 +-
airflow/providers/amazon/aws/hooks/kinesis.py | 8 +-
.../providers/amazon/aws/hooks/lambda_function.py | 8 +-
airflow/providers/amazon/aws/hooks/redshift.py | 4 +-
airflow/providers/amazon/aws/hooks/s3.py | 4 +-
airflow/providers/amazon/aws/hooks/sagemaker.py | 70 ++-
airflow/providers/amazon/aws/hooks/ses.py | 5 +-
airflow/providers/amazon/aws/hooks/sns.py | 4 +-
airflow/providers/amazon/aws/hooks/sqs.py | 4 +-
.../amazon/aws/log/cloudwatch_task_handler.py | 12 +-
.../providers/amazon/aws/log/s3_task_handler.py | 19 +-
airflow/providers/amazon/aws/operators/athena.py | 8 +-
airflow/providers/amazon/aws/operators/batch.py | 39 +-
.../amazon/aws/operators/cloud_formation.py | 13 +-
airflow/providers/amazon/aws/operators/datasync.py | 101 ++--
airflow/providers/amazon/aws/operators/ecs.py | 77 +--
airflow/providers/amazon/aws/operators/glue.py | 29 +-
.../providers/amazon/aws/operators/s3_bucket.py | 8 +-
.../amazon/aws/operators/s3_copy_object.py | 15 +-
.../amazon/aws/operators/s3_delete_objects.py | 12 +-
airflow/providers/amazon/aws/operators/s3_list.py | 13 +-
.../amazon/aws/operators/sagemaker_base.py | 11 +-
.../amazon/aws/operators/sagemaker_endpoint.py | 17 +-
.../aws/operators/sagemaker_endpoint_config.py | 4 +-
.../amazon/aws/operators/sagemaker_model.py | 4 +-
.../amazon/aws/operators/sagemaker_processing.py | 19 +-
.../amazon/aws/operators/sagemaker_training.py | 15 +-
.../amazon/aws/operators/sagemaker_transform.py | 17 +-
.../amazon/aws/operators/sagemaker_tuning.py | 13 +-
airflow/providers/amazon/aws/operators/sns.py | 11 +-
airflow/providers/amazon/aws/operators/sqs.py | 11 +-
.../step_function_get_execution_output.py | 10 +-
.../aws/operators/step_function_start_execution.py | 4 +-
.../amazon/aws/secrets/secrets_manager.py | 8 +-
.../amazon/aws/secrets/systems_manager.py | 8 +-
.../amazon/aws/sensors/cloud_formation.py | 25 +-
airflow/providers/amazon/aws/sensors/emr_base.py | 18 +-
airflow/providers/amazon/aws/sensors/glacier.py | 4 +-
airflow/providers/amazon/aws/sensors/glue.py | 2 +-
.../amazon/aws/sensors/glue_catalog_partition.py | 31 +-
airflow/providers/amazon/aws/sensors/redshift.py | 20 +-
airflow/providers/amazon/aws/sensors/s3_key.py | 23 +-
.../amazon/aws/sensors/s3_keys_unchanged.py | 4 +-
airflow/providers/amazon/aws/sensors/s3_prefix.py | 20 +-
.../providers/amazon/aws/sensors/sagemaker_base.py | 24 +-
.../amazon/aws/sensors/sagemaker_training.py | 7 +-
.../amazon/aws/sensors/sagemaker_transform.py | 2 +-
.../amazon/aws/sensors/sagemaker_tuning.py | 2 +-
airflow/providers/amazon/aws/sensors/sqs.py | 23 +-
.../amazon/aws/sensors/step_function_execution.py | 20 +-
.../providers/amazon/aws/transfers/gcs_to_s3.py | 4 +-
.../amazon/aws/transfers/google_api_to_s3.py | 4 +-
.../amazon/aws/transfers/hive_to_dynamodb.py | 4 +-
.../amazon/aws/transfers/imap_attachment_to_s3.py | 4 +-
.../providers/amazon/aws/transfers/mongo_to_s3.py | 4 +-
.../providers/amazon/aws/transfers/mysql_to_s3.py | 4 +-
.../amazon/aws/transfers/redshift_to_s3.py | 4 +-
.../providers/apache/cassandra/hooks/cassandra.py | 16 +-
airflow/providers/apache/druid/hooks/druid.py | 12 +-
.../apache/druid/operators/druid_check.py | 4 +-
.../apache/druid/transfers/hive_to_druid.py | 4 +-
airflow/providers/apache/hdfs/hooks/hdfs.py | 4 +-
airflow/providers/apache/hdfs/sensors/hdfs.py | 12 +-
airflow/providers/apache/hdfs/sensors/web_hdfs.py | 4 +-
airflow/providers/apache/hive/hooks/hive.py | 33 +-
airflow/providers/apache/hive/operators/hive.py | 8 +-
.../providers/apache/hive/operators/hive_stats.py | 4 +-
.../apache/hive/transfers/hive_to_mysql.py | 4 +-
.../apache/hive/transfers/hive_to_samba.py | 4 +-
.../apache/hive/transfers/mssql_to_hive.py | 8 +-
.../apache/hive/transfers/mysql_to_hive.py | 8 +-
.../providers/apache/hive/transfers/s3_to_hive.py | 4 +-
.../apache/hive/transfers/vertica_to_hive.py | 4 +-
airflow/providers/apache/livy/hooks/livy.py | 8 +-
airflow/providers/apache/livy/operators/livy.py | 8 +-
airflow/providers/apache/livy/sensors/livy.py | 4 +-
airflow/providers/apache/pig/hooks/pig.py | 4 +-
airflow/providers/apache/pinot/hooks/pinot.py | 12 +-
airflow/providers/apache/spark/hooks/spark_jdbc.py | 4 +-
.../apache/spark/hooks/spark_jdbc_script.py | 8 +-
airflow/providers/apache/spark/hooks/spark_sql.py | 4 +-
.../providers/apache/spark/hooks/spark_submit.py | 4 +-
.../providers/apache/spark/operators/spark_jdbc.py | 4 +-
.../providers/apache/spark/operators/spark_sql.py | 4 +-
.../apache/spark/operators/spark_submit.py | 4 +-
airflow/providers/apache/sqoop/hooks/sqoop.py | 8 +-
airflow/providers/apache/sqoop/operators/sqoop.py | 9 +-
.../example_spark_kubernetes_spark_pi.yaml | 2 +-
.../providers/cncf/kubernetes/hooks/kubernetes.py | 8 +-
.../cncf/kubernetes/operators/kubernetes_pod.py | 4 +-
airflow/providers/databricks/hooks/databricks.py | 4 +-
.../providers/databricks/operators/databricks.py | 12 +-
airflow/providers/datadog/hooks/datadog.py | 4 +-
.../dingding/example_dags/example_dingding.py | 2 +-
airflow/providers/dingding/hooks/dingding.py | 8 +-
airflow/providers/discord/hooks/discord_webhook.py | 4 +-
.../providers/discord/operators/discord_webhook.py | 4 +-
airflow/providers/docker/operators/docker.py | 8 +-
.../providers/elasticsearch/hooks/elasticsearch.py | 4 +-
airflow/providers/facebook/ads/hooks/ads.py | 8 +-
airflow/providers/ftp/hooks/ftp.py | 12 +-
airflow/providers/google/ads/hooks/ads.py | 15 +-
airflow/providers/google/ads/operators/ads.py | 4 +-
.../_internal_client/secret_manager_client.py | 4 +-
.../cloud/example_dags/example_cloud_build.yaml | 2 +-
.../example_dags/example_cloud_memorystore.py | 131 ++++-
airflow/providers/google/cloud/hooks/automl.py | 8 +-
airflow/providers/google/cloud/hooks/bigquery.py | 16 +-
.../providers/google/cloud/hooks/bigquery_dts.py | 8 +-
airflow/providers/google/cloud/hooks/bigtable.py | 4 +-
.../google/cloud/hooks/cloud_memorystore.py | 458 ++++++++++++++-
airflow/providers/google/cloud/hooks/cloud_sql.py | 24 +-
.../cloud/hooks/cloud_storage_transfer_service.py | 12 +-
airflow/providers/google/cloud/hooks/compute.py | 8 +-
.../providers/google/cloud/hooks/datacatalog.py | 4 +-
airflow/providers/google/cloud/hooks/dataflow.py | 20 +-
airflow/providers/google/cloud/hooks/datafusion.py | 16 +-
airflow/providers/google/cloud/hooks/dataprep.py | 4 +-
airflow/providers/google/cloud/hooks/dataproc.py | 20 +-
airflow/providers/google/cloud/hooks/datastore.py | 5 +-
airflow/providers/google/cloud/hooks/functions.py | 4 +-
airflow/providers/google/cloud/hooks/gcs.py | 8 +-
airflow/providers/google/cloud/hooks/kms.py | 4 +-
.../google/cloud/hooks/kubernetes_engine.py | 4 +-
airflow/providers/google/cloud/hooks/mlengine.py | 51 +-
.../google/cloud/hooks/natural_language.py | 4 +-
airflow/providers/google/cloud/hooks/pubsub.py | 8 +-
airflow/providers/google/cloud/hooks/spanner.py | 4 +-
.../providers/google/cloud/hooks/speech_to_text.py | 4 +-
.../providers/google/cloud/hooks/stackdriver.py | 8 +-
.../providers/google/cloud/hooks/text_to_speech.py | 4 +-
airflow/providers/google/cloud/hooks/translate.py | 4 +-
.../google/cloud/hooks/video_intelligence.py | 4 +-
airflow/providers/google/cloud/hooks/vision.py | 8 +-
.../providers/google/cloud/log/gcs_task_handler.py | 4 +-
.../google/cloud/log/stackdriver_task_handler.py | 4 +-
airflow/providers/google/cloud/operators/automl.py | 4 +-
.../providers/google/cloud/operators/bigquery.py | 16 +-
.../google/cloud/operators/bigquery_dts.py | 4 +-
.../providers/google/cloud/operators/bigtable.py | 8 +-
.../google/cloud/operators/cloud_memorystore.py | 616 ++++++++++++++++++++-
.../providers/google/cloud/operators/cloud_sql.py | 4 +-
.../operators/cloud_storage_transfer_service.py | 20 +-
.../providers/google/cloud/operators/compute.py | 8 +-
.../providers/google/cloud/operators/dataflow.py | 4 +-
.../providers/google/cloud/operators/datafusion.py | 4 +-
.../providers/google/cloud/operators/dataprep.py | 4 +-
.../providers/google/cloud/operators/dataproc.py | 16 +-
.../providers/google/cloud/operators/datastore.py | 4 +-
.../providers/google/cloud/operators/functions.py | 4 +-
airflow/providers/google/cloud/operators/gcs.py | 4 +-
.../google/cloud/operators/kubernetes_engine.py | 4 +-
.../providers/google/cloud/operators/mlengine.py | 8 +-
.../google/cloud/operators/natural_language.py | 4 +-
airflow/providers/google/cloud/operators/pubsub.py | 4 +-
.../providers/google/cloud/operators/spanner.py | 4 +-
.../google/cloud/operators/speech_to_text.py | 4 +-
.../google/cloud/operators/text_to_speech.py | 4 +-
.../providers/google/cloud/operators/translate.py | 4 +-
.../google/cloud/operators/translate_speech.py | 4 +-
.../google/cloud/operators/video_intelligence.py | 4 +-
airflow/providers/google/cloud/operators/vision.py | 4 +-
.../google/cloud/secrets/secret_manager.py | 4 +-
airflow/providers/google/cloud/sensors/bigquery.py | 4 +-
.../providers/google/cloud/sensors/bigquery_dts.py | 4 +-
airflow/providers/google/cloud/sensors/bigtable.py | 4 +-
.../sensors/cloud_storage_transfer_service.py | 4 +-
airflow/providers/google/cloud/sensors/dataproc.py | 4 +-
airflow/providers/google/cloud/sensors/gcs.py | 4 +-
airflow/providers/google/cloud/sensors/pubsub.py | 4 +-
.../google/cloud/transfers/bigquery_to_bigquery.py | 4 +-
.../google/cloud/transfers/bigquery_to_gcs.py | 4 +-
.../google/cloud/transfers/bigquery_to_mysql.py | 4 +-
.../google/cloud/transfers/cassandra_to_gcs.py | 40 +-
.../google/cloud/transfers/facebook_ads_to_gcs.py | 4 +-
.../google/cloud/transfers/gcs_to_bigquery.py | 4 +-
.../providers/google/cloud/transfers/gcs_to_gcs.py | 4 +-
.../google/cloud/transfers/gcs_to_sftp.py | 8 +-
.../google/cloud/transfers/local_to_gcs.py | 8 +-
.../google/cloud/transfers/mssql_to_gcs.py | 4 +-
.../google/cloud/transfers/mysql_to_gcs.py | 8 +-
.../google/cloud/transfers/postgres_to_gcs.py | 8 +-
.../google/cloud/transfers/presto_to_gcs.py | 12 +-
.../google/cloud/transfers/sftp_to_gcs.py | 8 +-
.../providers/google/cloud/transfers/sql_to_gcs.py | 9 +-
.../google/cloud/utils/credentials_provider.py | 4 +-
.../google/cloud/utils/field_sanitizer.py | 4 +-
.../google/cloud/utils/mlengine_operator_utils.py | 45 +-
.../cloud/utils/mlengine_prediction_summary.py | 154 +++---
.../providers/google/common/hooks/base_google.py | 16 +-
.../providers/google/common/hooks/discovery_api.py | 4 +-
.../google/common/utils/id_token_credentials.py | 12 +-
.../providers/google/config_templates/config.yml | 2 +-
.../google/marketing_platform/hooks/analytics.py | 12 +-
.../marketing_platform/hooks/campaign_manager.py | 12 +-
.../marketing_platform/hooks/display_video.py | 21 +-
.../google/marketing_platform/hooks/search_ads.py | 12 +-
.../marketing_platform/operators/analytics.py | 4 +-
.../operators/campaign_manager.py | 4 +-
.../marketing_platform/operators/display_video.py | 4 +-
.../marketing_platform/operators/search_ads.py | 4 +-
.../marketing_platform/sensors/campaign_manager.py | 4 +-
.../marketing_platform/sensors/display_video.py | 4 +-
.../marketing_platform/sensors/search_ads.py | 4 +-
airflow/providers/google/suite/hooks/sheets.py | 4 +-
.../google/suite/transfers/gcs_to_gdrive.py | 4 +-
airflow/providers/grpc/hooks/grpc.py | 4 +-
airflow/providers/hashicorp/secrets/vault.py | 4 +-
airflow/providers/imap/sensors/imap_attachment.py | 4 +-
airflow/providers/jenkins/hooks/jenkins.py | 8 +-
.../jenkins/operators/jenkins_job_trigger.py | 4 +-
.../providers/microsoft/azure/hooks/azure_batch.py | 4 +-
.../azure/hooks/azure_container_volume.py | 8 +-
.../microsoft/azure/hooks/azure_cosmos.py | 56 +-
.../microsoft/azure/log/wasb_task_handler.py | 8 +-
.../microsoft/azure/operators/azure_batch.py | 5 +-
.../microsoft/azure/secrets/azure_key_vault.py | 4 +-
airflow/providers/microsoft/mssql/hooks/mssql.py | 12 +-
airflow/providers/mongo/hooks/mongo.py | 4 +-
airflow/providers/mysql/hooks/mysql.py | 16 +-
airflow/providers/odbc/hooks/odbc.py | 36 +-
airflow/providers/openfaas/hooks/openfaas.py | 16 +-
airflow/providers/opsgenie/hooks/opsgenie_alert.py | 4 +-
.../providers/opsgenie/operators/opsgenie_alert.py | 4 +-
airflow/providers/oracle/hooks/oracle.py | 4 +-
airflow/providers/pagerduty/README.md | 2 +-
airflow/providers/pagerduty/hooks/pagerduty.py | 72 ++-
.../papermill}/example_dags/input_notebook.ipynb | 0
airflow/providers/papermill/operators/papermill.py | 4 +-
airflow/providers/postgres/hooks/postgres.py | 12 +-
airflow/providers/presto/hooks/presto.py | 65 ++-
airflow/providers/qubole/hooks/qubole_check.py | 28 +-
airflow/providers/qubole/operators/qubole_check.py | 8 +-
airflow/providers/qubole/sensors/qubole.py | 4 +-
airflow/providers/redis/hooks/redis.py | 8 +-
airflow/providers/redis/sensors/redis_key.py | 4 +-
airflow/providers/salesforce/hooks/salesforce.py | 4 +-
.../salesforce/sensors/tableau_job_status.py | 4 +-
airflow/providers/samba/hooks/samba.py | 8 +-
airflow/providers/segment/hooks/segment.py | 4 +-
airflow/providers/sendgrid/utils/emailer.py | 4 +-
airflow/providers/sftp/hooks/sftp.py | 17 +-
airflow/providers/sftp/operators/sftp.py | 8 +-
airflow/providers/sftp/sensors/sftp.py | 4 +-
airflow/providers/slack/hooks/slack_webhook.py | 6 +-
airflow/providers/slack/operators/slack_webhook.py | 4 +-
airflow/providers/snowflake/hooks/snowflake.py | 8 +-
airflow/providers/snowflake/operators/snowflake.py | 4 +-
.../snowflake/transfers/s3_to_snowflake.py | 4 +-
airflow/providers/sqlite/hooks/sqlite.py | 8 +-
airflow/providers/ssh/operators/ssh.py | 4 +-
airflow/providers/vertica/hooks/vertica.py | 8 +-
airflow/providers/yandex/hooks/yandex.py | 4 +-
airflow/providers/zendesk/hooks/zendesk.py | 4 +-
airflow/secrets/base_secrets.py | 4 +-
airflow/secrets/environment_variables.py | 8 +-
airflow/secrets/local_filesystem.py | 12 +-
airflow/secrets/metastore.py | 8 +-
airflow/sensors/base_sensor_operator.py | 4 +-
airflow/sensors/external_task_sensor.py | 4 +-
airflow/sensors/smart_sensor_operator.py | 33 +-
airflow/sentry.py | 40 +-
airflow/serialization/json_schema.py | 8 +-
airflow/serialization/serialized_objects.py | 33 +-
airflow/settings.py | 4 +-
airflow/task/task_runner/base_task_runner.py | 12 +-
airflow/task/task_runner/standard_task_runner.py | 4 +-
airflow/ti_deps/deps/__init__.py | 4 +-
airflow/ti_deps/deps/dag_ti_slots_available_dep.py | 4 +-
airflow/ti_deps/deps/dag_unpaused_dep.py | 4 +-
airflow/ti_deps/deps/dagrun_exists_dep.py | 4 +-
airflow/ti_deps/deps/dagrun_id_dep.py | 4 +-
.../ti_deps/deps/exec_date_after_start_date_dep.py | 4 +-
airflow/ti_deps/deps/not_in_retry_period_dep.py | 4 +-
airflow/ti_deps/deps/pool_slots_available_dep.py | 4 +-
airflow/ti_deps/deps/ready_to_reschedule.py | 4 +-
airflow/ti_deps/deps/runnable_exec_date_dep.py | 4 +-
airflow/ti_deps/deps/task_concurrency_dep.py | 4 +-
airflow/utils/cli.py | 12 +-
airflow/utils/code_utils.py | 4 +-
airflow/utils/compression.py | 4 +-
airflow/utils/dag_cycle_tester.py | 8 +-
airflow/utils/dag_processing.py | 88 +--
airflow/utils/dates.py | 8 +-
airflow/utils/db.py | 24 +-
airflow/utils/dot_renderer.py | 4 +-
airflow/utils/email.py | 12 +-
airflow/utils/file.py | 4 +-
airflow/utils/helpers.py | 32 +-
airflow/utils/json.py | 8 +-
airflow/utils/log/cloudwatch_task_handler.py | 4 +-
airflow/utils/log/colored_log.py | 4 +-
airflow/utils/log/es_task_handler.py | 4 +-
airflow/utils/log/gcs_task_handler.py | 4 +-
airflow/utils/log/json_formatter.py | 8 +-
airflow/utils/log/logging_mixin.py | 39 +-
airflow/utils/log/s3_task_handler.py | 4 +-
airflow/utils/log/stackdriver_task_handler.py | 4 +-
.../log/task_handler_with_custom_formatter.py | 8 +-
airflow/utils/log/wasb_task_handler.py | 4 +-
airflow/utils/mixins.py | 4 +-
airflow/utils/net.py | 4 +-
airflow/utils/operator_resources.py | 24 +-
airflow/utils/orm_event_handlers.py | 4 +-
airflow/utils/platform.py | 11 +-
airflow/utils/process_utils.py | 4 +-
airflow/utils/python_virtualenv.py | 4 +-
airflow/utils/session.py | 4 +-
airflow/utils/sqlalchemy.py | 8 +-
airflow/utils/state.py | 12 +-
airflow/utils/strings.py | 12 +-
airflow/utils/task_group.py | 56 +-
airflow/utils/timeout.py | 8 +-
airflow/utils/trigger_rule.py | 12 +-
airflow/utils/types.py | 4 +-
airflow/utils/weekday.py | 8 +-
airflow/utils/weight_rule.py | 12 +-
airflow/www/api/experimental/endpoints.py | 4 +-
airflow/www/decorators.py | 12 +-
airflow/www/extensions/init_appbuilder.py | 4 +-
airflow/www/forms.py | 12 +-
airflow/www/security.py | 28 +-
airflow/www/static/css/bootstrap-theme.css | 1 +
airflow/www/static/css/main.css | 1 +
airflow/www/templates/airflow/dags.html | 51 +-
airflow/www/templates/airflow/graph.html | 8 +-
airflow/www/templates/airflow/master.html | 3 +-
airflow/www/templates/airflow/traceback.html | 2 +-
.../loading_dots.html} | 11 +-
airflow/www/utils.py | 32 +-
airflow/www/views.py | 23 +-
breeze | 22 +-
chart/Chart.yaml | 1 +
chart/dockerfiles/statsd-exporter/mappings.yml | 2 +-
...yaml => pod-template-file.kubernetes-helm-yaml} | 0
chart/requirements.yaml | 2 +-
chart/templates/_helpers.yaml | 7 +-
chart/templates/configmap.yaml | 2 +-
.../templates/pgbouncer/pgbouncer-deployment.yaml | 5 +
chart/templates/rbac/pod-launcher-rolebinding.yaml | 4 +
chart/templates/redis/redis-statefulset.yaml | 6 +-
.../templates/scheduler/scheduler-deployment.yaml | 1 +
chart/templates/statsd/statsd-deployment.yaml | 6 +-
chart/templates/workers/worker-deployment.yaml | 1 +
chart/tests/git-sync-scheduler_test.yaml | 8 +-
chart/tests/migrate-database-job_test.yaml | 2 +-
chart/tests/pod-template-file_test.yaml | 8 +-
chart/values.schema.json | 40 +-
chart/values.yaml | 78 ++-
codecov.yml | 10 +-
dags/test_dag.py | 4 +-
dev/remove_artifacts.sh | 2 +-
docs/build_docs.py | 54 +-
docs/exts/exampleinclude.py | 4 +-
docs/exts/redirects.py | 4 +-
docs/exts/sphinx_script_update.py | 4 +-
.../google/cloud/cloud_memorystore_memcached.rst | 158 ++++++
docs/operators-and-hooks-ref.rst | 8 +-
docs/production-deployment.rst | 2 +-
metastore_browser/hive_metastore.py | 36 +-
scripts/ci/docker-compose/integration-kerberos.yml | 46 +-
scripts/ci/docker-compose/integration-presto.yml | 22 +-
scripts/ci/dockerfiles/krb5-kdc-server/Dockerfile | 93 ++--
.../dockerfiles/krb5-kdc-server/build_and_push.sh | 2 +-
.../ci/dockerfiles/krb5-kdc-server/entrypoint.sh | 17 +-
scripts/ci/dockerfiles/krb5-kdc-server/kadm5.acl | 8 +
.../krb5-kdc-server/{krb-conf/server => }/kdc.conf | 36 +-
.../krb5-kdc-server/krb-conf/server/kadm5.acl | 18 -
.../{krb-conf/client => }/krb5.conf | 42 +-
.../ci/dockerfiles/krb5-kdc-server/start_kdc.sh | 46 --
.../dockerfiles/krb5-kdc-server/supervisord.conf | 18 +-
.../krb5-kdc-server/utils/create_admin.sh} | 34 +-
.../krb5-kdc-server/utils/create_client.sh} | 37 +-
.../krb5-kdc-server/utils/create_service.sh | 62 +++
scripts/ci/dockerfiles/presto/Dockerfile | 22 +
.../{krb5-kdc-server => presto}/build_and_push.sh | 12 +-
scripts/ci/dockerfiles/presto/entrypoint.sh | 119 ++++
scripts/ci/kubernetes/ci_run_helm_testing.sh | 2 +-
scripts/ci/libraries/_all_libs.sh | 2 +
scripts/ci/libraries/_build_images.sh | 2 +-
scripts/ci/libraries/_kerberos.sh | 57 ++
scripts/ci/libraries/_push_pull_remove_images.sh | 2 +-
scripts/ci/libraries/_start_end.sh | 12 +-
.../{selective_tests.sh => selective_ci_checks.sh} | 308 ++++++++---
scripts/ci/static_checks/bats_tests.sh | 43 +-
...static_checks.sh => run_basic_static_checks.sh} | 23 +-
scripts/ci/testing/ci_run_airflow_testing.sh | 4 +
scripts/in_container/_in_container_utils.sh | 34 --
scripts/in_container/check_environment.sh | 32 +-
scripts/in_container/entrypoint_ci.sh | 18 -
scripts/in_container/krb5/krb5.conf | 36 --
setup.py | 3 +-
tests/bats/{ => breeze}/test_breeze_complete.bats | 54 +-
tests/bats/{ => breeze}/test_breeze_params.bats | 2 +-
.../libraries/_local_mounts.bats} | 2 +-
tests/cli/commands/test_celery_command.py | 19 +-
tests/cli/commands/test_connection_command.py | 5 +-
tests/cli/commands/test_dag_command.py | 12 -
...rsion_command.py => test_kubernetes_command.py} | 25 +-
tests/conftest.py | 2 +-
tests/kubernetes/basic_pod.yaml | 2 +-
tests/models/test_dagbag.py | 4 +-
tests/models/test_serialized_dag.py | 4 +-
tests/providers/amazon/aws/operators/test_batch.py | 6 +-
tests/providers/dingding/hooks/test_dingding.py | 2 +-
.../google/cloud/hooks/test_cloud_memorystore.py | 188 ++++++-
.../providers/google/cloud/hooks/test_mlengine.py | 58 +-
.../cloud/operators/test_cloud_memorystore.py | 147 +++++
.../operators/test_cloud_memorystore_system.py | 8 +-
tests/providers/pagerduty/hooks/test_pagerduty.py | 66 +--
tests/providers/presto/hooks/test_presto.py | 138 ++++-
tests/providers/sftp/hooks/test_sftp.py | 8 +
tests/providers/snowflake/__init__.py | 8 +
tests/utils/test_logging_mixin.py | 7 +
.../api/experimental/test_kerberos_endpoints.py | 3 -
573 files changed, 5086 insertions(+), 3934 deletions(-)
delete mode 100644 .github/workflows/codeql-cancel.yml
create mode 100644 airflow/cli/commands/kubernetes_command.py
rename airflow/{ => providers/papermill}/example_dags/input_notebook.ipynb (100%)
copy airflow/www/templates/{airflow/conn_edit.html => appbuilder/loading_dots.html} (73%)
rename chart/files/{pod-template-file.yaml => pod-template-file.kubernetes-helm-yaml} (100%)
create mode 100644 docs/howto/operator/google/cloud/cloud_memorystore_memcached.rst
rename tests/bats/test_empty_test.bats => scripts/ci/dockerfiles/krb5-kdc-server/entrypoint.sh (83%)
mode change 100644 => 100755
create mode 100644 scripts/ci/dockerfiles/krb5-kdc-server/kadm5.acl
rename scripts/ci/dockerfiles/krb5-kdc-server/{krb-conf/server => }/kdc.conf (52%)
delete mode 100644 scripts/ci/dockerfiles/krb5-kdc-server/krb-conf/server/kadm5.acl
rename scripts/ci/dockerfiles/krb5-kdc-server/{krb-conf/client => }/krb5.conf (54%)
delete mode 100644 scripts/ci/dockerfiles/krb5-kdc-server/start_kdc.sh
copy scripts/{in_container/prod/clean-logs.sh => ci/dockerfiles/krb5-kdc-server/utils/create_admin.sh} (64%)
copy scripts/ci/{libraries/_script_init.sh => dockerfiles/krb5-kdc-server/utils/create_client.sh} (56%)
create mode 100755 scripts/ci/dockerfiles/krb5-kdc-server/utils/create_service.sh
create mode 100644 scripts/ci/dockerfiles/presto/Dockerfile
copy scripts/ci/dockerfiles/{krb5-kdc-server => presto}/build_and_push.sh (79%)
create mode 100755 scripts/ci/dockerfiles/presto/entrypoint.sh
create mode 100644 scripts/ci/libraries/_kerberos.sh
rename scripts/ci/{selective_tests.sh => selective_ci_checks.sh} (58%)
copy scripts/ci/static_checks/{run_static_checks.sh => run_basic_static_checks.sh} (68%)
delete mode 100644 scripts/in_container/krb5/krb5.conf
rename tests/bats/{ => breeze}/test_breeze_complete.bats (92%)
rename tests/bats/{ => breeze}/test_breeze_params.bats (99%)
rename tests/bats/{test_local_mounts.bats => ci/libraries/_local_mounts.bats} (98%)
copy tests/cli/commands/{test_version_command.py => test_kubernetes_command.py} (51%)