You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by gi...@apache.org on 2020/11/13 02:38:30 UTC
[airflow] tag nightly-master updated (08d67ad -> 32b59f8)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 08d67ad (commit)
to 32b59f8 (commit)
from 08d67ad Beautify Output of setup-installation pre-commit (#12218)
add f37c6e6 Add Compute Engine SSH hook (#9879)
add 7463b6b Add Markdown linting to pre-commit (#11465)
add 58d60ab Enable Markdownlint rule - MD032/blanks-around-lists (#12224)
add 0443f04 Fix typo (#12222)
add dd2095f Simplify string expressions & Use f-string (#12216)
add cd85d01 [#12012]: Update INTHEWILD.md (#12235)
add 75065ac CSS Changes to adjust content width as per screen size and responsive table with multiline td. (#12227)
add 502ba30 Enable Markdownlint rule - MD022/blanks-around-headings (#12225)
add 50fedfb Add docs about security on GCP (#12187)
add 250436d Fix spelling in Python files (#12230)
add 45587a6 Enable markdownlint rule - MD031/blanks-around-fences (#12238)
add 5912d0c Fixes automated provider installation with extras (#12233)
add 249d174 Add back missing api_connextion/__init__.py file (#12240)
add 3ab5828 Sign release files with an apache.org key by default (#12241)
add f8ae6e5 Remove Unnecessary comprehension (#12221)
add 5ac1738 Update versions in UPDATING.md for 2.0.0b1 release (#12244)
add c5806ef Added missing sendgrid readme (#12245)
add 1521965 Release 2.0.0b2 (#12243)
add 3ddf4d0 Fix spelling (#12250)
add 09febee Fixes continuous image rebuilding with Breeze (#12256)
add ae397df Adds provider package documentation in installation.rst (#12203)
add cf08eb2 Added Farfetch to the list of companies using Airflow (#12260)
add 4f9439d Fix spelling (#12253)
add 0cd1c84 Remove providers imports from core examples (#12252)
add 938c512 Fix: Conditionally update button URL only when it is present (#12268)
add e0b7cae Fix spelling (#12266)
add 7d5d334 Fix pause/unpause toggle to display failed state when unsuccessful (#12267)
add 9b7e760 Docker context files should be available earlier (#12219)
add e03a3f4 Unify user session lifetime configuration (#11970)
add cbf4984 Don't treat warning message as rich formatting codes. (#12283)
add 348510f Providers in extras are properly configured and verified (#12265)
add 21999dd Added k9s as integrated tool to help with kubernetes testing (#12163)
add 0d51a12 Don't wrap warrning messages when stderr is not a TTY (#12285)
add cbe4ef2 Fix indentation for affinities in helm chart (#12288)
add 6b5640d Fix test - TestImpersonation (#12274)
add 3cd317e Remove unneeded parentheses from Python files (#12270)
add fbf874a Add an alias to improve git shortlog output (#12286)
add 388736b Detect partial examples DAGs for Google (#12277)
add 0d37c59 Make dag_id, task_id, and execution_date nullable in event log schema (#12287)
add 7478e18 Handle naive datetimes in REST APIi (#12248)
add 289c9b5 Use default view in TriggerDagRunLink (#11778)
add 4f5e0ed Update INTHEWILD.md (#12293)
add 9276607 Add session_parameters option to snowflake_hook (#12071)
add fa2b033 Add reference for SubDagOperator (#12297)
add ee544b4 Replace remaining decorated DAGs reference (#12299)
add af2f2e8 Wait option for dagrun operator (#12126)
add 3c2c291 Python base image is shared between CI and PROD image (#12280)
add ae93fdb Remove deprecated BashTaskRunner (#12295)
add 5d5c119 Remove deprecated Elasticsearch Configs (#12296)
add 82eef2e Fix - TestSchedulerJobQueriesCount::test_process_dags_queries_count (#12273)
add dd9ac78 Fix prod image build (#12314)
add af19b12 Deploy was not working from Breeze (#12319)
add 7f828b0 Get all "tags" parameters not just one (#12324)
add 571f831 Update automated PR labels (#12326)
add 32b59f8 Fixes the sending of an empty list to BigQuery `list_rows` (#12307)
No new revisions were added by this update.
Summary of changes:
.github/boring-cyborg.yml | 6 +-
.github/workflows/ci.yml | 17 +-
.gitignore | 2 +
.mailmap | 3 +
.../test_pubsub_system.py => .markdownlint.yml | 61 +++-
.pre-commit-config.yaml | 28 +-
BREEZE.rst | 26 +-
CONTRIBUTING.rst | 8 +-
Dockerfile.ci | 8 +-
INSTALL | 6 +-
INTHEWILD.md | 3 +
README.md | 2 +
STATIC_CODE_CHECKS.rst | 10 +-
TESTING.rst | 151 ++++++++-
UPDATING.md | 129 +++++++-
UPGRADING_TO_2.0.md | 50 ++-
.../hooks => airflow/api_connexion}/__init__.py | 0
.../api_connexion/endpoints/dag_run_endpoint.py | 6 +-
airflow/api_connexion/openapi/v1.yaml | 3 +
airflow/api_connexion/parameters.py | 6 +
airflow/api_connexion/schemas/dag_run_schema.py | 15 +-
.../api_connexion/schemas/task_instance_schema.py | 25 +-
airflow/cli/cli_parser.py | 8 +-
airflow/cli/commands/user_command.py | 2 +-
airflow/config_templates/config.yml | 19 +-
airflow/config_templates/default_airflow.cfg | 9 +-
airflow/configuration.py | 12 +-
airflow/example_dags/example_dag_decorator.py | 29 +-
airflow/executors/kubernetes_executor.py | 12 +-
.../2c6edca13270_resource_based_permissions.py | 4 +-
airflow/models/baseoperator.py | 2 +-
airflow/models/connection.py | 4 +
airflow/models/dagrun.py | 2 +-
airflow/models/log.py | 2 +-
airflow/operators/dagrun_operator.py | 49 ++-
airflow/operators/email.py | 1 +
airflow/operators/python.py | 3 +
.../amazon/aws/example_dags/example_datasync_1.py | 2 +-
.../amazon/aws/example_dags/example_datasync_2.py | 2 +-
airflow/providers/apache/druid/hooks/druid.py | 2 +-
.../hive/example_dags/example_twitter_README.md | 1 +
.../apache/hive/sensors/named_hive_partition.py | 2 +-
airflow/providers/dependencies.json | 3 +-
airflow/providers/google/cloud/ADDITIONAL_INFO.md | 1 +
.../cloud/example_dags/example_compute_ssh.py | 89 ++++++
.../cloud/example_dags/example_stackdriver.py | 2 +-
airflow/providers/google/cloud/hooks/bigquery.py | 14 +-
airflow/providers/google/cloud/hooks/compute.py | 80 ++++-
.../providers/google/cloud/hooks/compute_ssh.py | 307 ++++++++++++++++++
airflow/providers/google/cloud/hooks/gcs.py | 4 +-
airflow/providers/google/cloud/hooks/kms.py | 2 +-
airflow/providers/google/cloud/hooks/mlengine.py | 8 +-
airflow/providers/google/cloud/hooks/os_login.py | 87 ++++++
.../google/cloud/operators/speech_to_text.py | 4 +-
.../google/cloud/transfers/local_to_gcs.py | 2 +-
.../google/cloud/transfers/postgres_to_gcs.py | 4 +-
.../cloud/utils/mlengine_prediction_summary.py | 2 +-
.../providers/google/common/hooks/base_google.py | 18 ++
.../providers/microsoft/azure/hooks/azure_batch.py | 2 +-
airflow/providers/redis/operators/redis_publish.py | 2 +-
.../providers/sendgrid/PROVIDER_CHANGES_1.0.0b1.md | 12 +
airflow/providers/sendgrid/README.md | 71 +++++
airflow/providers/slack/hooks/slack.py | 2 +-
airflow/providers/slack/hooks/slack_webhook.py | 2 +-
airflow/providers/snowflake/hooks/snowflake.py | 3 +
airflow/providers/snowflake/operators/snowflake.py | 6 +
airflow/providers/ssh/hooks/ssh.py | 4 +-
airflow/sensors/external_task_sensor.py | 5 +-
airflow/settings.py | 19 +-
airflow/utils/helpers.py | 12 +
airflow/version.py | 2 +-
airflow/www/app.py | 24 +-
airflow/www/extensions/init_session.py | 20 --
airflow/www/static/css/switch.css | 16 +-
airflow/www/templates/airflow/dag.html | 37 +--
airflow/www/templates/airflow/dags.html | 37 +--
airflow/www/utils.py | 1 +
breeze | 9 +
breeze-complete | 5 +-
chart/README.md | 8 +-
chart/files/pod-template-file.kubernetes-helm-yaml | 9 +-
chart/requirements.lock | 4 +-
chart/tests/test_celery_kubernetes_executor.py | 2 +-
chart/tests/test_git_sync_webserver.py | 4 +-
chart/tests/test_git_sync_worker.py | 4 +-
chart/tests/test_pod_template_file.py | 51 +++
clients/README.md | 1 +
dev/PROVIDER_PACKAGES.md | 4 +-
dev/README.md | 18 +-
dev/import_all_classes.py | 2 +-
dev/provider_packages/README.md | 1 +
dev/sign.sh | 6 +-
docs/concepts.rst | 20 +-
docs/conf.py | 8 -
docs/howto/connection/aws.rst | 12 +
docs/howto/connection/gcp.rst | 2 +
docs/howto/connection/gcp_ssh.rst | 73 +++++
docs/howto/operator/google/cloud/compute_ssh.rst | 72 +++++
docs/img/aws-web-identity-federation-gcp.png | Bin 0 -> 40170 bytes
docs/installation.rst | 125 +++++++-
docs/operators-and-hooks-ref.rst | 11 +
docs/production-deployment.rst | 43 +++
docs/spelling_wordlist.txt | 2 +-
docs/static/exampleinclude.css | 12 +
images/testing/k9s.png | Bin 0 -> 238713 bytes
images/testing/kubeconfig-env.png | Bin 0 -> 231280 bytes
images/testing/kubernetes-virtualenv.png | Bin 0 -> 110011 bytes
images/testing/pytest-runner.png | Bin 0 -> 131589 bytes
images/testing/run-test.png | Bin 0 -> 140728 bytes
kubernetes_tests/test_kubernetes_executor.py | 7 +-
kubernetes_tests/test_kubernetes_pod_operator.py | 9 +-
.../krb5-kdc-server/utils/create_admin.sh | 2 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 2 -
...up_cluster_and_deploy_airflow_to_kubernetes.sh} | 3 +-
scripts/ci/kubernetes/kind-cluster-conf.yaml | 5 +
.../ci/kubernetes/nodeport.yaml | 20 +-
...oy_app_to_kubernetes.sh => redeploy_airflow.sh} | 6 +-
scripts/ci/libraries/_kind.sh | 125 ++++----
scripts/ci/libraries/_push_pull_remove_images.sh | 9 +-
.../pre_commit_check_extras_have_providers.py | 68 ++++
scripts/in_container/quarantine_issue_header.md | 1 +
setup.py | 28 +-
tests/always/test_project_structure.py | 149 ++++++++-
.../endpoints/test_dag_run_endpoint.py | 49 ++-
.../endpoints/test_event_log_endpoint.py | 16 +-
.../endpoints/test_task_instance_endpoint.py | 95 ++++++
tests/api_connexion/test_parameters.py | 22 +-
tests/core/test_configuration.py | 16 +-
tests/core/test_core.py | 2 +-
tests/core/test_impersonation_tests.py | 7 +-
tests/dags/README.md | 1 +
tests/dags_corrupted/README.md | 3 +-
tests/jobs/test_scheduler_job.py | 56 ++--
tests/operators/test_dagrun_operator.py | 36 ++-
tests/providers/amazon/aws/sensors/test_sqs.py | 2 +-
tests/providers/apache/druid/hooks/test_druid.py | 10 +-
.../elasticsearch/log/test_es_task_handler.py | 2 +-
.../providers/google/cloud/hooks/test_bigquery.py | 21 ++
.../google/cloud/hooks/test_compute_ssh.py | 342 +++++++++++++++++++++
.../test_compute_ssh_system.py} | 65 ++--
.../providers/google/cloud/hooks/test_dataflow.py | 2 +-
.../google/cloud/hooks/test_kubernetes_engine.py | 2 +-
.../providers/google/cloud/hooks/test_os_login.py | 176 +++++++++++
.../operators/test_cloud_sql_system_helper.py | 4 +-
.../google/cloud/operators/test_compute.py | 4 +-
.../google/cloud/utils/gcp_authenticator.py | 1 +
tests/providers/snowflake/hooks/test_snowflake.py | 3 +-
tests/test_utils/README.md | 1 +
tests/utils/test_helpers.py | 13 +-
tests/utils/test_operator_helpers.py | 10 +-
tests/www/test_app.py | 25 ++
151 files changed, 3041 insertions(+), 523 deletions(-)
copy tests/providers/google/cloud/operators/test_pubsub_system.py => .markdownlint.yml (53%)
copy {tests/providers/zendesk/hooks => airflow/api_connexion}/__init__.py (100%)
create mode 100644 airflow/providers/google/cloud/example_dags/example_compute_ssh.py
create mode 100644 airflow/providers/google/cloud/hooks/compute_ssh.py
create mode 100644 airflow/providers/google/cloud/hooks/os_login.py
create mode 100644 airflow/providers/sendgrid/PROVIDER_CHANGES_1.0.0b1.md
create mode 100644 airflow/providers/sendgrid/README.md
create mode 100644 docs/howto/connection/gcp_ssh.rst
create mode 100644 docs/howto/operator/google/cloud/compute_ssh.rst
create mode 100644 docs/img/aws-web-identity-federation-gcp.png
create mode 100644 images/testing/k9s.png
create mode 100644 images/testing/kubeconfig-env.png
create mode 100644 images/testing/kubernetes-virtualenv.png
create mode 100644 images/testing/pytest-runner.png
create mode 100644 images/testing/run-test.png
copy scripts/ci/kubernetes/{ci_deploy_app_to_kubernetes.sh => ci_setup_cluster_and_deploy_airflow_to_kubernetes.sh} (94%)
copy tests/kubernetes/basic_pod.yaml => scripts/ci/kubernetes/nodeport.yaml (77%)
rename scripts/ci/kubernetes/{ci_deploy_app_to_kubernetes.sh => redeploy_airflow.sh} (86%)
create mode 100755 scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
create mode 100644 tests/providers/google/cloud/hooks/test_compute_ssh.py
copy tests/providers/google/cloud/{operators/test_bigtable_system.py => hooks/test_compute_ssh_system.py} (50%)
create mode 100644 tests/providers/google/cloud/hooks/test_os_login.py