You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/01 11:06:11 UTC
[airflow] tag nightly-master updated (37eaac3 -> 7a467ce)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/airflow.git.
*** WARNING: tag nightly-master was modified! ***
from 37eaac3 (commit)
to 7a467ce (commit)
from 37eaac3 The PRs which are not approved run subset of tests (#11828)
add 3d97167 Quarantine test_exception_propagation (#11933)
add c2b4c2e Add Umami Collective as Airflow user (#11935)
add f9cd953 ci: Fix CodeQL Workflow for Javascript (#11941)
add 3455395 Update flask_wtf version to work with werkzeug>=1.0 (#11939)
add 039a86b Use Github Discussions to asking User Questions (#11940)
add e5713e0 Add drain option when canceling Dataflow pipelines (#11374)
add 9320fe6 Standardize quotes in HTML files (#11724)
add 8222851 Add Template Fields to RedshiftToS3Operator & S3ToRedshiftOperator (#11844)
add ba9c044 Add How-to guide for JDBC Operator (#11472)
add 96583e2 Remove archived link from README.md (#11945)
add a7ad204 Rename example JDBC dag (#11946)
add 2fe2011 Fix failing docs build on Master (#11951)
add 470c1b2 Fix issue rendering k8s V1Pod (#11952)
add 164a707 Fix broken master (isort fix) (#11954)
add b4b90da Fix oversized width of DAGs table with hide/reveal of "links" (#11866)
add 09a2db2 Make mypy happy with airflow.executors.local_executor (#11944)
add d8e59aa Add missing space to log message in task completion logs (#11934)
add 69b760a Cache CLI parser objects (#11957)
add b104516 Update flask-caching dep to version that works with Werkzeug 1 (#11955)
add 4bd1131 Speed up task execution in Celery by pre-loading "expensive" modules (#11956)
add 0473c9e Migration commands shouldn't print a bunch of data. (#11961)
add 44f6e6f Log instead of raise an Error for unregistered OperatorLinks (#11959)
add 1fb6741 Clean up command-line arguments (#11682)
add f000a69 Move test_serve_logs to quarantine as it fails on mysql sometimes (#11963)
add 2124453 Moves tests that should be always executed to 'always' directory (#11948)
add ecc3a4d Add autocommit property for snowflake connection (#10838)
add 1faf985 Fix: Responsive layout of DAGs (Home) view (#11958)
add 9322f3e Migrate from helm-unittest to python unittest (#11827)
add 3ad0378 Bump attrs and cattrs dependencies (#11969)
add 78cd521 Allow fractional seconds for timeout values (#11966)
add 9687b3bc Pin `kubernetes` to a max version of 11.0.0. (#11974)
add 2f703df Add SalesforceToGcsOperator (#10760)
add 69c00d9 Fixes broken ci.yaml workflow (#11981)
add 3c723e3 Move Project focus and Principles higher in the README (#11973)
add b4324bb9 Change should_response to should_respond (#11978)
add 4b5c885 Improve instalation command (#11971)
add 06597da Update doc images to reflect latest UI (#11984)
add 36b5cc6 Implements canceling of future duplicate runs (but the latest) (#11980)
add 912fe52 Fix broken link in config.yml (#11986)
add 0204eae Improve formatting in configurations docs (#11987)
add 3122b35 Added Bloomreach to the list of companies using Apache Airflow (#11995)
add b02a451 Update "click" to 7.x (#11999)
add d1df2a1 Remove unused "poke_exception_cache_ttl" param from SmartSensorOperator (#11972)
add eb023f4 Disable XCom pickling by default (#11991)
add e238b88 Validate airflow chart values.yaml & values.schema.json (#11990)
add 644ac1b Add Flower Authentication to Helm Chart (#11836)
add 4c54718 fix helm chart worker deployment without kerberos (#11681)
add 3c85c2c All k8s object must comply with JSON Schema (#12003)
add 069b1f7 fix helm scheduler deployment / scheduler logs (#11685)
add 21350aa Switches to "cancel-all-duplicates' mode of cancelling. (#12004)
add d363adb Adding SnowflakeOperator howto-documentation and example DAG (#11975)
add 72f3fcb Fix K8S CI job name rendering (#12007)
add d85a31f Adds documentation about the optimized PR workflow (#12006)
add 719af8e Remove hacktoberfest label (#12013)
add 7a467ce Adding Raisin to list of companies using Airflow (#12015)
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 -
.github/ISSUE_TEMPLATE/config.yml | 2 +-
.github/workflows/build-images-workflow-run.yml | 24 +-
.github/workflows/ci.yml | 18 +-
.github/workflows/codeql-analysis.yml | 4 +-
.pre-commit-config.yaml | 12 +-
CI.rst | 101 +------
CONTRIBUTING.rst | 2 +-
INTHEWILD.md | 3 +
PULL_REQUEST_WORKFLOW.rst | 260 +++++++++++++++++
README.md | 35 ++-
UPDATING.md | 10 +-
airflow/cli/cli_parser.py | 18 +-
airflow/config_templates/config.yml | 77 +++--
airflow/config_templates/default_airflow.cfg | 72 +++--
airflow/executors/celery_executor.py | 28 +-
airflow/executors/local_executor.py | 1 +
.../2c6edca13270_resource_based_permissions.py | 6 -
airflow/models/dagbag.py | 2 +-
airflow/models/taskinstance.py | 6 +-
airflow/models/xcom.py | 11 -
.../amazon/aws/transfers/redshift_to_s3.py | 2 +-
.../amazon/aws/transfers/s3_to_redshift.py | 5 +-
airflow/providers/dependencies.json | 1 +
..._ads_to_gcs.py => example_salesforce_to_gcs.py} | 94 +++----
airflow/providers/google/cloud/hooks/dataflow.py | 23 +-
.../providers/google/cloud/operators/dataflow.py | 25 +-
.../google/cloud/transfers/salesforce_to_gcs.py | 125 +++++++++
.../providers/jdbc/example_dags}/__init__.py | 0
.../jdbc/example_dags/example_jdbc_queries.py} | 61 ++--
.../snowflake/example_dags/example_snowflake.py | 105 ++++---
airflow/providers/snowflake/hooks/snowflake.py | 4 +
airflow/sensors/smart_sensor_operator.py | 11 +-
airflow/serialization/serialized_objects.py | 3 +-
airflow/utils/json.py | 2 +-
airflow/utils/timeout.py | 4 +-
airflow/www/static/css/dags.css | 126 +++++++++
airflow/www/static/css/main.css | 15 +-
airflow/www/templates/airflow/dags.html | 269 +++++++++---------
airflow/www/templates/airflow/task_instance.html | 8 +-
airflow/www/views.py | 4 +-
airflow/www/webpack.config.js | 1 +
chart/README.md | 2 +-
chart/files/pod-template-file.kubernetes-helm-yaml | 2 +-
chart/templates/_helpers.yaml | 4 +
chart/templates/flower/flower-deployment.yaml | 29 +-
chart/templates/redis/redis-statefulset.yaml | 4 +-
.../templates/scheduler/scheduler-deployment.yaml | 2 +-
.../scheduler/scheduler-serviceaccount.yaml | 6 +-
...uncer-config-secret.yaml => flower-secret.yaml} | 9 +-
chart/templates/secrets/redis-secrets.yaml | 3 +
chart/templates/webserver/webserver-ingress.yaml | 6 +-
chart/templates/webserver/webserver-service.yaml | 2 +-
.../webserver/webserver-serviceaccount.yaml | 10 +-
chart/templates/workers/worker-deployment.yaml | 2 +
chart/templates/workers/worker-serviceaccount.yaml | 12 +-
chart/tests/celery-kubernetes-executor_test.yaml | 51 ----
.../tests/celery-kubernetes-pod-launcher-role.yaml | 29 --
chart/tests/conftest.py | 6 +-
chart/tests/dags-persistent-volume-claim_test.yaml | 64 -----
chart/tests/git-sync-webserver_test.yaml | 73 -----
chart/tests/git-sync-worker_test.yaml | 76 -----
chart/tests/helm_template_generator.py | 37 +++
chart/tests/migrate-database-job_test.yaml | 27 --
chart/tests/pod-template-file_test.yaml | 171 -----------
chart/tests/scheduler_test.yaml | 38 ---
chart/tests/test_basic_helm_chart.py | 59 +++-
chart/tests/test_celery_kubernetes_executor.py | 47 ++++
.../test_celery_kubernetes_pod_launcher_role.py | 24 +-
.../tests/test_chart_quality.py | 32 ++-
chart/tests/test_dags_persistent_volume_claim.py | 72 +++++
chart/tests/test_flower_authorization.py | 68 +++++
chart/tests/test_git_sync_scheduler.py | 311 ++++++++-------------
chart/tests/test_git_sync_webserver.py | 62 ++++
chart/tests/test_git_sync_worker.py | 75 +++++
.../tests/test_kerberos.py | 20 +-
.../tests/test_migrate_database_job.py | 25 +-
chart/tests/test_pod_template_file.py | 185 ++++++++++++
.../tests/test_scheduler.py | 39 ++-
.../configuration.py => chart/tests/test_worker.py | 39 ++-
chart/tests/worker_test.yaml | 38 ---
chart/values.schema.json | 241 +++++++++++-----
chart/values.yaml | 11 +-
docs/howto/connection/jdbc.rst | 2 +
...cebook_ads_to_gcs.rst => salesforce_to_gcs.rst} | 21 +-
docs/howto/operator/index.rst | 2 +
docs/howto/operator/jdbc.rst | 97 +++++++
docs/howto/operator/snowflake.rst | 124 ++++++++
docs/img/add-dag-tags.png | Bin 79600 -> 68537 bytes
docs/img/add-role.png | Bin 68070 -> 30509 bytes
docs/img/dags.png | Bin 297947 -> 211122 bytes
docs/img/example_passing_conf.png | Bin 97482 -> 41080 bytes
docs/img/operator_extra_link.png | Bin 276679 -> 90359 bytes
docs/img/subdag_zoom.png | Bin 255915 -> 145017 bytes
docs/img/ui-timezone-chooser.png | Bin 12548 -> 16644 bytes
docs/installation.rst | 18 +-
docs/operators-and-hooks-ref.rst | 11 +-
images/pr/pr-full-tests-needed.png | Bin 0 -> 88512 bytes
images/pr/pr-likely-ok-to-merge.png | Bin 0 -> 98362 bytes
images/pr/pr-no-tests-needed-comment.png | Bin 0 -> 80852 bytes
images/pr/selective_checks.md5 | 1 +
images/pr/selective_checks.mermaid | 35 +++
images/pr/selective_checks.png | Bin 0 -> 64501 bytes
.../templates/metastore_browser/base.html | 2 +-
scripts/ci/kubernetes/ci_run_helm_testing.sh | 29 --
scripts/ci/selective_ci_checks.sh | 4 +-
scripts/in_container/entrypoint_ci.sh | 5 +
setup.py | 14 +-
tests/{core => always}/test_example_dags.py | 0
tests/{core => always}/test_project_structure.py | 0
.../endpoints/test_config_endpoint.py | 6 +-
.../endpoints/test_connection_endpoint.py | 26 +-
tests/api_connexion/endpoints/test_dag_endpoint.py | 38 +--
.../endpoints/test_dag_run_endpoint.py | 12 +-
.../endpoints/test_dag_source_endpoint.py | 8 +-
.../endpoints/test_event_log_endpoint.py | 6 +-
.../endpoints/test_extra_link_endpoint.py | 12 +-
tests/api_connexion/endpoints/test_log_endpoint.py | 4 +-
.../api_connexion/endpoints/test_task_endpoint.py | 10 +-
.../endpoints/test_task_instance_endpoint.py | 16 +-
.../endpoints/test_variable_endpoint.py | 6 +-
.../endpoints/test_version_endpoint.py | 2 +-
.../api_connexion/endpoints/test_xcom_endpoint.py | 8 +-
tests/cli/commands/test_task_command.py | 4 +-
tests/executors/test_celery_executor.py | 1 +
.../amazon/aws/transfers/test_redshift_to_s3.py | 5 +
.../amazon/aws/transfers/test_s3_to_redshift.py | 5 +
.../providers/google/cloud/hooks/test_dataflow.py | 60 ++++
.../google/cloud/operators/test_mlengine_utils.py | 2 +-
.../cloud/transfers/test_salesforce_to_gcs.py | 94 +++++++
..._system.py => test_salesforce_to_gcs_system.py} | 31 +-
tests/serialization/test_dag_serialization.py | 36 ++-
.../salesforce_system_helpers.py} | 42 +--
tests/utils/test_json.py | 23 ++
tests/utils/test_serve_logs.py | 2 +
135 files changed, 2728 insertions(+), 1690 deletions(-)
create mode 100644 PULL_REQUEST_WORKFLOW.rst
copy airflow/providers/google/cloud/example_dags/{example_facebook_ads_to_gcs.py => example_salesforce_to_gcs.py} (55%)
create mode 100644 airflow/providers/google/cloud/transfers/salesforce_to_gcs.py
copy {tests/providers/zendesk/hooks => airflow/providers/jdbc/example_dags}/__init__.py (100%)
copy airflow/{example_dags/example_bash_operator.py => providers/jdbc/example_dags/example_jdbc_queries.py} (57%)
create mode 100644 airflow/www/static/css/dags.css
copy chart/templates/secrets/{pgbouncer-config-secret.yaml => flower-secret.yaml} (81%)
delete mode 100644 chart/tests/celery-kubernetes-executor_test.yaml
delete mode 100644 chart/tests/celery-kubernetes-pod-launcher-role.yaml
delete mode 100644 chart/tests/dags-persistent-volume-claim_test.yaml
delete mode 100644 chart/tests/git-sync-webserver_test.yaml
delete mode 100644 chart/tests/git-sync-worker_test.yaml
delete mode 100644 chart/tests/migrate-database-job_test.yaml
delete mode 100644 chart/tests/pod-template-file_test.yaml
delete mode 100644 chart/tests/scheduler_test.yaml
create mode 100644 chart/tests/test_celery_kubernetes_executor.py
copy tests/providers/apache/hive/sensors/test_hdfs.py => chart/tests/test_celery_kubernetes_pod_launcher_role.py (57%)
copy tests/providers/apache/hive/sensors/test_hdfs.py => chart/tests/test_chart_quality.py (51%)
create mode 100644 chart/tests/test_dags_persistent_volume_claim.py
create mode 100644 chart/tests/test_flower_authorization.py
create mode 100644 chart/tests/test_git_sync_webserver.py
create mode 100644 chart/tests/test_git_sync_worker.py
copy tests/utils/test_weight_rule.py => chart/tests/test_kerberos.py (58%)
copy tests/providers/amazon/aws/hooks/test_sqs.py => chart/tests/test_migrate_database_job.py (62%)
create mode 100644 chart/tests/test_pod_template_file.py
copy airflow/utils/configuration.py => chart/tests/test_scheduler.py (50%)
copy airflow/utils/configuration.py => chart/tests/test_worker.py (50%)
delete mode 100644 chart/tests/worker_test.yaml
copy docs/howto/operator/google/transfer/{facebook_ads_to_gcs.rst => salesforce_to_gcs.rst} (65%)
create mode 100644 docs/howto/operator/jdbc.rst
create mode 100644 docs/howto/operator/snowflake.rst
create mode 100644 images/pr/pr-full-tests-needed.png
create mode 100644 images/pr/pr-likely-ok-to-merge.png
create mode 100644 images/pr/pr-no-tests-needed-comment.png
create mode 100644 images/pr/selective_checks.md5
create mode 100644 images/pr/selective_checks.mermaid
create mode 100644 images/pr/selective_checks.png
delete mode 100755 scripts/ci/kubernetes/ci_run_helm_testing.sh
rename tests/{core => always}/test_example_dags.py (100%)
rename tests/{core => always}/test_project_structure.py (100%)
create mode 100644 tests/providers/google/cloud/transfers/test_salesforce_to_gcs.py
copy tests/providers/google/cloud/transfers/{test_bigquery_to_gcs_system.py => test_salesforce_to_gcs_system.py} (65%)
copy tests/{providers/google/cloud/transfers/test_facebook_ads_to_gcs_system.py => test_utils/salesforce_system_helpers.py} (51%)